- Finally fixed the bug in compile() and exec where a string ending

  with an indented code block but no newline would raise SyntaxError.
  This would have been a four-line change in parsetok.c...  Except
  codeop.py depends on this behavior, so a compilation flag had to be
  invented that causes the tokenizer to revert to the old behavior;
  this required extra changes to 2 .h files, 2 .c files, and 2 .py
  files.  (Fixes SF bug #501622.)

diff --git a/Include/parsetok.h b/Include/parsetok.h
index 58ef76ab69dcf2112d9fd7a9e8fb2ff9da38de1d..99a79b700914fc69c93899f4827e1d931331c04b 100644 (file)
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -21,6 +21,8 @@ typedef struct {
 #define PyPARSE_YIELD_IS_KEYWORD       0x0001
 #endif
 
+#define PyPARSE_DONT_IMPLY_DEDENT      0x0002
+
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
 PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 87d787416696a8081ca4545c0790081c6a029751..746ba7ed93cc98e86064535d7cd1d319d7421e08 100644 (file)
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -10,6 +10,7 @@ extern "C" {
 #define PyCF_MASK (CO_FUTURE_DIVISION)
 #define PyCF_MASK_OBSOLETE (CO_GENERATOR_ALLOWED | CO_NESTED)
 #define PyCF_SOURCE_IS_UTF8  0x0100
+#define PyCF_DONT_IMPLY_DEDENT 0x0200
 
 typedef struct {
        int cf_flags;  /* bitmask of CO_xxx flags relevant to future */

diff --git a/Lib/code.py b/Lib/code.py
index 75c64e60e4bbaf2b11303d37d6d027ea1f041491..6bdc658add03d8cea96ca87f7fbefbb86dc859f2 100644 (file)
--- a/Lib/code.py
+++ b/Lib/code.py
@@ -303,4 +303,5 @@ def interact(banner=None, readfunc=None, local=None):
 
 
 if __name__ == '__main__':
-    interact()
+    import pdb
+    pdb.run("interact()\n")

diff --git a/Lib/codeop.py b/Lib/codeop.py
index c97b4df2935407ea273bdbba2fc3363101abff5e..cc9d5b26f174f978dc8e2885623e848c3c9fc271 100644 (file)
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -63,6 +63,8 @@ _features = [getattr(__future__, fname)
 
 __all__ = ["compile_command", "Compile", "CommandCompiler"]
 
+PyCF_DONT_IMPLY_DEDENT = 0x200          # Matches pythonrun.h
+
 def _maybe_compile(compiler, source, filename, symbol):
     # Check for source consisting of only blank lines and comments
     for line in source.split("\n"):
@@ -103,6 +105,9 @@ def _maybe_compile(compiler, source, filename, symbol):
     if not code1 and e1 == e2:
         raise SyntaxError, err1
 
+def _compile(source, filename, symbol):
+    return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
+
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -121,7 +126,7 @@ def compile_command(source, filename="<input>", symbol="single"):
       syntax error (OverflowError and ValueError can be produced by
       malformed literals).
     """
-    return _maybe_compile(compile, source, filename, symbol)
+    return _maybe_compile(_compile, source, filename, symbol)
 
 class Compile:
     """Instances of this class behave much like the built-in compile
@@ -129,7 +134,7 @@ class Compile:
     statement, it "remembers" and compiles all subsequent program texts
     with the statement in force."""
     def __init__(self):
-        self.flags = 0
+        self.flags = PyCF_DONT_IMPLY_DEDENT
 
     def __call__(self, source, filename, symbol):
         codeob = compile(source, filename, symbol, self.flags, 1)

diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index a0ce0bcb7aa5e05824b685afed2d5ecf5604ee57..7af83ebbcc882da8743de03f381682e9b5677118 100644 (file)
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -5,13 +5,13 @@
 import unittest
 from test.test_support import run_unittest
 
-from codeop import compile_command
+from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 
 class CodeopTests(unittest.TestCase):
 
     def assertValid(self, str, symbol='single'):
         '''succeed iff str is a valid piece of code'''
-        expected = compile(str, "<input>", symbol)
+        expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT)
         self.assertEquals( compile_command(str, "<input>", symbol), expected)
 
 
@@ -42,7 +42,8 @@ class CodeopTests(unittest.TestCase):
 
         # special case
         self.assertEquals(compile_command(""),
-                          compile("pass", "<input>", 'single'))
+                          compile("pass", "<input>", 'single',
+                                  PyCF_DONT_IMPLY_DEDENT))
 
         av("3**3","eval")
         av("(lambda z: \n z**3)","eval")

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 1fe7f5263ffdca4594f90cb89c31462239123cf4..1812fbc3f89d37178010d85b72d761b7ce216286 100644 (file)
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -89,6 +89,15 @@ expect_error("2.0e+")
 expect_error("1e-")
 expect_error("3-4e/21")
 
+if verbose:
+    print "testing compile() of indented block w/o trailing newline"
+
+s = """
+if 1:
+    if 2:
+        pass"""
+compile(s, "<string>", "exec")
+
 
 if verbose:
     print "testing literals with leading zeroes"

diff --git a/Misc/NEWS b/Misc/NEWS
index dc6f1dbf444f6c0240e2aa4fce69743f3a3f30de..f2069858308d96fa67808ec5390c82a99cecf929 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,14 @@ What's New in Python 2.3 alpha 2?
 Core and builtins
 -----------------
 
+- Finally fixed the bug in compile() and exec where a string ending
+  with an indented code block but no newline would raise SyntaxError.
+  This would have been a four-line change in parsetok.c...  Except
+  codeop.py depends on this behavior, so a compilation flag had to be
+  invented that causes the tokenizer to revert to the old behavior;
+  this required extra changes to 2 .h files, 2 .c files, and 2 .py
+  files.
+
 - If a new-style class defines neither __new__ nor __init__, its
   constructor would ignore all arguments.  This is changed now: the
   constructor refuses arguments in this case.  This might break code

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 82a5450d8f71d3de2a970c454ff220209bacaf01..0b3e957e0ca913f9f8dad3a7952dfaa65c7d0afe 100644 (file)
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -130,6 +130,15 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
                if (type == ENDMARKER && started) {
                        type = NEWLINE; /* Add an extra newline */
                        started = 0;
+                       /* Add the right number of dedent tokens,
+                          except if a certain flag is given --
+                          codeop.py uses this. */
+                       if (tok->indent &&
+                           !(flags & PyPARSE_DONT_IMPLY_DEDENT))
+                       {
+                               tok->pendin = -tok->indent;
+                               tok->indent = 0;
+                       }
                }
                else
                        started = 1;

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 338e38d70a7b17739de0328dae5464b62ada2e2f..62aa5121f5bbb04eef6141a07d799ac6006feb07 100644 (file)
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -380,7 +380,9 @@ builtin_compile(PyObject *self, PyObject *args)
                return NULL;
        }
 
-       if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE)) {
+       if (supplied_flags &
+           ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT))
+       {
                PyErr_SetString(PyExc_ValueError,
                                "compile(): unrecognised flags");
                return NULL;

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index fa72fe8813f6d1877286d12683e6e9ec6206f61e..1faab509ba9d6a1afcd26eed65f726ff541f02c0 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -548,13 +548,9 @@ PyRun_InteractiveOne(FILE *fp, const char *filename)
 }
 
 /* compute parser flags based on compiler flags */
-#if 0 /* future keyword */
 #define PARSER_FLAGS(flags) \
-       (((flags) && (flags)->cf_flags & CO_GENERATOR_ALLOWED) ? \
-               PyPARSE_YIELD_IS_KEYWORD : 0)
-#else
-#define PARSER_FLAGS(flags) 0
-#endif
+       (((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
+               PyPARSE_DONT_IMPLY_DEDENT : 0)
 
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)