From: Raymond Hettinger <python@rcn.com>
Date: Tue, 21 Jun 2005 07:53:56 +0000 (+0000)
Subject: SF bug #1224621: tokenize module does not detect inconsistent dedents
X-Git-Tag: v2.4.2c1~175
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=1ec9c52de39a94fb6607f8698c9309129ac471f0;p=thirdparty%2FPython%2Fcpython.git

SF bug #1224621: tokenize module does not detect inconsistent dedents
---

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index d21740468f17..5dd20d36f9f0 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,4 +1,4 @@
-from test.test_support import verbose, findfile
+from test.test_support import verbose, findfile, TestFailed
 import tokenize, os, sys
 
 if verbose:
@@ -10,3 +10,21 @@ f.close()
 
 if verbose:
     print 'finished'
+
+###### Test detecton of IndentationError ######################
+
+from cStringIO import StringIO
+
+sampleBadText = """
+def foo():
+    bar
+  baz
+"""
+
+try:
+    for tok in tokenize.generate_tokens(StringIO(sampleBadText).readline):
+        pass
+except IndentationError:
+    pass
+else:
+    raise TestFailed("Did not detect IndentationError:")
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 9087e84ca0e2..fc97a2874e0e 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -225,6 +225,9 @@ def generate_tokens(readline):
                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
             while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level")
                 indents = indents[:-1]
                 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
 
diff --git a/Misc/NEWS b/Misc/NEWS
index b4e626f59edf..76ec7c572ecf 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -39,6 +39,8 @@ Extension Modules
 Library
 -------
 
+- Bug #1224621: tokenize module does not detect inconsistent dedents
+
 - Bug #1196315: fix weakref.WeakValueDictionary constructor.
 
 - Bug #1213894: os.path.realpath didn't resolve symlinks that were the first