diff options
| author |   Jason R. Coombs <jaraco@jaraco.com> | 2015-06-28 11:13:30 -0400 |
|---|---|---|
| committer | Jason R. Coombs <jaraco@jaraco.com> | 2015-06-28 11:13:30 -0400 |
| commit | a95a476b3ae93d890209e592d675ae64c82e05dc (patch) | |
| tree | a8769511b04f31f00521a2e4e0f3d42d8e561dee /Lib/tokenize.py | |
| parent | Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() (diff) | |
| parent | Issue #20387: Merge (diff) | |
| download | cpython-a95a476b3ae93d890209e592d675ae64c82e05dc.tar.gz cpython-a95a476b3ae93d890209e592d675ae64c82e05dc.tar.bz2 cpython-a95a476b3ae93d890209e592d675ae64c82e05dc.zip | |
Issue #20387: Merge test and patch from 3.4.4
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index f58c2869017..3ec90184bee 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ class Untokenizer: def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -254,6 +256,21 @@ class Untokenizer: continue if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end |