diff options
| author |   Albert-Jan Nijburg <albertjan@trinket.io> | 2017-05-24 12:31:57 +0100 |
|---|---|---|
| committer |   Serhiy Storchaka <storchaka@gmail.com> | 2017-05-24 14:31:57 +0300 |
| commit | c471ca448cf336d7eb4a7cbe14d0012baf122d1f (patch) | |
| tree | 6159481f56939bfe00238eb2091cb511f164548e /Lib/tokenize.py | |
| parent | bpo-28707: Add the directory parameter to http.server.SimpleHTTPRequestHandle... (diff) | |
| download | cpython-c471ca448cf336d7eb4a7cbe14d0012baf122d1f.tar.gz cpython-c471ca448cf336d7eb4a7cbe14d0012baf122d1f.tar.bz2 cpython-c471ca448cf336d7eb4a7cbe14d0012baf122d1f.zip | |
bpo-30377: Simplify handling of COMMENT and NL in tokenize.py (#1607)
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 8 |
1 files changed, 3 insertions, 5 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 634662da265..9017bb13e78 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -560,13 +560,11 @@ def _tokenize(readline, encoding): if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) yield TokenInfo(COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token)), line) - yield TokenInfo(NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:], + pos += len(comment_token) + + yield TokenInfo(NL, line[pos:], (lnum, pos), (lnum, len(line)), line) continue |