fix for files with coding cookies and BOMs

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 89721371b998db1ddff4b283799ad1164b0663b9..519dfa5784927c5872f2b1cc82102a456d9f6ef4 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -333,9 +333,11 @@ def detect_encoding(readline):
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
-        if bom_found and codec.name != 'utf-8':
-            # This behaviour mimics the Python interpreter
-            raise SyntaxError('encoding problem: utf-8')
+        if bom_found:
+            if codec.name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
         return encoding
 
     first = read_or_stop()