Make the example a little more interesting and useful.

diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index c627f1a833a1cf950705478e49f7faf74a687808..423540c48bdaa0287726e3fc7bf1b9e24bfa7f32 100644 (file)
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -1300,6 +1300,7 @@ successive matches::
     Token = collections.namedtuple('Token', 'typ value line column')
 
     def tokenize(s):
+        keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}
         tok_spec = [
             ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number
             ('ASSIGN', r':='),          # Assignment operator
@@ -1320,6 +1321,8 @@ successive matches::
                 line_start = pos
                 line += 1
             elif typ != 'SKIP':
+                if typ == 'ID' and val in keywords:
+                    typ = val
                 yield Token(typ, mo.group(typ), line, mo.start()-line_start)
             pos = mo.end()
             mo = gettok(s, pos)