From c2c7c373778967c084a060c4e313cff04146db66 Mon Sep 17 00:00:00 2001
From: Raymond Hettinger <python@rcn.com>
Date: Tue, 7 Dec 2010 09:44:21 +0000
Subject: [PATCH] Make the example a little more interesting and useful.

---
 Doc/library/re.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index c627f1a833..423540c48b 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -1300,6 +1300,7 @@ successive matches::
     Token = collections.namedtuple('Token', 'typ value line column')
 
     def tokenize(s):
+        keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}
         tok_spec = [
             ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number
             ('ASSIGN', r':='),          # Assignment operator
@@ -1320,6 +1321,8 @@ successive matches::
                 line_start = pos
                 line += 1
             elif typ != 'SKIP':
+                if typ == 'ID' and val in keywords:
+                    typ = val
                 yield Token(typ, mo.group(typ), line, mo.start()-line_start)
             pos = mo.end()
             mo = gettok(s, pos)
-- 
2.50.1