Issue #10546: UTF-16-LE and UTF-16-BE *do* support non-BMP characters

Fix the doc and add tests.

diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 5416d3b2593f0493dda7191637b0fb2941c43153..26e31a42862bb292e2b9329a05b5bf60177b3906 100644 (file)
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -1114,9 +1114,9 @@ particular, the following variants typically exist:
 +-----------------+--------------------------------+--------------------------------+
 | utf_16          | U16, utf16                     | all languages                  |
 +-----------------+--------------------------------+--------------------------------+
-| utf_16_be       | UTF-16BE                       | all languages (BMP only)       |
+| utf_16_be       | UTF-16BE                       | all languages                  |
 +-----------------+--------------------------------+--------------------------------+
-| utf_16_le       | UTF-16LE                       | all languages (BMP only)       |
+| utf_16_le       | UTF-16LE                       | all languages                  |
 +-----------------+--------------------------------+--------------------------------+
 | utf_7           | U7, unicode-1-1-utf-7          | all languages                  |
 +-----------------+--------------------------------+--------------------------------+

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index bc29e06c4f5007b2562f534e9bc8a076f5449558..8287a5b4ec536290c9c2c23f36b141fa8288828a 100644 (file)
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -544,6 +544,12 @@ class UTF16LETest(ReadTest):
         self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
                           b"\xff", "strict", True)
 
+    def test_nonbmp(self):
+        self.assertEqual("\U00010203".encode(self.encoding),
+                         b'\x00\xd8\x03\xde')
+        self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding),
+                         "\U00010203")
+
 class UTF16BETest(ReadTest):
     encoding = "utf-16-be"
 
@@ -566,6 +572,12 @@ class UTF16BETest(ReadTest):
         self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
                           b"\xff", "strict", True)
 
+    def test_nonbmp(self):
+        self.assertEqual("\U00010203".encode(self.encoding),
+                         b'\xd8\x00\xde\x03')
+        self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
+                         "\U00010203")
+
 class UTF8Test(ReadTest):
     encoding = "utf-8"