From 7fe04f1dca5820c43416145789f4cc2b94e918b8 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Fri, 13 Feb 2015 15:08:36 +0200
Subject: [PATCH] Issue #21849: Fixed xmlrpclib serialization of non-ASCII
 unicode strings in the multiprocessing module.

---
 Lib/multiprocessing/connection.py   |  4 ++--
 Lib/multiprocessing/sharedctypes.py |  7 ++++++-
 Lib/test/test_multiprocessing.py    | 25 ++++++++++++++++++-------
 Misc/NEWS                           |  3 +++
 4 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index e4d520f5a1..463d4723ef 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -454,10 +454,10 @@ class ConnectionWrapper(object):
         return self._loads(s)
 
 def _xml_dumps(obj):
-    return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')
+    return xmlrpclib.dumps((obj,), None, None, None, 1)
 
 def _xml_loads(s):
-    (obj,), method = xmlrpclib.loads(s.decode('utf8'))
+    (obj,), method = xmlrpclib.loads(s)
     return obj
 
 class XmlListener(Listener):
diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py
index 1eb044dd54..58415fc449 100644
--- a/Lib/multiprocessing/sharedctypes.py
+++ b/Lib/multiprocessing/sharedctypes.py
@@ -46,13 +46,18 @@ __all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
 #
 
 typecode_to_type = {
-    'c': ctypes.c_char,  'u': ctypes.c_wchar,
+    'c': ctypes.c_char,
     'b': ctypes.c_byte,  'B': ctypes.c_ubyte,
     'h': ctypes.c_short, 'H': ctypes.c_ushort,
     'i': ctypes.c_int,   'I': ctypes.c_uint,
     'l': ctypes.c_long,  'L': ctypes.c_ulong,
     'f': ctypes.c_float, 'd': ctypes.c_double
     }
+try:
+    typecode_to_type['u'] = ctypes.c_wchar
+except AttributeError:
+    pass
+
 
 #
 #
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
index 42f6dd9326..ec4507ef74 100644
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -1371,6 +1371,16 @@ SERIALIZER = 'xmlrpclib'
 class _TestRemoteManager(BaseTestCase):
 
     ALLOWED_TYPES = ('manager',)
+    values = ['hello world', None, True, 2.25,
+              #'hall\xc3\xa5 v\xc3\xa4rlden'] # UTF-8
+              ]
+    result = values[:]
+    if test_support.have_unicode:
+        #result[-1] = u'hall\xe5 v\xe4rlden'
+        uvalue = test_support.u(r'\u043f\u0440\u0438\u0432\u0456\u0442 '
+                                r'\u0441\u0432\u0456\u0442')
+        values.append(uvalue)
+        result.append(uvalue)
 
     @classmethod
     def _putter(cls, address, authkey):
@@ -1379,7 +1389,8 @@ class _TestRemoteManager(BaseTestCase):
             )
         manager.connect()
         queue = manager.get_queue()
-        queue.put(('hello world', None, True, 2.25))
+        # Note that xmlrpclib will deserialize object as a list not a tuple
+        queue.put(tuple(cls.values))
 
     def test_remote(self):
         authkey = os.urandom(32)
@@ -1399,8 +1410,7 @@ class _TestRemoteManager(BaseTestCase):
         manager2.connect()
         queue = manager2.get_queue()
 
-        # Note that xmlrpclib will deserialize object as a list not a tuple
-        self.assertEqual(queue.get(), ['hello world', None, True, 2.25])
+        self.assertEqual(queue.get(), self.result)
 
         # Because we are using xmlrpclib for serialization instead of
         # pickle this will cause a serialization error.
@@ -2392,12 +2402,12 @@ class TestNoForkBomb(unittest.TestCase):
         name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
         if WIN32:
             rc, out, err = test.script_helper.assert_python_failure(name)
-            self.assertEqual('', out.decode('ascii'))
-            self.assertIn('RuntimeError', err.decode('ascii'))
+            self.assertEqual(out, '')
+            self.assertIn('RuntimeError', err)
         else:
             rc, out, err = test.script_helper.assert_python_ok(name)
-            self.assertEqual('123', out.decode('ascii').rstrip())
-            self.assertEqual('', err.decode('ascii'))
+            self.assertEqual(out.rstrip(), '123')
+            self.assertEqual(err, '')
 
 #
 # Issue 12098: check sys.flags of child matches that for parent
@@ -2421,6 +2431,7 @@ class TestFlags(unittest.TestCase):
         flags = (tuple(sys.flags), grandchild_flags)
         print(json.dumps(flags))
 
+    @test_support.requires_unicode  # XXX json needs unicode support
     def test_flags(self):
         import json, subprocess
         # start child process using unusual flags
diff --git a/Misc/NEWS b/Misc/NEWS
index 60a44eeb1e..64ab3e5e18 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -18,6 +18,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #21849: Fixed xmlrpclib serialization of non-ASCII unicode strings in
+  the multiprocessing module.
+
 - Issue #21840: Fixed expanding unicode variables of form $var in
   posixpath.expandvars().  Fixed all os.path implementations on
   unicode-disabled builds.
-- 
2.50.1