patch 8.2.2178: Python 3: non-utf8 character cannot be handled

Problem:    Python 3: non-utf8 character cannot be handled.
Solution:   Change the string decode. (Björn Linse, closes #1053)

diff --git a/src/if_py_both.h b/src/if_py_both.h
index 7b748b25e444d0070a528bfbe33869b901372499..2903b0ba9a1f42c3e8a5572d66acf0b9f8bb297b 100644 (file)
--- a/src/if_py_both.h
+++ b/src/if_py_both.h
@@ -130,10 +130,11 @@ StringToChars(PyObject *obj, PyObject **todecref)
     {
        PyObject        *bytes;
 
-       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+                                                          ERRORS_ENCODE_ARG)))
            return NULL;
 
-       if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1
+       if (PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1
                || str == NULL)
        {
            Py_DECREF(bytes);
@@ -4243,7 +4244,8 @@ StringToLine(PyObject *obj)
     }
     else if (PyUnicode_Check(obj))
     {
-       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+                                                          ERRORS_ENCODE_ARG)))
            return NULL;
 
        if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1
@@ -6290,11 +6292,11 @@ _ConvertFromPyObject(PyObject *obj, typval_T *tv, PyObject *lookup_dict)
        PyObject        *bytes;
        char_u  *str;
 
-       bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL);
+       bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, ERRORS_ENCODE_ARG);
        if (bytes == NULL)
            return -1;
 
-       if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1)
+       if (PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1)
            return -1;
        if (str == NULL)
            return -1;

diff --git a/src/if_python.c b/src/if_python.c
index 6338a5b8da4393c1228717dbde677707d8eb4ea1..7f90ede156ab234dc66b1bbbc2a5c174316bef19 100644 (file)
--- a/src/if_python.c
+++ b/src/if_python.c
@@ -69,6 +69,10 @@
 # undef PY_SSIZE_T_CLEAN
 #endif
 
+// these are NULL for Python 2
+#define ERRORS_DECODE_ARG NULL
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #undef main // Defined in python.h - aargh
 #undef HAVE_FCNTL_H // Clash with os_win32.h
 

diff --git a/src/if_python3.c b/src/if_python3.c
index a51be2949e1a68d16c88ead6651860a2a239ad3e..ea4fd7dd8bfc72111d45e505c9f35ef179d76203 100644 (file)
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -81,12 +81,15 @@
 // Python 3 does not support CObjects, always use Capsules
 #define PY_USE_CAPSULE
 
+#define ERRORS_DECODE_ARG CODEC_ERROR_HANDLER
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #define PyInt Py_ssize_t
 #ifndef PyString_Check
 # define PyString_Check(obj) PyUnicode_Check(obj)
 #endif
 #define PyString_FromString(repr) \
-    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, NULL)
+    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, ERRORS_DECODE_ARG)
 #define PyString_FromFormat PyUnicode_FromFormat
 #ifndef PyInt_Check
 # define PyInt_Check(obj) PyLong_Check(obj)
@@ -1088,8 +1091,8 @@ DoPyCommand(const char *cmd, rangeinitializer init_range, runner run, void *arg)
     // PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause
     // SyntaxError (unicode error).
     cmdstr = PyUnicode_Decode(cmd, strlen(cmd),
-                                       (char *)ENC_OPT, CODEC_ERROR_HANDLER);
-    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", CODEC_ERROR_HANDLER);
+                                       (char *)ENC_OPT, ERRORS_DECODE_ARG);
+    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", ERRORS_ENCODE_ARG);
     Py_XDECREF(cmdstr);
 
     run(PyBytes_AsString(cmdbytes), arg, &pygilstate);
@@ -1745,7 +1748,7 @@ LineToString(const char *str)
     }
     *p = '\0';
 
-    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, CODEC_ERROR_HANDLER);
+    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, ERRORS_DECODE_ARG);
 
     vim_free(tmp);
     return result;

diff --git a/src/testdir/test_python2.vim b/src/testdir/test_python2.vim
index afefd10fed8206a4f35dc6c5e99d610fa9d62c80..cf35a50b32fa1079528095d3ac2e76d8b1319a0c 100644 (file)
--- a/src/testdir/test_python2.vim
+++ b/src/testdir/test_python2.vim
@@ -3775,4 +3775,11 @@ func Test_python_keyboard_interrupt()
   close!
 endfunc
 
+func Test_python_non_utf8_string()
+  smap <Esc>@ <A-@>
+  python vim.command('redir => _tmp_smaps | smap | redir END')
+  python vim.eval('_tmp_smaps').splitlines()
+  sunmap <Esc>@
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

diff --git a/src/testdir/test_python3.vim b/src/testdir/test_python3.vim
index 78fe5d4c427545ce1f99f02e869f4e7debe62fd8..1a86a7e97077c252a0aec03161378e914f1e4703 100644 (file)
--- a/src/testdir/test_python3.vim
+++ b/src/testdir/test_python3.vim
@@ -4008,4 +4008,11 @@ func Test_python3_iter_ref()
   call assert_equal(1, g:options_iter_ref_count_increase)
 endfunc
 
+func Test_python3_non_utf8_string()
+  smap <Esc>@ <A-@>
+  py3 vim.command('redir => _tmp_smaps | smap | redir END')
+  py3 vim.eval('_tmp_smaps').splitlines()
+  sunmap <Esc>@
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

diff --git a/src/version.c b/src/version.c
index 05f08e5c7a7a84e35994825f1f41255353194c9b..aa6967d8587030a5f0843628e830b58cd43246b3 100644 (file)
--- a/src/version.c
+++ b/src/version.c
@@ -750,6 +750,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    2178,
 /**/
     2177,
 /**/