patch 7.4.852

Problem:    On MS-Windows console Vim uses ANSI APIs for keyboard input and
            console output, it cannot input/output Unicode characters.
Solution:   Use Unicode APIs for console I/O. (Ken Takata, Yasuhiro Matsumoto)

diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index b3cd34cb30906f2bbb81e4eb07dc5d0e0255ec69..48793a94dee8a5ab62785386d8092361e318e914 100644 (file)
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -7396,14 +7396,12 @@ A jump table for the options with a short description can be found at |Q_op|.
        the GUI it only applies to the keyboard ( 'encoding' is used for the
        display).  Except for the Mac when 'macatsui' is off, then
        'termencoding' should be "macroman".
-       In the Win32 console version the default value is the console codepage
-       when it differs from the ANSI codepage.
                                                                *E617*
        Note: This does not apply to the GTK+ 2 GUI.  After the GUI has been
        successfully initialized, 'termencoding' is forcibly set to "utf-8".
        Any attempts to set a different value will be rejected, and an error
        message is shown.
-       For the Win32 GUI 'termencoding' is not used for typed characters,
+       For the Win32 GUI and console versions 'termencoding' is not used,
        because the Win32 system always passes Unicode characters.
        When empty, the same encoding is used as for the 'encoding' option.
        This is the normal value.

diff --git a/src/os_win32.c b/src/os_win32.c
index 8f97c3bc274720394f689f952f0e7f8a743ed163..7695e938c3a393f064af0ac0b26310ed931ddea5 100644 (file)
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -213,8 +213,8 @@ static void standout(void);
 static void standend(void);
 static void visual_bell(void);
 static void cursor_visible(BOOL fVisible);
-static BOOL write_chars(LPCSTR pchBuf, DWORD cchToWrite);
-static char_u tgetch(int *pmodifiers, char_u *pch2);
+static DWORD write_chars(char_u *pchBuf, DWORD cbToWrite);
+static WCHAR tgetch(int *pmodifiers, WCHAR *pch2);
 static void create_conin(void);
 static int s_cursor_visible = TRUE;
 static int did_create_conin = FALSE;
@@ -265,15 +265,15 @@ read_console_input(
     if (!win8_or_later)
     {
        if (nLength == -1)
-           return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
-       return ReadConsoleInput(hInput, lpBuffer, 1, &dwEvents);
+           return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
+       return ReadConsoleInputW(hInput, lpBuffer, 1, &dwEvents);
     }
 
     if (s_dwMax == 0)
     {
        if (nLength == -1)
-           return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
-       if (!ReadConsoleInput(hInput, s_irCache, IRSIZE, &dwEvents))
+           return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
+       if (!ReadConsoleInputW(hInput, s_irCache, IRSIZE, &dwEvents))
            return FALSE;
        s_dwIndex = 0;
        s_dwMax = dwEvents;
@@ -868,9 +868,9 @@ static const struct
 #endif
 
 #if defined(__GNUC__) && !defined(__MINGW32__)  && !defined(__CYGWIN__)
-# define AChar AsciiChar
+# define UChar UnicodeChar
 #else
-# define AChar uChar.AsciiChar
+# define UChar uChar.UnicodeChar
 #endif
 
 /* The return code indicates key code size. */
@@ -889,12 +889,12 @@ win32_kbd_patch_key(
 
     if (s_iIsDead == 2)
     {
-       pker->AChar = (CHAR) awAnsiCode[1];
+       pker->UChar = (WCHAR) awAnsiCode[1];
        s_iIsDead = 0;
        return 1;
     }
 
-    if (pker->AChar != 0)
+    if (pker->UChar != 0)
        return 1;
 
     vim_memset(abKeystate, 0, sizeof (abKeystate));
@@ -909,7 +909,7 @@ win32_kbd_patch_key(
     }
 
     /* Clear any pending dead keys */
-    ToAscii(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 0);
+    ToUnicode(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 2, 0);
 
     if (uMods & SHIFT_PRESSED)
        abKeystate[VK_SHIFT] = 0x80;
@@ -922,11 +922,11 @@ win32_kbd_patch_key(
            abKeystate[VK_MENU] = abKeystate[VK_RMENU] = 0x80;
     }
 
-    s_iIsDead = ToAscii(pker->wVirtualKeyCode, pker->wVirtualScanCode,
-                       abKeystate, awAnsiCode, 0);
+    s_iIsDead = ToUnicode(pker->wVirtualKeyCode, pker->wVirtualScanCode,
+                       abKeystate, awAnsiCode, 2, 0);
 
     if (s_iIsDead > 0)
-       pker->AChar = (CHAR) awAnsiCode[0];
+       pker->UChar = (WCHAR) awAnsiCode[0];
 
     return s_iIsDead;
 }
@@ -953,8 +953,8 @@ static BOOL g_fJustGotFocus = FALSE;
     static BOOL
 decode_key_event(
     KEY_EVENT_RECORD   *pker,
-    char_u             *pch,
-    char_u             *pch2,
+    WCHAR              *pch,
+    WCHAR              *pch2,
     int                        *pmodifiers,
     BOOL               fDoPost)
 {
@@ -982,7 +982,7 @@ decode_key_event(
     }
 
     /* special cases */
-    if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->AChar == NUL)
+    if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->UChar == NUL)
     {
        /* Ctrl-6 is Ctrl-^ */
        if (pker->wVirtualKeyCode == '6')
@@ -1044,7 +1044,7 @@ decode_key_event(
        *pch = NUL;
     else
     {
-       *pch = (i > 0) ? pker->AChar : NUL;
+       *pch = (i > 0) ? pker->UChar : NUL;
 
        if (pmodifiers != NULL)
        {
@@ -1436,7 +1436,7 @@ WaitForChar(long msec)
     DWORD          dwNow = 0, dwEndTime = 0;
     INPUT_RECORD    ir;
     DWORD          cRecords;
-    char_u         ch, ch2;
+    WCHAR          ch, ch2;
 
     if (msec > 0)
        /* Wait until the specified time has elapsed. */
@@ -1523,7 +1523,7 @@ WaitForChar(long msec)
 #ifdef FEAT_MBYTE_IME
                /* Windows IME sends two '\n's with only one 'ENTER'.  First:
                 * wVirtualKeyCode == 13. second: wVirtualKeyCode == 0 */
-               if (ir.Event.KeyEvent.uChar.UnicodeChar == 0
+               if (ir.Event.KeyEvent.UChar == 0
                        && ir.Event.KeyEvent.wVirtualKeyCode == 13)
                {
                    read_console_input(g_hConIn, &ir, 1, &cRecords);
@@ -1586,10 +1586,10 @@ create_conin(void)
 /*
  * Get a keystroke or a mouse event
  */
-    static char_u
-tgetch(int *pmodifiers, char_u *pch2)
+    static WCHAR
+tgetch(int *pmodifiers, WCHAR *pch2)
 {
-    char_u ch;
+    WCHAR ch;
 
     for (;;)
     {
@@ -1658,11 +1658,6 @@ mch_inchar(
 #define TYPEAHEADLEN 20
     static char_u   typeahead[TYPEAHEADLEN];   /* previously typed bytes. */
     static int     typeaheadlen = 0;
-#ifdef FEAT_MBYTE
-    static char_u   *rest = NULL;      /* unconverted rest of previous read */
-    static int     restlen = 0;
-    int                    unconverted;
-#endif
 
     /* First use any typeahead that was kept because "buf" was too small. */
     if (typeaheadlen > 0)
@@ -1761,38 +1756,11 @@ mch_inchar(
        else
 #endif
        {
-           char_u      ch2 = NUL;
+           WCHAR       ch2 = NUL;
            int         modifiers = 0;
 
            c = tgetch(&modifiers, &ch2);
 
-#ifdef FEAT_MBYTE
-           /* stolen from fill_input_buf() in ui.c */
-           if (rest != NULL)
-           {
-               /* Use remainder of previous call, starts with an invalid
-                * character that may become valid when reading more. */
-               if (restlen > TYPEAHEADLEN - typeaheadlen)
-                   unconverted = TYPEAHEADLEN - typeaheadlen;
-               else
-                   unconverted = restlen;
-               mch_memmove(typeahead + typeaheadlen, rest, unconverted);
-               if (unconverted == restlen)
-               {
-                   vim_free(rest);
-                   rest = NULL;
-               }
-               else
-               {
-                   restlen -= unconverted;
-                   mch_memmove(rest, rest + unconverted, restlen);
-               }
-               typeaheadlen += unconverted;
-           }
-           else
-               unconverted = 0;
-#endif
-
            if (typebuf_changed(tb_change_cnt))
            {
                /* "buf" may be invalid now if a client put something in the
@@ -1816,27 +1784,36 @@ mch_inchar(
                int     n = 1;
                int     conv = FALSE;
 
-               typeahead[typeaheadlen] = c;
-               if (ch2 != NUL)
-               {
-                   typeahead[typeaheadlen + 1] = 3;
-                   typeahead[typeaheadlen + 2] = ch2;
-                   n += 2;
-               }
 #ifdef FEAT_MBYTE
-               /* Only convert normal characters, not special keys.  Need to
-                * convert before applying ALT, otherwise mapping <M-x> breaks
-                * when 'tenc' is set. */
-               if (input_conv.vc_type != CONV_NONE
-                                               && (ch2 == NUL || c != K_NUL))
+               if (ch2 == NUL)
                {
-                   conv = TRUE;
-                   typeaheadlen -= unconverted;
-                   n = convert_input_safe(typeahead + typeaheadlen,
-                               n + unconverted, TYPEAHEADLEN - typeaheadlen,
-                               rest == NULL ? &rest : NULL, &restlen);
+                   int     i;
+                   char_u  *p;
+                   WCHAR   ch[2];
+
+                   ch[0] = c;
+                   if (c >= 0xD800 && c <= 0xDBFF)     /* High surrogate */
+                   {
+                       ch[1] = tgetch(&modifiers, &ch2);
+                       n++;
+                   }
+                   p = utf16_to_enc(ch, &n);
+                   if (p != NULL)
+                   {
+                       for (i = 0; i < n; i++)
+                           typeahead[typeaheadlen + i] = p[i];
+                       vim_free(p);
+                   }
                }
+               else
 #endif
+                   typeahead[typeaheadlen] = c;
+               if (ch2 != NUL)
+               {
+                   typeahead[typeaheadlen + n] = 3;
+                   typeahead[typeaheadlen + n + 1] = (char_u)ch2;
+                   n += 2;
+               }
 
                if (conv)
                {
@@ -5366,27 +5343,73 @@ cursor_visible(BOOL fVisible)
 
 
 /*
- * write `cchToWrite' characters in `pchBuf' to the screen
- * Returns the number of characters actually written (at least one).
+ * write `cbToWrite' bytes in `pchBuf' to the screen
+ * Returns the number of bytes actually written (at least one).
  */
-    static BOOL
+    static DWORD
 write_chars(
-    LPCSTR pchBuf,
-    DWORD  cchToWrite)
+    char_u *pchBuf,
+    DWORD  cbToWrite)
 {
     COORD coord = g_coord;
     DWORD written;
 
-    FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cchToWrite,
-                               coord, &written);
-    /* When writing fails or didn't write a single character, pretend one
-     * character was written, otherwise we get stuck. */
-    if (WriteConsoleOutputCharacter(g_hConOut, pchBuf, cchToWrite,
-                               coord, &written) == 0
-           || written == 0)
-       written = 1;
+#ifdef FEAT_MBYTE
+    if (enc_codepage >= 0 && (int)GetACP() != enc_codepage)
+    {
+       static WCHAR    *unicodebuf = NULL;
+       static int      unibuflen = 0;
+       int             length;
+       DWORD           n, cchwritten, cells;
 
-    g_coord.X += (SHORT) written;
+       length = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite, 0, 0);
+       if (unicodebuf == NULL || length > unibuflen)
+       {
+           vim_free(unicodebuf);
+           unicodebuf = (WCHAR *)lalloc(length * sizeof(WCHAR), FALSE);
+           unibuflen = length;
+       }
+       MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite,
+                           unicodebuf, unibuflen);
+
+       cells = mb_string2cells(pchBuf, cbToWrite);
+       FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cells,
+                                   coord, &written);
+       /* When writing fails or didn't write a single character, pretend one
+        * character was written, otherwise we get stuck. */
+       if (WriteConsoleOutputCharacterW(g_hConOut, unicodebuf, length,
+                   coord, &cchwritten) == 0
+               || cchwritten == 0)
+           cchwritten = 1;
+
+       if (cchwritten == length)
+       {
+           written = cbToWrite;
+           g_coord.X += (SHORT)cells;
+       }
+       else
+       {
+           char_u *p = pchBuf;
+           for (n = 0; n < cchwritten; n++)
+               mb_cptr_adv(p);
+           written = p - pchBuf;
+           g_coord.X += (SHORT)mb_string2cells(pchBuf, written);
+       }
+    }
+    else
+#endif
+    {
+       FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cbToWrite,
+                                   coord, &written);
+       /* When writing fails or didn't write a single character, pretend one
+        * character was written, otherwise we get stuck. */
+       if (WriteConsoleOutputCharacter(g_hConOut, (LPCSTR)pchBuf, cbToWrite,
+                   coord, &written) == 0
+               || written == 0)
+           written = 1;
+
+       g_coord.X += (SHORT) written;
+    }
 
     while (g_coord.X > g_srScrollRegion.Right)
     {

diff --git a/src/ui.c b/src/ui.c
index 7bbf2838e2a70906e7a4faafaa8e1f2c03963a0d..e0e54d92429dc4ac082cc5738fb227baa9cc604e 100644 (file)
--- a/src/ui.c
+++ b/src/ui.c
@@ -42,7 +42,7 @@ ui_write(s, len)
     /* Don't output anything in silent mode ("ex -s") unless 'verbose' set */
     if (!(silent_mode && p_verbose == 0))
     {
-#ifdef FEAT_MBYTE
+#if defined(FEAT_MBYTE) && !defined(WIN3264)
        char_u  *tofree = NULL;
 
        if (output_conv.vc_type != CONV_NONE)
@@ -56,7 +56,7 @@ ui_write(s, len)
 
        mch_write(s, len);
 
-#ifdef FEAT_MBYTE
+#if defined(FEAT_MBYTE) && !defined(WIN3264)
        if (output_conv.vc_type != CONV_NONE)
            vim_free(tofree);
 #endif

diff --git a/src/version.c b/src/version.c
index 559f0c1abbf3c792f86916e0230c44d866a0a329..2aa3807475b5b0c10ab4aa3eafbd7d84edbd220c 100644 (file)
--- a/src/version.c
+++ b/src/version.c
@@ -741,6 +741,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    852,
 /**/
     851,
 /**/