Filter directional markers that corrupt the screen. (closes #3854)

Thanks to Vincent Lefèvre for working on these utf-8 screen display issues.

diff --git a/mbyte.c b/mbyte.c
index d476a3a89a5a190cdddfe03caaff2ad936b44e22..3df143f5b536c7c6d24baecd4c00ab632b17235a 100644 (file)
--- a/mbyte.c
+++ b/mbyte.c
@@ -525,6 +525,18 @@ wchar_t replacement_char (void)
   return Charset_is_utf8 ? 0xfffd : '?';
 }
 
+int is_display_corrupting_utf8 (wchar_t wc)
+{
+  if (wc == (wchar_t)0x200f ||   /* bidi markers: #3827 */
+      wc == (wchar_t)0x200e ||
+      wc == (wchar_t)0x00ad ||   /* soft hyphen: #3848 */
+      (wc >= (wchar_t)0x202a &&  /* misc directional markers: #3854 */
+       wc <= (wchar_t)0x202e))
+    return 1;
+  else
+    return 0;
+}
+
 int mutt_filter_unprintable (char **s)
 {
   BUFFER *b = NULL;
@@ -548,13 +560,8 @@ int mutt_filter_unprintable (char **s)
     }
     if (!IsWPrint (wc))
       wc = '?';
-    /* Filter out the RIGHT-TO-LEFT and LEFT-TO-RIGHT bidi markers because
-     * they result in screen corruption.   See ticket #3827.
-     * Filter soft-hypen.  See ticket #3848.
-     */
     else if (Charset_is_utf8 &&
-             ((wc == (wchar_t)0x200f) || (wc == (wchar_t)0x200e) ||
-              (wc == (wchar_t)0x00ad)))
+             is_display_corrupting_utf8 (wc))
       continue;
     k2 = wcrtomb (scratch, wc, &mbstate2);
     scratch[k2] = '\0';

diff --git a/mbyte.h b/mbyte.h
index bb5601c43109a8a04f400a0d2ed149038ec459dd..9c58c9ec6b668eec294dc895f3b3e1fa06532e4c 100644 (file)
--- a/mbyte.h
+++ b/mbyte.h
@@ -49,5 +49,6 @@ void mutt_set_charset (char *charset);
 extern int Charset_is_utf8;
 size_t utf8rtowc (wchar_t *pwc, const char *s, size_t n, mbstate_t *_ps);
 wchar_t replacement_char (void);
+int is_display_corrupting_utf8 (wchar_t wc);
 
 #endif /* _MBYTE_H */

diff --git a/pager.c b/pager.c
index c1bd8caf18455ce5b57b5399a09e6cbb48d12e22..9404b7e143572fe264952d44424d7683d691501e 100644 (file)
--- a/pager.c
+++ b/pager.c
@@ -1156,9 +1156,7 @@ static int format_line (struct line_t **lineInfo, int n, unsigned char *buf,
        dprint (3, (debugfile, "skip zero-width character U+%04X\n", (unsigned short)wc));
        continue;
       }
-      /* Filter bidi markers, see ticket #3827
-         Filter soft hyphen, see ticket #3848 */
-      if (wc == (wchar_t)0x200f || wc == (wchar_t)0x200e || wc == (wchar_t)0x00ad)
+      if (is_display_corrupting_utf8 (wc))
       {
        dprint (3, (debugfile, "filtered U+%04X\n", (unsigned short)wc));
        continue;