def test_incremental_surrogatepass(self):
# Test incremental decoder for surrogatepass handler:
# see issue #24214
+ # High surrogate
data = '\uD901'.encode(self.encoding, 'surrogatepass')
for i in range(1, len(data)):
dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
self.assertEqual(dec.decode(data[:i]), '')
self.assertEqual(dec.decode(data[i:], True), '\uD901')
+ # Low surrogate
+ data = '\uDC02'.encode(self.encoding, 'surrogatepass')
+ for i in range(1, len(data)):
+ dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
+ self.assertEqual(dec.decode(data[:i]), '')
+ self.assertEqual(dec.decode(data[i:]), '\uDC02')
class UTF32Test(ReadTest, unittest.TestCase):
with self.assertRaises(UnicodeDecodeError):
b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
+ def test_incremental_errors(self):
+ # Test that the incremental decoder can fail with final=False.
+ # See issue #24214
+ cases = [b'\x80', b'\xBF', b'\xC0', b'\xC1', b'\xF5', b'\xF6', b'\xFF']
+ for prefix in (b'\xC2', b'\xDF', b'\xE0', b'\xE0\xA0', b'\xEF',
+ b'\xEF\xBF', b'\xF0', b'\xF0\x90', b'\xF0\x90\x80',
+ b'\xF4', b'\xF4\x8F', b'\xF4\x8F\xBF'):
+ for suffix in b'\x7F', b'\xC0':
+ cases.append(prefix + suffix)
+ cases.extend((b'\xE0\x80', b'\xE0\x9F', b'\xED\xA0\x80',
+ b'\xED\xBF\xBF', b'\xF0\x80', b'\xF0\x8F', b'\xF4\x90'))
+
+ for data in cases:
+ with self.subTest(data=data):
+ dec = codecs.getincrementaldecoder(self.encoding)()
+ self.assertRaises(UnicodeDecodeError, dec.decode, data)
+
class UTF7Test(ReadTest, unittest.TestCase):
encoding = "utf-7"
goto InvalidContinuation1;
} else if (ch == 0xF4 && ch2 >= 0x90) {
/* invalid sequence
- \xF4\x90\x80\80- -- 110000- overflow */
+ \xF4\x90\x80\x80- -- 110000- overflow */
goto InvalidContinuation1;
}
if (!IS_CONTINUATION_BYTE(ch3)) {
}
/* UTF-16 code pair: */
- if (q >= e)
- goto UnexpectedEnd;
if (!Py_UNICODE_IS_HIGH_SURROGATE(ch))
goto IllegalEncoding;
+ if (q >= e)
+ goto UnexpectedEnd;
ch2 = (q[ihi] << 8) | q[ilo];
q += 2;
if (!Py_UNICODE_IS_LOW_SURROGATE(ch2))