gh-159739: Preserve non-BMP characters in multibyte encoders (#159987)

Author: jelle-openai

Lib/test/test_multibytecodec.py

@@ -43,6 +43,18 @@ def test_str_decode(self):
         for enc in ALL_CJKENCODINGS:
             self.assertEqual('abcd'.encode(enc), b'abcd')
 
+    def test_nonbmp_pair_modifier(self):
+        cases = [('big5hkscs', '\u00ca', '\u0304')]
+        cases.extend((encoding, '\u304b', '\u309a') for encoding in (
+            'euc_jis_2004', 'euc_jisx0213', 'shift_jis_2004', 'shift_jisx0213',
+            'iso2022_jp_2004', 'iso2022_jp_3'))
+        for encoding, base, modifier in cases:
+            with self.subTest(encoding=encoding):
+                pair = base + modifier
+                self.assertEqual(pair.encode(encoding).decode(encoding), pair)
+                nonbmp = base + chr(0x10000 + ord(modifier))
+                self.assertRaises(UnicodeEncodeError, nonbmp.encode, encoding)
+
     def test_errorcallback_longindex(self):
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: ('', sys.maxsize+1)

Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst

@@ -0,0 +1,2 @@
+Fix multibyte encoders silently consuming a non-BMP character when looking up
+a two-character mapping.

Modules/cjkcodecs/_codecs_hk.c

@@ -62,7 +62,7 @@ ENCODER(big5hkscs)
                     else
                         c2 = 0;
 
-                    if (inlen - *inpos >= 2 &&
+                    if (inlen - *inpos >= 2 && c2 <= 0xffff &&
                         ((c & 0xffdf) == 0x00ca) &&
                         ((c2 & 0xfff7) == 0x0304)) {
                         code = big5hkscs_pairenc_table[

Modules/cjkcodecs/_codecs_iso2022.c

@@ -804,7 +804,7 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
 
     case 2: /* second character of unicode pair */
         if (data[1] != 0) { /* Don't consume null char as part of pair */
-            coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1],
+            coded = find_pairencmap((ucs2_t)data[0], data[1],
                                     jisx0213_pair_encmap, JISX0213_ENCPAIRS);
             if (coded != DBCINV) {
                 return coded;

Modules/cjkcodecs/cjkcodecs.h

@@ -406,10 +406,13 @@ register_maps(PyObject *module)
 
 #ifdef USING_BINARY_PAIR_SEARCH
 static DBCHAR
-find_pairencmap(ucs2_t body, ucs2_t modifier,
+find_pairencmap(ucs2_t body, Py_UCS4 modifier,
                 const struct pair_encodemap *haystack, int haystacksize)
 {
     int pos, min, max;
+    if (modifier > 0xffff) {
+        return DBCINV;
+    }
     Py_UCS4 value = body << 16 | modifier;
 
     min = 0;