gh-153494: Encode imaplib search criteria to the declared CHARSET (GH-153495)

IMAP4.search(), sort(), thread() and the uid SORT/THREAD variants now encode
str search criteria to the declared charset, so international search text can
be passed as ordinary str.  A criterion passed as bytes is sent unchanged, for
use with a charset that Python has no codec for.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 3f26243f77dd5aa7699a2be7c3685846dce84067..7b79d81790b667d7eccc3bf1eaed86b7a9f52b20 100644 (file)
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -610,6 +610,13 @@ An :class:`IMAP4` instance has the following methods:
    If *uid* is true, the message numbers in the response are UIDs
    (``UID SEARCH``).
 
+   A criterion passed as :class:`str` is encoded to *charset*
+   (which must name a codec known to Python);
+   pass :class:`bytes` to send a criterion that is already encoded,
+   for example when *charset* is one that Python does not support.
+   When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``),
+   the criterion is sent using the connection's encoding instead.
+
    Example::
 
       # M is a connected IMAP4 instance...
@@ -621,6 +628,9 @@ An :class:`IMAP4` instance has the following methods:
    .. versionchanged:: next
       Added the *uid* parameter.
 
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
 
 .. method:: IMAP4.select(mailbox='INBOX', readonly=False)
 
@@ -682,11 +692,18 @@ An :class:`IMAP4` instance has the following methods:
 
    If *uid* is true, the message numbers in the response are UIDs (``UID SORT``).
 
+   As with :meth:`search`,
+   a *search_criterion* passed as :class:`str` is encoded to *charset*;
+   pass :class:`bytes` to send one already encoded.
+
    This is an ``IMAP4rev1`` extension command.
 
    .. versionchanged:: next
       Added the *uid* parameter.
 
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
 
 .. method:: IMAP4.starttls(ssl_context=None)
 
@@ -772,11 +789,18 @@ An :class:`IMAP4` instance has the following methods:
    If *uid* is true, the message numbers in the response are UIDs
    (``UID THREAD``).
 
+   As with :meth:`search`,
+   a *search_criterion* passed as :class:`str` is encoded to *charset*;
+   pass :class:`bytes` to send one already encoded.
+
    This is an ``IMAP4rev1`` extension command.
 
    .. versionchanged:: next
       Added the *uid* parameter.
 
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
 
 .. method:: IMAP4.uid(command, arg[, ...])
 

diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst
index 9e1070e8efdaa57cf60c8c90835d3b293a949e2a..06c3dbb2f0f1adcb0826f24224c6aeb2f5d06381 100644 (file)
--- a/Doc/whatsnew/3.16.rst
+++ b/Doc/whatsnew/3.16.rst
@@ -276,6 +276,14 @@ imaplib
   :meth:`~imaplib.IMAP4.uid`.
   (Contributed by Serhiy Storchaka in :gh:`153502`.)
 
+* :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort`
+  and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands)
+  now encode :class:`str` search criteria to the declared *charset*,
+  so international search text can be passed as an ordinary :class:`str`.
+  When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``),
+  the criteria are sent using the connection encoding instead.
+  (Contributed by Serhiy Storchaka in :gh:`153494`.)
+
 
 ipaddress
 ---------

diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index 4e7619b61e89cd30f340870a76044cd94e387dc2..ed5b528976448a7163df9177a354f25f22a5b87b 100644 (file)
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -950,11 +950,15 @@ class IMAP4:
         If UTF8 is enabled, charset MUST be None.
         If 'uid' is true, the message numbers in the response are UIDs
         (UID SEARCH).
+
+        A 'criteria' passed as str is encoded to 'charset'; pass bytes to
+        send criteria that are already encoded.
         """
         name = 'SEARCH'
         if charset is not None:
             if self.utf8_enabled:
                 raise IMAP4.error("Non-None charset not valid in UTF8 mode")
+            criteria = self._encode_criteria(charset, criteria)
             args = ('CHARSET', self._astring(charset), *criteria)
         else:
             args = criteria
@@ -1036,6 +1040,7 @@ class IMAP4:
         #if not name in self.capabilities:      # Let the server decide!
         #       raise self.error('unimplemented extension command: %s' % name)
         sort_criteria = self._set_quote(sort_criteria)
+        search_criteria = self._encode_criteria(charset, search_criteria)
         if charset is not None:
             charset = self._astring(charset)
         args = (sort_criteria, charset, *search_criteria)
@@ -1117,6 +1122,7 @@ class IMAP4:
         (UID THREAD).
         """
         name = 'THREAD'
+        search_criteria = self._encode_criteria(charset, search_criteria)
         if charset is not None:
             charset = self._astring(charset)
         args = (self._atom(threading_algorithm), charset, *search_criteria)
@@ -1158,12 +1164,14 @@ class IMAP4:
                     self._set_quote(flags))
         elif command == 'SORT':
             sort_criteria, charset, *search_criteria = args
+            search_criteria = self._encode_criteria(charset, search_criteria)
             if charset is not None:
                 charset = self._astring(charset)
             args = (self._set_quote(sort_criteria), charset,
                     *search_criteria)
         elif command == 'THREAD':
             threading_algorithm, charset, *search_criteria = args
+            search_criteria = self._encode_criteria(charset, search_criteria)
             if charset is not None:
                 charset = self._astring(charset)
             args = (self._atom(threading_algorithm), charset,
@@ -1576,6 +1584,17 @@ class IMAP4:
             return arg
         return self._set_quote(arg)
 
+    def _encode_criteria(self, charset, criteria):
+        # Encode str search criteria to the declared CHARSET so the bytes on
+        # the wire match it.  bytes criteria are already encoded and pass
+        # through unchanged.  charset is None when no CHARSET is sent.
+        if charset is None:
+            return criteria
+        if isinstance(charset, (bytes, bytearray)):
+            charset = str(charset, 'ascii')
+        return tuple(c.encode(charset) if isinstance(c, str) else c
+                     for c in criteria)
+
     def _quote(self, arg):
         if isinstance(arg, str):
             arg = bytes(arg, self._encoding)

diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 2413a8728ec72be9603a641856700e2e5296705c..3eaca67299e7c3a1bfb5f7b50c4a0a409f719310 100644 (file)
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -205,6 +205,32 @@ class TestImaplib(unittest.TestCase):
         self.assertEqual(m._astring('Entwürfe'), '"Entwürfe"'.encode())
         self.assertEqual(m._astring(b'Entw\xc3\xbcrfe'), b'"Entw\xc3\xbcrfe"')
 
+    def test_encode_criteria(self):
+        m = imaplib.IMAP4.__new__(imaplib.IMAP4)
+        enc = m._encode_criteria
+        # No charset: criteria are returned unchanged.
+        self.assertEqual(enc(None, ('TEXT', 'x')), ('TEXT', 'x'))
+        # str criteria are encoded to the charset; ASCII is charset-independent.
+        self.assertEqual(enc('UTF-8', ('TEXT', 'XXXXXX')), (b'TEXT', b'XXXXXX'))
+        # Non-ASCII text is encoded to the declared charset, including charsets
+        # other than ASCII, Latin-1 and UTF-8.
+        self.assertEqual(enc('UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),))
+        self.assertEqual(enc('ISO-8859-1', ('"café"',)),
+                         ('"café"'.encode('latin-1'),))
+        self.assertEqual(enc('KOI8-U', ('"Київ"',)),
+                         ('"Київ"'.encode('koi8-u'),))
+        self.assertEqual(enc('SHIFT_JIS', ('"日本"',)),
+                         ('"日本"'.encode('shift_jis'),))
+        # bytes criteria are already encoded and pass through unchanged.
+        self.assertEqual(enc('SHIFT_JIS', (b'"already"',)), (b'"already"',))
+        # The charset name may itself be bytes.
+        self.assertEqual(enc(b'UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),))
+        # A charset with no codec at all (not even via the iconv codec) cannot
+        # encode str criteria; bytes criteria must be used with such
+        # server-only charsets.
+        self.assertRaises(LookupError, enc, 'no-such-charset', ('TEXT',))
+        self.assertEqual(enc('no-such-charset', (b'TEXT',)), (b'TEXT',))
+
     def test_astring_idempotent(self):
         # Quoting an already quoted argument should not change it, so that
         # quoting twice gives the same result as quoting once.
@@ -337,7 +363,11 @@ class SimpleIMAPHandler(socketserver.StreamRequestHandler):
                 except StopIteration:
                     self.continuation = None
                 continue
-            splitline = splitargs(line.decode().removesuffix('\r\n'))
+            self.server.line = line
+            # surrogateescape so a criterion encoded in a non-UTF-8 charset
+            # does not crash the handler; tests inspect server.line for bytes.
+            splitline = splitargs(line.decode('utf-8', 'surrogateescape')
+                                  .removesuffix('\r\n'))
             tag = splitline[0]
             cmd = splitline[1]
             args = splitline[2:]
@@ -1546,7 +1576,17 @@ class NewIMAPTestsMixin:
         self.assertEqual(data, [b'43'])
         self.assertEqual(server.args, ['CHARSET', 'UTF-8', 'TEXT', 'XXXXXX'])
 
-        typ, data = client.search('NF_Z_62-010_(1973)', 'TEXT', 'XXXXXX')
+        # A non-ASCII str criterion is encoded to the declared charset (KOI8-U
+        # here, which is not UTF-8, so check the encoded bytes on the wire).
+        response[:] = ['* SEARCH 43']
+        typ, data = client.search('KOI8-U', 'SUBJECT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn(b'CHARSET KOI8-U ', server.line)
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
+        # bytes criteria keep this focused on charset-name quoting (the
+        # parentheses force the name to be quoted) without criteria encoding.
+        typ, data = client.search('NF_Z_62-010_(1973)', b'TEXT', b'XXXXXX')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX'])
 
@@ -1616,10 +1656,16 @@ class NewIMAPTestsMixin:
         self.assertEqual(data, [br''])
         self.assertEqual(server.args, ['(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"'])
 
-        typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"')
+        typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* SORT']
+        typ, data = client.sort('(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_uid_sort(self):
         response = []
         client, server = self._setup(make_simple_handler('UID', response,
@@ -1644,10 +1690,16 @@ class NewIMAPTestsMixin:
         self.assertEqual(data, [br''])
         self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"'])
 
-        typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"')
+        typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['SORT', '(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* SORT']
+        typ, data = client.uid('sort', '(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
         # The uid=True keyword is a shorthand for uid('SORT', ...).
         response[:] = ['* SORT 2 84 882']
         typ, data = client.sort('(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994', uid=True)
@@ -1692,10 +1744,16 @@ class NewIMAPTestsMixin:
             b'(199)(200 202)(201)(203)(204)(205 206 207)(208)'])
         self.assertEqual(server.args, ['ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"'])
 
-        typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"')
+        typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* THREAD (1)']
+        typ, data = client.thread('ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_uid_thread(self):
         response = []
         client, server = self._setup(make_simple_handler('UID', response,
@@ -1734,10 +1792,16 @@ class NewIMAPTestsMixin:
             b'(199)(200 202)(201)(203)(204)(205 206 207)(208)'])
         self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"'])
 
-        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"')
+        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* THREAD (1)']
+        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
         # The uid=True keyword is a shorthand for uid('THREAD', ...).
         response[:] = ['* THREAD (166)(167)(168)']
         typ, data = client.thread('ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000',

diff --git a/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst
new file mode 100644 (file)
index 0000000..6e83940
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst
@@ -0,0 +1,8 @@
+:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort`
+and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands)
+now encode :class:`str` search criteria to the declared *charset*,
+so international search text can be passed as ordinary :class:`str`.
+When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``),
+the criteria are sent using the connection encoding instead.
+A criterion passed as :class:`bytes` is sent unchanged,
+for use with a charset that Python has no codec for.