| | | code actually uses UTF-8 |
| | | by default. |
+--------------------+---------+---------------------------+
+| utf-7-imap | mUTF-7 | Modified UTF-7 encoding |
+| | | of :rfc:`3501` for IMAP4 |
+| | | mailbox names. Only |
+| | | ``errors='strict'`` is |
+| | | supported. |
+| | | |
+| | | .. versionadded:: next |
++--------------------+---------+---------------------------+
.. versionchanged:: 3.8
"unicode_internal" codec is removed.
wide-character support.
(Contributed by Serhiy Storchaka in :gh:`133031`.)
+encodings
+---------
+
+* Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding
+ used for international IMAP4 mailbox names (:rfc:`3501`).
+ (Contributed by Serhiy Storchaka in :gh:`66788`.)
+
+
gzip
----
'utf7' : 'utf_7',
'unicode_1_1_utf_7' : 'utf_7',
+ # utf_7_imap codec
+ 'csutf7imap' : 'utf_7_imap',
+ 'mutf_7' : 'utf_7_imap',
+
# utf_8 codec
'csutf8' : 'utf_8',
'u8' : 'utf_8',
--- /dev/null
+""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names,
+as specified in RFC 3501, section 5.1.3.
+
+It differs from UTF-7 (RFC 2152) as follows:
+
+* "&" (not "+") is the shift character introducing a Base64 sequence,
+ and "&-" encodes a literal "&";
+* "," is used instead of "/" in the modified Base64 alphabet;
+* only printable US-ASCII characters (except "&") may be represented
+ directly, so all other characters, including other controls, are
+ Base64-encoded.
+"""
+
+import binascii
+import codecs
+
+# The modified Base64 alphabet of RFC 3501: standard Base64 but with "," in
+# place of "/".
+_alphabet = binascii.BASE64_ALPHABET[:-1] + b','
+
+### Codec APIs
+
+def utf_7_imap_encode(input, errors='strict'):
+ if errors != 'strict':
+ raise UnicodeError(f"Unsupported error handling: {errors}")
+ res = bytearray()
+ start = 0 # start of the current run
+ b64run = False # is input[start:i] a Base64 run?
+ def flush(end):
+ if start < end:
+ if b64run:
+ b64 = binascii.b2a_base64(input[start:end].encode('utf-16-be'),
+ alphabet=_alphabet, padded=False,
+ newline=False)
+ res.extend(b'&' + b64 + b'-')
+ else:
+ res.extend(input[start:end].encode('ascii'))
+ for i, ch in enumerate(input):
+ if ch == '&':
+ flush(i)
+ res.extend(b'&-')
+ start = i + 1
+ b64run = False
+ elif ' ' <= ch <= '~': # printable ASCII, represented directly
+ if b64run:
+ flush(i)
+ start = i
+ b64run = False
+ else: # everything else is Base64-encoded
+ if not b64run:
+ flush(i)
+ start = i
+ b64run = True
+ flush(len(input))
+ return res.take_bytes(), len(input)
+
+def utf_7_imap_decode(input, errors='strict'):
+ if errors != 'strict':
+ raise UnicodeError(f"Unsupported error handling: {errors}")
+ input = bytes(input)
+ res = []
+ start = 0 # start of the current direct ASCII run
+ i = 0
+ n = len(input)
+ def flush(end):
+ if start < end:
+ res.append(input[start:end].decode('ascii'))
+ while i < n:
+ c = input[i]
+ if c == b'&'[0]:
+ flush(i)
+ j = input.find(b'-', i + 1)
+ if j < 0:
+ raise UnicodeDecodeError('utf-7-imap', input, i, n,
+ 'unterminated shift sequence')
+ if j == i + 1: # '&-'
+ res.append('&')
+ else:
+ b64 = input[i + 1:j]
+ try:
+ data = binascii.a2b_base64(b64, alphabet=_alphabet,
+ strict_mode=True, padded=False)
+ except binascii.Error:
+ data = b''
+ if not data or len(data) % 2:
+ raise UnicodeDecodeError('utf-7-imap', input, i, j + 1,
+ 'invalid shift sequence')
+ res.append(data.decode('utf-16-be'))
+ i = j + 1
+ start = i
+ elif b' '[0] <= c <= b'~'[0]:
+ i += 1
+ else:
+ raise UnicodeDecodeError('utf-7-imap', input, i, i + 1,
+ 'unexpected byte')
+ flush(n)
+ return ''.join(res), len(input)
+
+class Codec(codecs.Codec):
+ def encode(self, input, errors='strict'):
+ return utf_7_imap_encode(input, errors)
+ def decode(self, input, errors='strict'):
+ return utf_7_imap_decode(input, errors)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return utf_7_imap_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return utf_7_imap_decode(input, self.errors)[0]
+
+class StreamWriter(Codec, codecs.StreamWriter):
+ pass
+
+class StreamReader(Codec, codecs.StreamReader):
+ pass
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='utf-7-imap',
+ encode=Codec().encode,
+ decode=Codec().decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamwriter=StreamWriter,
+ streamreader=StreamReader,
+ )
self.assertEqual(puny.decode("punycode", errors), expected)
+utf_7_imap_testcases = [
+ # (unicode, modified UTF-7)
+ ('', b''),
+ ('INBOX', b'INBOX'),
+ # Printable US-ASCII represents itself.
+ ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'),
+ # "&" is escaped as "&-".
+ ('&', b'&-'),
+ ('&&', b'&-&-'),
+ ('A&B', b'A&-B'),
+ # "+" and "+-" are literal, unlike in UTF-7 where "+" is the shift character.
+ ('+', b'+'),
+ ('+-', b'+-'),
+ # RFC 3501 section 5.1.3 example.
+ ('~peter/mail/台北/日本語',
+ b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'),
+ # Non-printable ASCII (including TAB) is Base64-encoded, not direct.
+ ('a\tb', b'a&AAk-b'),
+ ('\x00', b'&AAA-'),
+ ('Entw\xfcrfe', b'Entw&APw-rfe'),
+ ('ϰ', b'&A,A-'), # "," in the Base64 alphabet ("&A/A-" is invalid)
+ ('☃', b'&JgM-'), # snowman
+ ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair)
+ ('Sent &\N{DELETE}', b'Sent &-&AH8-'),
+]
+
+class Utf7ImapTest(unittest.TestCase):
+ @support.subTests('uni,encoded', utf_7_imap_testcases)
+ def test_encode(self, uni, encoded):
+ self.assertEqual(uni.encode('utf-7-imap'), encoded)
+
+ @support.subTests('uni,encoded', utf_7_imap_testcases)
+ def test_decode(self, uni, encoded):
+ self.assertEqual(encoded.decode('utf-7-imap'), uni)
+
+ # 'start' is the position of the first offending byte in each case.
+ @support.subTests('encoded,start', [
+ (b'x&', 1), # "&" just before the end, unterminated
+ (b'&AAAA', 0), # unterminated, though the Base64 is valid
+ (b'&AB-', 0), # Base64 length not a multiple of a code unit
+ (b'&A/A-', 0), # "/" not in the alphabet ("&A,A-" is valid)
+ (b'&@@@-', 0), # invalid Base64
+ (b'a\x80b', 1), # 8-bit byte outside a shift sequence
+ (b'a\x1fb', 1), # control byte outside a shift sequence
+ ])
+ def test_decode_invalid(self, encoded, start):
+ with self.assertRaises(UnicodeDecodeError) as cm:
+ encoded.decode('utf-7-imap')
+ self.assertEqual(cm.exception.encoding, 'utf-7-imap')
+ self.assertEqual(cm.exception.start, start)
+
+ def test_encode_lone_surrogate(self):
+ with self.assertRaises(UnicodeEncodeError):
+ '\ud800'.encode('utf-7-imap')
+
+ def test_only_strict_errors(self):
+ with self.assertRaises(UnicodeError):
+ 'x'.encode('utf-7-imap', 'replace')
+ with self.assertRaises(UnicodeError):
+ b'x'.decode('utf-7-imap', 'ignore')
+
+ def test_stateless(self):
+ # The codec is registered and exposes the standard interface.
+ info = codecs.lookup('utf-7-imap')
+ self.assertEqual(info.name, 'utf-7-imap')
+
+
# From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
nameprep_tests = [
# 3.1 Map to nothing.
--- /dev/null
+Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding
+used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3).
projects / thirdparty / Python / cpython.git / commitdiff
