Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 32 additions & 31 deletions Lib/email/header.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,28 +88,29 @@ def decode_header(header):
# two parts will be None.
words = []
for line in header.splitlines():
# ecre.split() returns [unencoded, charset, encoding, encoded,
# unencoded, ...]. Walk it by index: popping from the front would
# be quadratic in the number of encoded words.
parts = ecre.split(line)
first = True
while parts:
unencoded = parts.pop(0)
if first:
for i in range(0, len(parts), 4):
unencoded = parts[i]
if i == 0:
unencoded = unencoded.lstrip()
first = False
if unencoded:
words.append((unencoded, None, None))
if parts:
charset = parts.pop(0).lower()
encoding = parts.pop(0).lower()
encoded = parts.pop(0)
if i + 1 < len(parts):
charset = parts[i + 1].lower()
encoding = parts[i + 2].lower()
encoded = parts[i + 3]
words.append((encoded, encoding, charset))
# Now loop over words and remove words that consist of whitespace
# between two encoded strings.
droplist = []
for n, w in enumerate(words):
if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace():
droplist.append(n-1)
for d in reversed(droplist):
del words[d]
# Now remove words that consist of whitespace between two encoded
# strings. Build a new list instead of deleting items one by one,
# which would be quadratic.
droplist = {n - 1 for n, w in enumerate(words)
if n > 1 and w[1] and words[n-2][1]
and words[n-1][0].isspace()}
if droplist:
words = [w for n, w in enumerate(words) if n not in droplist]

# The next step is to decode each encoded word by applying the reverse
# base64 or quopri transformation. decoded_words is now a list of the
Expand All @@ -135,24 +136,24 @@ def decode_header(header):
else:
raise AssertionError('Unexpected encoding: ' + encoding)
# Now convert all words to bytes and collapse consecutive runs of
# similarly encoded words.
# similarly encoded words. Each run is collected in a list and joined
# once, since repeated bytes concatenation would be quadratic.
def join_run(run, charset):
return (BSPACE if charset is None else b'').join(run)

collapsed = []
last_word = last_charset = None
run = []
last_charset = None
for word, charset in decoded_words:
if isinstance(word, str):
word = bytes(word, 'raw-unicode-escape')
if last_word is None:
last_word = word
last_charset = charset
elif charset != last_charset:
collapsed.append((last_word, last_charset))
last_word = word
last_charset = charset
elif last_charset is None:
last_word += BSPACE + word
else:
last_word += word
collapsed.append((last_word, last_charset))
if run and charset != last_charset:
collapsed.append((join_run(run, last_charset), last_charset))
run = []
run.append(word)
last_charset = charset
collapsed.append((join_run(run, last_charset) if run else None,
last_charset))
return collapsed


Expand Down
15 changes: 15 additions & 0 deletions Lib/test/test_email/test_email.py
Original file line number Diff line number Diff line change
Expand Up @@ -2393,6 +2393,21 @@ def test_rfc2047_multiline(self):
Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?=
=?mac-iceland?q?=9Arg=8Cs?=""")

def test_decode_header_linear_complexity(self):
# Ensure that decode_header() is fast for many encoded words.
# See https://github.com/python/cpython/issues/136063.
N = 100_000
# Whitespace between adjacent encoded words is dropped and runs
# with the same charset are collapsed.
dh = decode_header(' '.join(['=?us-ascii?q?a?='] * N))
self.assertEqual(dh, [(b'a' * N, 'us-ascii')])
dh = decode_header(' '.join(['=?us-ascii?q?a?= =?utf-8?q?b?='] * N))
self.assertEqual(len(dh), 2 * N)
self.assertEqual(dh[-2:], [(b'a', 'us-ascii'), (b'b', 'utf-8')])
dh = decode_header(' x '.join(['=?us-ascii?q?a?='] * N))
self.assertEqual(len(dh), 2 * N - 1)
self.assertEqual(dh[-2:], [(b' x ', None), (b'a', 'us-ascii')])

def test_whitespace_keeper_unicode(self):
eq = self.assertEqual
s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard@dom.ain>'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix quadratic complexity in :func:`email.header.decode_header` for header
values containing many :rfc:`2047` encoded words.
Loading