From 994b826cd1598d8927917dbf51d3d1834a111315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 11 Sep 2024 20:15:48 +0200 Subject: [PATCH] Update to 3.9.20 --- ...-addresses-in-email-parseaddr-111116.patch | 500 ------------------ ...h-non-int-suffix-gh-112771-gh-112774.patch | 66 --- ...113179-gh-113186-gh-118177-gh-118472.patch | 399 -------------- ...d-verify-headers-are-sound-gh-122233.patch | 356 ------------- ...22905-sanitize-names-in-zipfile-path.patch | 128 ----- python3.9.spec | 75 +-- sources | 4 +- 7 files changed, 7 insertions(+), 1521 deletions(-) delete mode 100644 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch delete mode 100644 00419-gh-112769-test_zlib-fix-comparison-of-zlib_runtime_version-with-non-int-suffix-gh-112771-gh-112774.patch delete mode 100644 00431-gh-113171-gh-65056-fix-private-non-global-ip-address-ranges-gh-113179-gh-113186-gh-118177-gh-118472.patch delete mode 100644 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch delete mode 100644 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch diff --git a/00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch b/00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch deleted file mode 100644 index af44881..0000000 --- a/00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch +++ /dev/null @@ -1,500 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Victor Stinner -Date: Fri, 15 Dec 2023 16:10:40 +0100 -Subject: [PATCH] 00415: [CVE-2023-27043] gh-102988: Reject malformed addresses - in email.parseaddr() (#111116) - -Detect email address parsing errors and return empty tuple to -indicate the parsing error (old API). Add an optional 'strict' -parameter to getaddresses() and parseaddr() functions. Patch by -Thomas Dwyer. - -Co-Authored-By: Thomas Dwyer ---- - Doc/library/email.utils.rst | 19 +- - Lib/email/utils.py | 151 ++++++++++++- - Lib/test/test_email/test_email.py | 204 +++++++++++++++++- - ...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 + - 4 files changed, 361 insertions(+), 21 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst - -diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst -index 4d0e920eb0..104229e9e5 100644 ---- a/Doc/library/email.utils.rst -+++ b/Doc/library/email.utils.rst -@@ -60,13 +60,18 @@ of the new API. - begins with angle brackets, they are stripped off. - - --.. function:: parseaddr(address) -+.. function:: parseaddr(address, *, strict=True) - - Parse address -- which should be the value of some address-containing field such - as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and - *email address* parts. Returns a tuple of that information, unless the parse - fails, in which case a 2-tuple of ``('', '')`` is returned. - -+ If *strict* is true, use a strict parser which rejects malformed inputs. -+ -+ .. versionchanged:: 3.13 -+ Add *strict* optional parameter and reject malformed inputs by default. -+ - - .. function:: formataddr(pair, charset='utf-8') - -@@ -84,12 +89,15 @@ of the new API. - Added the *charset* option. - - --.. function:: getaddresses(fieldvalues) -+.. function:: getaddresses(fieldvalues, *, strict=True) - - This method returns a list of 2-tuples of the form returned by ``parseaddr()``. - *fieldvalues* is a sequence of header field values as might be returned by -- :meth:`Message.get_all `. Here's a simple -- example that gets all the recipients of a message:: -+ :meth:`Message.get_all `. -+ -+ If *strict* is true, use a strict parser which rejects malformed inputs. -+ -+ Here's a simple example that gets all the recipients of a message:: - - from email.utils import getaddresses - -@@ -99,6 +107,9 @@ of the new API. - resent_ccs = msg.get_all('resent-cc', []) - all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) - -+ .. versionchanged:: 3.13 -+ Add *strict* optional parameter and reject malformed inputs by default. -+ - - .. function:: parsedate(date) - -diff --git a/Lib/email/utils.py b/Lib/email/utils.py -index 48d30160aa..7ca7a7c886 100644 ---- a/Lib/email/utils.py -+++ b/Lib/email/utils.py -@@ -48,6 +48,7 @@ TICK = "'" - specialsre = re.compile(r'[][\\()<>@,:;".]') - escapesre = re.compile(r'[\\"]') - -+ - def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" - # This check is based on the fact that unless there are surrogates, utf8 -@@ -106,12 +107,127 @@ def formataddr(pair, charset='utf-8'): - return address - - -+def _iter_escaped_chars(addr): -+ pos = 0 -+ escape = False -+ for pos, ch in enumerate(addr): -+ if escape: -+ yield (pos, '\\' + ch) -+ escape = False -+ elif ch == '\\': -+ escape = True -+ else: -+ yield (pos, ch) -+ if escape: -+ yield (pos, '\\') - --def getaddresses(fieldvalues): -- """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" -- all = COMMASPACE.join(str(v) for v in fieldvalues) -- a = _AddressList(all) -- return a.addresslist -+ -+def _strip_quoted_realnames(addr): -+ """Strip real names between quotes.""" -+ if '"' not in addr: -+ # Fast path -+ return addr -+ -+ start = 0 -+ open_pos = None -+ result = [] -+ for pos, ch in _iter_escaped_chars(addr): -+ if ch == '"': -+ if open_pos is None: -+ open_pos = pos -+ else: -+ if start != open_pos: -+ result.append(addr[start:open_pos]) -+ start = pos + 1 -+ open_pos = None -+ -+ if start < len(addr): -+ result.append(addr[start:]) -+ -+ return ''.join(result) -+ -+ -+supports_strict_parsing = True -+ -+def getaddresses(fieldvalues, *, strict=True): -+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. -+ -+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in -+ its place. -+ -+ If strict is true, use a strict parser which rejects malformed inputs. -+ """ -+ -+ # If strict is true, if the resulting list of parsed addresses is greater -+ # than the number of fieldvalues in the input list, a parsing error has -+ # occurred and consequently a list containing a single empty 2-tuple [('', -+ # '')] is returned in its place. This is done to avoid invalid output. -+ # -+ # Malformed input: getaddresses(['alice@example.com ']) -+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] -+ # Safe output: [('', '')] -+ -+ if not strict: -+ all = COMMASPACE.join(str(v) for v in fieldvalues) -+ a = _AddressList(all) -+ return a.addresslist -+ -+ fieldvalues = [str(v) for v in fieldvalues] -+ fieldvalues = _pre_parse_validation(fieldvalues) -+ addr = COMMASPACE.join(fieldvalues) -+ a = _AddressList(addr) -+ result = _post_parse_validation(a.addresslist) -+ -+ # Treat output as invalid if the number of addresses is not equal to the -+ # expected number of addresses. -+ n = 0 -+ for v in fieldvalues: -+ # When a comma is used in the Real Name part it is not a deliminator. -+ # So strip those out before counting the commas. -+ v = _strip_quoted_realnames(v) -+ # Expected number of addresses: 1 + number of commas -+ n += 1 + v.count(',') -+ if len(result) != n: -+ return [('', '')] -+ -+ return result -+ -+ -+def _check_parenthesis(addr): -+ # Ignore parenthesis in quoted real names. -+ addr = _strip_quoted_realnames(addr) -+ -+ opens = 0 -+ for pos, ch in _iter_escaped_chars(addr): -+ if ch == '(': -+ opens += 1 -+ elif ch == ')': -+ opens -= 1 -+ if opens < 0: -+ return False -+ return (opens == 0) -+ -+ -+def _pre_parse_validation(email_header_fields): -+ accepted_values = [] -+ for v in email_header_fields: -+ if not _check_parenthesis(v): -+ v = "('', '')" -+ accepted_values.append(v) -+ -+ return accepted_values -+ -+ -+def _post_parse_validation(parsed_email_header_tuples): -+ accepted_values = [] -+ # The parser would have parsed a correctly formatted domain-literal -+ # The existence of an [ after parsing indicates a parsing failure -+ for v in parsed_email_header_tuples: -+ if '[' in v[1]: -+ v = ('', '') -+ accepted_values.append(v) -+ -+ return accepted_values - - - def _format_timetuple_and_zone(timetuple, zone): -@@ -202,16 +318,33 @@ def parsedate_to_datetime(data): - tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) - - --def parseaddr(addr): -+def parseaddr(addr, *, strict=True): - """ - Parse addr into its constituent realname and email address parts. - - Return a tuple of realname and email address, unless the parse fails, in - which case return a 2-tuple of ('', ''). -+ -+ If strict is True, use a strict parser which rejects malformed inputs. - """ -- addrs = _AddressList(addr).addresslist -- if not addrs: -- return '', '' -+ if not strict: -+ addrs = _AddressList(addr).addresslist -+ if not addrs: -+ return ('', '') -+ return addrs[0] -+ -+ if isinstance(addr, list): -+ addr = addr[0] -+ -+ if not isinstance(addr, str): -+ return ('', '') -+ -+ addr = _pre_parse_validation([addr])[0] -+ addrs = _post_parse_validation(_AddressList(addr).addresslist) -+ -+ if not addrs or len(addrs) > 1: -+ return ('', '') -+ - return addrs[0] - - -diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py -index 761ea90b78..0c689643de 100644 ---- a/Lib/test/test_email/test_email.py -+++ b/Lib/test/test_email/test_email.py -@@ -16,6 +16,7 @@ from unittest.mock import patch - - import email - import email.policy -+import email.utils - - from email.charset import Charset - from email.header import Header, decode_header, make_header -@@ -3263,15 +3264,154 @@ Foo - [('Al Person', 'aperson@dom.ain'), - ('Bud Person', 'bperson@dom.ain')]) - -+ def test_getaddresses_comma_in_name(self): -+ """GH-106669 regression test.""" -+ self.assertEqual( -+ utils.getaddresses( -+ [ -+ '"Bud, Person" ', -+ 'aperson@dom.ain (Al Person)', -+ '"Mariusz Felisiak" ', -+ ] -+ ), -+ [ -+ ('Bud, Person', 'bperson@dom.ain'), -+ ('Al Person', 'aperson@dom.ain'), -+ ('Mariusz Felisiak', 'to@example.com'), -+ ], -+ ) -+ -+ def test_parsing_errors(self): -+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056""" -+ alice = 'alice@example.org' -+ bob = 'bob@example.com' -+ empty = ('', '') -+ -+ # Test utils.getaddresses() and utils.parseaddr() on malformed email -+ # addresses: default behavior (strict=True) rejects malformed address, -+ # and strict=False which tolerates malformed address. -+ for invalid_separator, expected_non_strict in ( -+ ('(', [(f'<{bob}>', alice)]), -+ (')', [('', alice), empty, ('', bob)]), -+ ('<', [('', alice), empty, ('', bob), empty]), -+ ('>', [('', alice), empty, ('', bob)]), -+ ('[', [('', f'{alice}[<{bob}>]')]), -+ (']', [('', alice), empty, ('', bob)]), -+ ('@', [empty, empty, ('', bob)]), -+ (';', [('', alice), empty, ('', bob)]), -+ (':', [('', alice), ('', bob)]), -+ ('.', [('', alice + '.'), ('', bob)]), -+ ('"', [('', alice), ('', f'<{bob}>')]), -+ ): -+ address = f'{alice}{invalid_separator}<{bob}>' -+ with self.subTest(address=address): -+ self.assertEqual(utils.getaddresses([address]), -+ [empty]) -+ self.assertEqual(utils.getaddresses([address], strict=False), -+ expected_non_strict) -+ -+ self.assertEqual(utils.parseaddr([address]), -+ empty) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Comma (',') is treated differently depending on strict parameter. -+ # Comma without quotes. -+ address = f'{alice},<{bob}>' -+ self.assertEqual(utils.getaddresses([address]), -+ [('', alice), ('', bob)]) -+ self.assertEqual(utils.getaddresses([address], strict=False), -+ [('', alice), ('', bob)]) -+ self.assertEqual(utils.parseaddr([address]), -+ empty) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Real name between quotes containing comma. -+ address = '"Alice, alice@example.org" ' -+ expected_strict = ('Alice, alice@example.org', 'bob@example.com') -+ self.assertEqual(utils.getaddresses([address]), [expected_strict]) -+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) -+ self.assertEqual(utils.parseaddr([address]), expected_strict) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Valid parenthesis in comments. -+ address = 'alice@example.org (Alice)' -+ expected_strict = ('Alice', 'alice@example.org') -+ self.assertEqual(utils.getaddresses([address]), [expected_strict]) -+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) -+ self.assertEqual(utils.parseaddr([address]), expected_strict) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Invalid parenthesis in comments. -+ address = 'alice@example.org )Alice(' -+ self.assertEqual(utils.getaddresses([address]), [empty]) -+ self.assertEqual(utils.getaddresses([address], strict=False), -+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')]) -+ self.assertEqual(utils.parseaddr([address]), empty) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Two addresses with quotes separated by comma. -+ address = '"Jane Doe" , "John Doe" ' -+ self.assertEqual(utils.getaddresses([address]), -+ [('Jane Doe', 'jane@example.net'), -+ ('John Doe', 'john@example.net')]) -+ self.assertEqual(utils.getaddresses([address], strict=False), -+ [('Jane Doe', 'jane@example.net'), -+ ('John Doe', 'john@example.net')]) -+ self.assertEqual(utils.parseaddr([address]), empty) -+ self.assertEqual(utils.parseaddr([address], strict=False), -+ ('', address)) -+ -+ # Test email.utils.supports_strict_parsing attribute -+ self.assertEqual(email.utils.supports_strict_parsing, True) -+ - def test_getaddresses_nasty(self): -- eq = self.assertEqual -- eq(utils.getaddresses(['foo: ;']), [('', '')]) -- eq(utils.getaddresses( -- ['[]*-- =~$']), -- [('', ''), ('', ''), ('', '*--')]) -- eq(utils.getaddresses( -- ['foo: ;', '"Jason R. Mastaler" ']), -- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) -+ for addresses, expected in ( -+ (['"Sürname, Firstname" '], -+ [('Sürname, Firstname', 'to@example.com')]), -+ -+ (['foo: ;'], -+ [('', '')]), -+ -+ (['foo: ;', '"Jason R. Mastaler" '], -+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]), -+ -+ ([r'Pete(A nice \) chap) '], -+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]), -+ -+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'], -+ [('', '')]), -+ -+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'], -+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]), -+ -+ (['John Doe '], -+ [('John Doe (comment)', 'jdoe@machine.example')]), -+ -+ (['"Mary Smith: Personal Account" '], -+ [('Mary Smith: Personal Account', 'smith@home.example')]), -+ -+ (['Undisclosed recipients:;'], -+ [('', '')]), -+ -+ ([r', "Giant; \"Big\" Box" '], -+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]), -+ ): -+ with self.subTest(addresses=addresses): -+ self.assertEqual(utils.getaddresses(addresses), -+ expected) -+ self.assertEqual(utils.getaddresses(addresses, strict=False), -+ expected) -+ -+ addresses = ['[]*-- =~$'] -+ self.assertEqual(utils.getaddresses(addresses), -+ [('', '')]) -+ self.assertEqual(utils.getaddresses(addresses, strict=False), -+ [('', ''), ('', ''), ('', '*--')]) - - def test_getaddresses_embedded_comment(self): - """Test proper handling of a nested comment""" -@@ -3460,6 +3600,54 @@ multipart/report - m = cls(*constructor, policy=email.policy.default) - self.assertIs(m.policy, email.policy.default) - -+ def test_iter_escaped_chars(self): -+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')), -+ [(0, 'a'), -+ (2, '\\\\'), -+ (3, 'b'), -+ (5, '\\"'), -+ (6, 'c'), -+ (8, '\\\\'), -+ (9, '"'), -+ (10, 'd')]) -+ self.assertEqual(list(utils._iter_escaped_chars('a\\')), -+ [(0, 'a'), (1, '\\')]) -+ -+ def test_strip_quoted_realnames(self): -+ def check(addr, expected): -+ self.assertEqual(utils._strip_quoted_realnames(addr), expected) -+ -+ check('"Jane Doe" , "John Doe" ', -+ ' , ') -+ check(r'"Jane \"Doe\"." ', -+ ' ') -+ -+ # special cases -+ check(r'before"name"after', 'beforeafter') -+ check(r'before"name"', 'before') -+ check(r'b"name"', 'b') # single char -+ check(r'"name"after', 'after') -+ check(r'"name"a', 'a') # single char -+ check(r'"name"', '') -+ -+ # no change -+ for addr in ( -+ 'Jane Doe , John Doe ', -+ 'lone " quote', -+ ): -+ self.assertEqual(utils._strip_quoted_realnames(addr), addr) -+ -+ -+ def test_check_parenthesis(self): -+ addr = 'alice@example.net' -+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)')) -+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice(')) -+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))')) -+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)')) -+ -+ # Ignore real name between quotes -+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}')) -+ - - # Test the iterator/generators - class TestIterators(TestEmailBase): -diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst -new file mode 100644 -index 0000000000..3d0e9e4078 ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst -@@ -0,0 +1,8 @@ -+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now -+return ``('', '')`` 2-tuples in more situations where invalid email -+addresses are encountered instead of potentially inaccurate values. Add -+optional *strict* parameter to these two functions: use ``strict=False`` to -+get the old behavior, accept malformed inputs. -+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check -+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor -+Stinner to improve the CVE-2023-27043 fix. diff --git a/00419-gh-112769-test_zlib-fix-comparison-of-zlib_runtime_version-with-non-int-suffix-gh-112771-gh-112774.patch b/00419-gh-112769-test_zlib-fix-comparison-of-zlib_runtime_version-with-non-int-suffix-gh-112771-gh-112774.patch deleted file mode 100644 index ce6c9f4..0000000 --- a/00419-gh-112769-test_zlib-fix-comparison-of-zlib_runtime_version-with-non-int-suffix-gh-112771-gh-112774.patch +++ /dev/null @@ -1,66 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= -Date: Tue, 5 Dec 2023 21:02:06 +0100 -Subject: [PATCH] 00419: gh-112769: test_zlib: Fix comparison of - ZLIB_RUNTIME_VERSION with non-int suffix (GH-112771) (GH-112774) - -zlib-ng defines the version as "1.3.0.zlib-ng". -(cherry picked from commit d384813ff18b33280a90b6d2011654528a2b6ad1) ---- - Lib/test/test_zlib.py | 28 ++++++++++++++++------------ - 1 file changed, 16 insertions(+), 12 deletions(-) - -diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py -index 02509cdf55..2a9e7e5ed3 100644 ---- a/Lib/test/test_zlib.py -+++ b/Lib/test/test_zlib.py -@@ -17,6 +17,20 @@ requires_Decompress_copy = unittest.skipUnless( - 'requires Decompress.copy()') - - -+def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): -+ # Register "1.2.3" as "1.2.3.0" -+ # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" -+ v = zlib_version.split('-', 1)[0].split('.') -+ if len(v) < 4: -+ v.append('0') -+ elif not v[-1].isnumeric(): -+ v[-1] = '0' -+ return tuple(map(int, v)) -+ -+ -+ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() -+ -+ - class VersionTestCase(unittest.TestCase): - - def test_library_version(self): -@@ -437,9 +451,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', - 'Z_PARTIAL_FLUSH'] - -- ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) - # Z_BLOCK has a known failure prior to 1.2.5.3 -- if ver >= (1, 2, 5, 3): -+ if ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 5, 3): - sync_opt.append('Z_BLOCK') - - sync_opt = [getattr(zlib, opt) for opt in sync_opt -@@ -768,16 +781,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): - - def test_wbits(self): - # wbits=0 only supported since zlib v1.2.3.5 -- # Register "1.2.3" as "1.2.3.0" -- # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" -- v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.') -- if len(v) < 4: -- v.append('0') -- elif not v[-1].isnumeric(): -- v[-1] = '0' -- -- v = tuple(map(int, v)) -- supports_wbits_0 = v >= (1, 2, 3, 5) -+ supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5) - - co = zlib.compressobj(level=1, wbits=15) - zlib15 = co.compress(HAMLET_SCENE) + co.flush() diff --git a/00431-gh-113171-gh-65056-fix-private-non-global-ip-address-ranges-gh-113179-gh-113186-gh-118177-gh-118472.patch b/00431-gh-113171-gh-65056-fix-private-non-global-ip-address-ranges-gh-113179-gh-113186-gh-118177-gh-118472.patch deleted file mode 100644 index 1f3563b..0000000 --- a/00431-gh-113171-gh-65056-fix-private-non-global-ip-address-ranges-gh-113179-gh-113186-gh-118177-gh-118472.patch +++ /dev/null @@ -1,399 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Petr Viktorin -Date: Tue, 7 May 2024 11:57:58 +0200 -Subject: [PATCH] 00431: gh-113171: gh-65056: Fix "private" (non-global) IP - address ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472) - -The _private_networks variables, used by various is_private -implementations, were missing some ranges and at the same time had -overly strict ranges (where there are more specific ranges considered -globally reachable by the IANA registries). - -This patch updates the ranges with what was missing or otherwise -incorrect. - -100.64.0.0/10 is left alone, for now, as it's been made special in [1]. - -The _address_exclude_many() call returns 8 networks for IPv4, 121 -networks for IPv6. - -[1] https://github.com/python/cpython/issues/61602 - -In 3.10 and below, is_private checks whether the network and broadcast -address are both private. -In later versions (where the test wss backported from), it checks -whether they both are in the same private network. - -For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private, -but one is in 0.0.0.0/8 ("This network") and the other in -255.255.255.255/32 ("Limited broadcast"). - ---------- - -Co-authored-by: Jakub Stasiak ---- - Doc/library/ipaddress.rst | 43 ++++++++- - Doc/tools/susp-ignored.csv | 8 ++ - Doc/whatsnew/3.9.rst | 9 ++ - Lib/ipaddress.py | 95 +++++++++++++++---- - Lib/test/test_ipaddress.py | 52 ++++++++++ - ...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++ - 6 files changed, 195 insertions(+), 21 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst - -diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst -index 9c2dff5570..f9c1ebf3f3 100644 ---- a/Doc/library/ipaddress.rst -+++ b/Doc/library/ipaddress.rst -@@ -188,18 +188,53 @@ write code that handles both IP versions correctly. Address objects are - - .. attribute:: is_private - -- ``True`` if the address is allocated for private networks. See -+ ``True`` if the address is defined as not globally reachable by - iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -- (for IPv6). -+ (for IPv6) with the following exceptions: -+ -+ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) -+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: -+ -+ address.is_private == address.ipv4_mapped.is_private -+ -+ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space -+ (``100.64.0.0/10`` range) where they are both ``False``. -+ -+ .. versionchanged:: 3.9.20 -+ -+ Fixed some false positives and false negatives. -+ -+ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and -+ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). -+ * ``64:ff9b:1::/48`` is considered private. -+ * ``2002::/16`` is considered private. -+ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, -+ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. -+ The exceptions are not considered private. - - .. attribute:: is_global - -- ``True`` if the address is allocated for public networks. See -+ ``True`` if the address is defined as globally reachable by - iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -- (for IPv6). -+ (for IPv6) with the following exception: -+ -+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: -+ -+ address.is_global == address.ipv4_mapped.is_global -+ -+ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space -+ (``100.64.0.0/10`` range) where they are both ``False``. - - .. versionadded:: 3.4 - -+ .. versionchanged:: 3.9.20 -+ -+ Fixed some false positives and false negatives, see :attr:`is_private` for details. -+ - .. attribute:: is_unspecified - - ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) -diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv -index 3eb3d7954f..de91a50bad 100644 ---- a/Doc/tools/susp-ignored.csv -+++ b/Doc/tools/susp-ignored.csv -@@ -169,6 +169,14 @@ library/ipaddress,,:db00,2001:db00::0/24 - library/ipaddress,,::,2001:db00::0/24 - library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: - library/ipaddress,,::,2001:db00::0/ffff:ff00:: -+library/ipaddress,,:ff9b,64:ff9b:1::/48 -+library/ipaddress,,::,64:ff9b:1::/48 -+library/ipaddress,,::,2001:: -+library/ipaddress,,::,2001:1:: -+library/ipaddress,,::,2001:3:: -+library/ipaddress,,::,2001:4:112:: -+library/ipaddress,,::,2001:20:: -+library/ipaddress,,::,2001:30:: - library/itertools,,:step,elements from seq[start:stop:step] - library/itertools,,:stop,elements from seq[start:stop:step] - library/itertools,,::,kernel = tuple(kernel)[::-1] -diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst -index 0064e074a3..1756a37338 100644 ---- a/Doc/whatsnew/3.9.rst -+++ b/Doc/whatsnew/3.9.rst -@@ -1616,3 +1616,12 @@ tarfile - :exc:`DeprecationWarning`. - In Python 3.14, the default will switch to ``'data'``. - (Contributed by Petr Viktorin in :pep:`706`.) -+ -+Notable changes in 3.9.20 -+========================= -+ -+ipaddress -+--------- -+ -+* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, -+ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. -diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py -index 25f373a06a..9b35340d9a 100644 ---- a/Lib/ipaddress.py -+++ b/Lib/ipaddress.py -@@ -1322,18 +1322,41 @@ class IPv4Address(_BaseV4, _BaseAddress): - @property - @functools.lru_cache() - def is_private(self): -- """Test if this address is allocated for private networks. -+ """``True`` if the address is defined as not globally reachable by -+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -+ (for IPv6) with the following exceptions: - -- Returns: -- A boolean, True if the address is reserved per -- iana-ipv4-special-registry. -+ * ``is_private`` is ``False`` for ``100.64.0.0/10`` -+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: - -+ address.is_private == address.ipv4_mapped.is_private -+ -+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` -+ IPv4 range where they are both ``False``. - """ -- return any(self in net for net in self._constants._private_networks) -+ return ( -+ any(self in net for net in self._constants._private_networks) -+ and all(self not in net for net in self._constants._private_networks_exceptions) -+ ) - - @property - @functools.lru_cache() - def is_global(self): -+ """``True`` if the address is defined as globally reachable by -+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -+ (for IPv6) with the following exception: -+ -+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: -+ -+ address.is_global == address.ipv4_mapped.is_global -+ -+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` -+ IPv4 range where they are both ``False``. -+ """ - return self not in self._constants._public_network and not self.is_private - - @property -@@ -1537,13 +1560,15 @@ class _IPv4Constants: - - _public_network = IPv4Network('100.64.0.0/10') - -+ # Not globally reachable address blocks listed on -+ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml - _private_networks = [ - IPv4Network('0.0.0.0/8'), - IPv4Network('10.0.0.0/8'), - IPv4Network('127.0.0.0/8'), - IPv4Network('169.254.0.0/16'), - IPv4Network('172.16.0.0/12'), -- IPv4Network('192.0.0.0/29'), -+ IPv4Network('192.0.0.0/24'), - IPv4Network('192.0.0.170/31'), - IPv4Network('192.0.2.0/24'), - IPv4Network('192.168.0.0/16'), -@@ -1554,6 +1579,11 @@ class _IPv4Constants: - IPv4Network('255.255.255.255/32'), - ] - -+ _private_networks_exceptions = [ -+ IPv4Network('192.0.0.9/32'), -+ IPv4Network('192.0.0.10/32'), -+ ] -+ - _reserved_network = IPv4Network('240.0.0.0/4') - - _unspecified_address = IPv4Address('0.0.0.0') -@@ -1995,23 +2025,42 @@ class IPv6Address(_BaseV6, _BaseAddress): - @property - @functools.lru_cache() - def is_private(self): -- """Test if this address is allocated for private networks. -+ """``True`` if the address is defined as not globally reachable by -+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -+ (for IPv6) with the following exceptions: - -- Returns: -- A boolean, True if the address is reserved per -- iana-ipv6-special-registry. -+ * ``is_private`` is ``False`` for ``100.64.0.0/10`` -+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: - -+ address.is_private == address.ipv4_mapped.is_private -+ -+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` -+ IPv4 range where they are both ``False``. - """ -- return any(self in net for net in self._constants._private_networks) -+ ipv4_mapped = self.ipv4_mapped -+ if ipv4_mapped is not None: -+ return ipv4_mapped.is_private -+ return ( -+ any(self in net for net in self._constants._private_networks) -+ and all(self not in net for net in self._constants._private_networks_exceptions) -+ ) - - @property - def is_global(self): -- """Test if this address is allocated for public networks. -+ """``True`` if the address is defined as globally reachable by -+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ -+ (for IPv6) with the following exception: - -- Returns: -- A boolean, true if the address is not reserved per -- iana-ipv6-special-registry. -+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the -+ semantics of the underlying IPv4 addresses and the following condition holds -+ (see :attr:`IPv6Address.ipv4_mapped`):: - -+ address.is_global == address.ipv4_mapped.is_global -+ -+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` -+ IPv4 range where they are both ``False``. - """ - return not self.is_private - -@@ -2252,19 +2301,31 @@ class _IPv6Constants: - - _multicast_network = IPv6Network('ff00::/8') - -+ # Not globally reachable address blocks listed on -+ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml - _private_networks = [ - IPv6Network('::1/128'), - IPv6Network('::/128'), - IPv6Network('::ffff:0:0/96'), -+ IPv6Network('64:ff9b:1::/48'), - IPv6Network('100::/64'), - IPv6Network('2001::/23'), -- IPv6Network('2001:2::/48'), - IPv6Network('2001:db8::/32'), -- IPv6Network('2001:10::/28'), -+ # IANA says N/A, let's consider it not globally reachable to be safe -+ IPv6Network('2002::/16'), - IPv6Network('fc00::/7'), - IPv6Network('fe80::/10'), - ] - -+ _private_networks_exceptions = [ -+ IPv6Network('2001:1::1/128'), -+ IPv6Network('2001:1::2/128'), -+ IPv6Network('2001:3::/32'), -+ IPv6Network('2001:4:112::/48'), -+ IPv6Network('2001:20::/28'), -+ IPv6Network('2001:30::/28'), -+ ] -+ - _reserved_networks = [ - IPv6Network('::/8'), IPv6Network('100::/8'), - IPv6Network('200::/7'), IPv6Network('400::/6'), -diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py -index 90897f6bed..bd14f04f6c 100644 ---- a/Lib/test/test_ipaddress.py -+++ b/Lib/test/test_ipaddress.py -@@ -2263,6 +2263,10 @@ class IpaddrUnitTest(unittest.TestCase): - self.assertEqual(True, ipaddress.ip_address( - '172.31.255.255').is_private) - self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) -+ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) -+ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) -+ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) -+ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) - - self.assertEqual(True, - ipaddress.ip_address('169.254.100.200').is_link_local) -@@ -2278,6 +2282,40 @@ class IpaddrUnitTest(unittest.TestCase): - self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) - self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) - -+ def testPrivateNetworks(self): -+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private) -+ self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) -+ -+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) -+ self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) -+ self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) -+ self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) -+ self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) -+ self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) -+ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) -+ self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) -+ self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) -+ self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) -+ self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) -+ self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) -+ self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) -+ self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) -+ self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) -+ -+ self.assertEqual(False, ipaddress.ip_network("::/0").is_private) -+ self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) -+ -+ self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) -+ self.assertEqual(True, ipaddress.ip_network("::/128").is_private) -+ self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) -+ self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) -+ self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) -+ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) -+ self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) -+ self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) -+ self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) -+ self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) -+ - def testReservedIpv6(self): - - self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) -@@ -2351,6 +2389,20 @@ class IpaddrUnitTest(unittest.TestCase): - self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) - self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) - -+ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) -+ self.assertFalse(ipaddress.ip_address('2001::').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) -+ self.assertFalse(ipaddress.ip_address('2001:2::').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:3::').is_global) -+ self.assertFalse(ipaddress.ip_address('2001:4::').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) -+ self.assertFalse(ipaddress.ip_address('2001:10::').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:20::').is_global) -+ self.assertTrue(ipaddress.ip_address('2001:30::').is_global) -+ self.assertFalse(ipaddress.ip_address('2001:40::').is_global) -+ self.assertFalse(ipaddress.ip_address('2002::').is_global) -+ - # some generic IETF reserved addresses - self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) - self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) -diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst -new file mode 100644 -index 0000000000..f9a72473be ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst -@@ -0,0 +1,9 @@ -+Fixed various false positives and false negatives in -+ -+* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) -+* :attr:`ipaddress.IPv4Address.is_global` -+* :attr:`ipaddress.IPv6Address.is_private` -+* :attr:`ipaddress.IPv6Address.is_global` -+ -+Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` -+attributes. diff --git a/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch b/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch deleted file mode 100644 index 432920d..0000000 --- a/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch +++ /dev/null @@ -1,356 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Petr Viktorin -Date: Wed, 31 Jul 2024 00:19:48 +0200 -Subject: [PATCH] 00435: gh-121650: Encode newlines in headers, and verify - headers are sound (GH-122233) - -Per RFC 2047: - -> [...] these encoding schemes allow the -> encoding of arbitrary octet values, mail readers that implement this -> decoding should also ensure that display of the decoded data on the -> recipient's terminal will not cause unwanted side-effects - -It seems that the "quoted-word" scheme is a valid way to include -a newline character in a header value, just like we already allow -undecodable bytes or control characters. -They do need to be properly quoted when serialized to text, though. - -This should fail for custom fold() implementations that aren't careful -about newlines. - -(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384) - -Co-authored-by: Petr Viktorin -Co-authored-by: Bas Bloemsaat -Co-authored-by: Serhiy Storchaka ---- - Doc/library/email.errors.rst | 6 ++ - Doc/library/email.policy.rst | 18 ++++++ - Doc/whatsnew/3.9.rst | 12 ++++ - Lib/email/_header_value_parser.py | 12 +++- - Lib/email/_policybase.py | 8 +++ - Lib/email/errors.py | 4 ++ - Lib/email/generator.py | 13 +++- - Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++ - Lib/test/test_email/test_policy.py | 26 ++++++++ - ...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++ - 10 files changed, 162 insertions(+), 4 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst - -diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst -index f4b9f52509..878c09bb04 100644 ---- a/Doc/library/email.errors.rst -+++ b/Doc/library/email.errors.rst -@@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module: - :class:`~email.mime.image.MIMEImage`). - - -+.. exception:: HeaderWriteError() -+ -+ Raised when an error occurs when the :mod:`~email.generator` outputs -+ headers. -+ -+ - Here is the list of the defects that the :class:`~email.parser.FeedParser` - can find while parsing messages. Note that the defects are added to the message - where the problem was found, so for example, if a message nested inside a -diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst -index bf53b9520f..57a75ce452 100644 ---- a/Doc/library/email.policy.rst -+++ b/Doc/library/email.policy.rst -@@ -229,6 +229,24 @@ added matters. To illustrate:: - - .. versionadded:: 3.6 - -+ -+ .. attribute:: verify_generated_headers -+ -+ If ``True`` (the default), the generator will raise -+ :exc:`~email.errors.HeaderWriteError` instead of writing a header -+ that is improperly folded or delimited, such that it would -+ be parsed as multiple headers or joined with adjacent data. -+ Such headers can be generated by custom header classes or bugs -+ in the ``email`` module. -+ -+ As it's a security feature, this defaults to ``True`` even in the -+ :class:`~email.policy.Compat32` policy. -+ For backwards compatible, but unsafe, behavior, it must be set to -+ ``False`` explicitly. -+ -+ .. versionadded:: 3.9.20 -+ -+ - The following :class:`Policy` method is intended to be called by code using - the email library to create policy instances with custom settings: - -diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst -index 1756a37338..eeda4e6955 100644 ---- a/Doc/whatsnew/3.9.rst -+++ b/Doc/whatsnew/3.9.rst -@@ -1625,3 +1625,15 @@ ipaddress - - * Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, - ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. -+ -+email -+----- -+ -+* Headers with embedded newlines are now quoted on output. -+ -+ The :mod:`~email.generator` will now refuse to serialize (write) headers -+ that are improperly folded or delimited, such that they would be parsed as -+ multiple headers or joined with adjacent data. -+ If you need to turn this safety feature off, -+ set :attr:`~email.policy.Policy.verify_generated_headers`. -+ (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.) -diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py -index 8a8fb8bc42..e394cfd2e1 100644 ---- a/Lib/email/_header_value_parser.py -+++ b/Lib/email/_header_value_parser.py -@@ -92,6 +92,8 @@ TOKEN_ENDS = TSPECIALS | WSP - ASPECIALS = TSPECIALS | set("*'%") - ATTRIBUTE_ENDS = ASPECIALS | WSP - EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') -+NLSET = {'\n', '\r'} -+SPECIALSNL = SPECIALS | NLSET - - def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -@@ -2778,9 +2780,13 @@ def _refold_parse_tree(parse_tree, *, policy): - wrap_as_ew_blocked -= 1 - continue - tstr = str(part) -- if part.token_type == 'ptext' and set(tstr) & SPECIALS: -- # Encode if tstr contains special characters. -- want_encoding = True -+ if not want_encoding: -+ if part.token_type == 'ptext': -+ # Encode if tstr contains special characters. -+ want_encoding = not SPECIALSNL.isdisjoint(tstr) -+ else: -+ # Encode if tstr contains newlines. -+ want_encoding = not NLSET.isdisjoint(tstr) - try: - tstr.encode(encoding) - charset = encoding -diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py -index c9cbadd2a8..d1f48211f9 100644 ---- a/Lib/email/_policybase.py -+++ b/Lib/email/_policybase.py -@@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): - message_factory -- the class to use to create new message objects. - If the value is None, the default is Message. - -+ verify_generated_headers -+ -- if true, the generator verifies that each header -+ they are properly folded, so that a parser won't -+ treat it as multiple headers, start-of-body, or -+ part of another header. -+ This is a check against custom Header & fold() -+ implementations. - """ - - raise_on_defect = False -@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): - max_line_length = 78 - mangle_from_ = False - message_factory = None -+ verify_generated_headers = True - - def handle_defect(self, obj, defect): - """Based on policy, either raise defect or call register_defect. -diff --git a/Lib/email/errors.py b/Lib/email/errors.py -index d28a680010..1a0d5c63e6 100644 ---- a/Lib/email/errors.py -+++ b/Lib/email/errors.py -@@ -29,6 +29,10 @@ class CharsetError(MessageError): - """An illegal charset was given.""" - - -+class HeaderWriteError(MessageError): -+ """Error while writing headers.""" -+ -+ - # These are parsing defects which the parser was able to work around. - class MessageDefect(ValueError): - """Base class for a message defect.""" -diff --git a/Lib/email/generator.py b/Lib/email/generator.py -index c9b121624e..89224ae41c 100644 ---- a/Lib/email/generator.py -+++ b/Lib/email/generator.py -@@ -14,12 +14,14 @@ import random - from copy import deepcopy - from io import StringIO, BytesIO - from email.utils import _has_surrogates -+from email.errors import HeaderWriteError - - UNDERSCORE = '_' - NL = '\n' # XXX: no longer used by the code below. - - NLCRE = re.compile(r'\r\n|\r|\n') - fcre = re.compile(r'^From ', re.MULTILINE) -+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') - - - -@@ -223,7 +225,16 @@ class Generator: - - def _write_headers(self, msg): - for h, v in msg.raw_items(): -- self.write(self.policy.fold(h, v)) -+ folded = self.policy.fold(h, v) -+ if self.policy.verify_generated_headers: -+ linesep = self.policy.linesep -+ if not folded.endswith(self.policy.linesep): -+ raise HeaderWriteError( -+ f'folded header does not end with {linesep!r}: {folded!r}') -+ if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): -+ raise HeaderWriteError( -+ f'folded header contains newline: {folded!r}') -+ self.write(folded) - # A blank line always separates headers from body - self.write(self._NL) - -diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py -index 89e7edeb63..d29400f0ed 100644 ---- a/Lib/test/test_email/test_generator.py -+++ b/Lib/test/test_email/test_generator.py -@@ -6,6 +6,7 @@ from email.message import EmailMessage - from email.generator import Generator, BytesGenerator - from email.headerregistry import Address - from email import policy -+import email.errors - from test.test_email import TestEmailBase, parameterize - - -@@ -216,6 +217,44 @@ class TestGeneratorBase: - g.flatten(msg) - self.assertEqual(s.getvalue(), self.typ(expected)) - -+ def test_keep_encoded_newlines(self): -+ msg = self.msgmaker(self.typ(textwrap.dedent("""\ -+ To: nobody -+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com -+ -+ None -+ """))) -+ expected = textwrap.dedent("""\ -+ To: nobody -+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com -+ -+ None -+ """) -+ s = self.ioclass() -+ g = self.genclass(s, policy=self.policy.clone(max_line_length=80)) -+ g.flatten(msg) -+ self.assertEqual(s.getvalue(), self.typ(expected)) -+ -+ def test_keep_long_encoded_newlines(self): -+ msg = self.msgmaker(self.typ(textwrap.dedent("""\ -+ To: nobody -+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com -+ -+ None -+ """))) -+ expected = textwrap.dedent("""\ -+ To: nobody -+ Subject: Bad subject -+ =?utf-8?q?=0A?=Bcc: -+ injection@example.com -+ -+ None -+ """) -+ s = self.ioclass() -+ g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) -+ g.flatten(msg) -+ self.assertEqual(s.getvalue(), self.typ(expected)) -+ - - class TestGenerator(TestGeneratorBase, TestEmailBase): - -@@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase): - ioclass = io.StringIO - typ = str - -+ def test_verify_generated_headers(self): -+ """gh-121650: by default the generator prevents header injection""" -+ class LiteralHeader(str): -+ name = 'Header' -+ def fold(self, **kwargs): -+ return self -+ -+ for text in ( -+ 'Value\r\nBad Injection\r\n', -+ 'NoNewLine' -+ ): -+ with self.subTest(text=text): -+ message = message_from_string( -+ "Header: Value\r\n\r\nBody", -+ policy=self.policy, -+ ) -+ -+ del message['Header'] -+ message['Header'] = LiteralHeader(text) -+ -+ with self.assertRaises(email.errors.HeaderWriteError): -+ message.as_string() -+ - - class TestBytesGenerator(TestGeneratorBase, TestEmailBase): - -diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py -index e87c275549..ff1ddf7d7a 100644 ---- a/Lib/test/test_email/test_policy.py -+++ b/Lib/test/test_email/test_policy.py -@@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase): - 'raise_on_defect': False, - 'mangle_from_': True, - 'message_factory': None, -+ 'verify_generated_headers': True, - } - # These default values are the ones set on email.policy.default. - # If any of these defaults change, the docs must be updated. -@@ -277,6 +278,31 @@ class PolicyAPITests(unittest.TestCase): - with self.assertRaises(email.errors.HeaderParseError): - policy.fold("Subject", subject) - -+ def test_verify_generated_headers(self): -+ """Turning protection off allows header injection""" -+ policy = email.policy.default.clone(verify_generated_headers=False) -+ for text in ( -+ 'Header: Value\r\nBad: Injection\r\n', -+ 'Header: NoNewLine' -+ ): -+ with self.subTest(text=text): -+ message = email.message_from_string( -+ "Header: Value\r\n\r\nBody", -+ policy=policy, -+ ) -+ class LiteralHeader(str): -+ name = 'Header' -+ def fold(self, **kwargs): -+ return self -+ -+ del message['Header'] -+ message['Header'] = LiteralHeader(text) -+ -+ self.assertEqual( -+ message.as_string(), -+ f"{text}\nBody", -+ ) -+ - # XXX: Need subclassing tests. - # For adding subclassed objects, make sure the usual rules apply (subclass - # wins), but that the order still works (right overrides left). -diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst -new file mode 100644 -index 0000000000..83dd28d4ac ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst -@@ -0,0 +1,5 @@ -+:mod:`email` headers with embedded newlines are now quoted on output. The -+:mod:`~email.generator` will now refuse to serialize (write) headers that -+are unsafely folded or delimited; see -+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas -+Bloemsaat and Petr Viktorin in :gh:`121650`.) diff --git a/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch b/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch deleted file mode 100644 index fed0497..0000000 --- a/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch +++ /dev/null @@ -1,128 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: "Jason R. Coombs" -Date: Mon, 19 Aug 2024 19:28:20 -0400 -Subject: [PATCH] 00436: [CVE-2024-8088] gh-122905: Sanitize names in - zipfile.Path. - -Co-authored-by: Jason R. Coombs ---- - Lib/test/test_zipfile.py | 17 ++++++ - Lib/zipfile.py | 61 ++++++++++++++++++- - ...-08-11-14-08-04.gh-issue-122905.7tDsxA.rst | 1 + - 3 files changed, 78 insertions(+), 1 deletion(-) - create mode 100644 Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst - -diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py -index 17e95eb862..9a72152357 100644 ---- a/Lib/test/test_zipfile.py -+++ b/Lib/test/test_zipfile.py -@@ -3054,6 +3054,23 @@ class TestPath(unittest.TestCase): - data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] - zipfile.CompleteDirs._implied_dirs(data) - -+ def test_malformed_paths(self): -+ """ -+ Path should handle malformed paths. -+ """ -+ data = io.BytesIO() -+ zf = zipfile.ZipFile(data, "w") -+ zf.writestr("/one-slash.txt", b"content") -+ zf.writestr("//two-slash.txt", b"content") -+ zf.writestr("../parent.txt", b"content") -+ zf.filename = '' -+ root = zipfile.Path(zf) -+ assert list(map(str, root.iterdir())) == [ -+ 'one-slash.txt', -+ 'two-slash.txt', -+ 'parent.txt', -+ ] -+ - - if __name__ == "__main__": - unittest.main() -diff --git a/Lib/zipfile.py b/Lib/zipfile.py -index 95f95ee112..2e9b2868cd 100644 ---- a/Lib/zipfile.py -+++ b/Lib/zipfile.py -@@ -9,6 +9,7 @@ import io - import itertools - import os - import posixpath -+import re - import shutil - import stat - import struct -@@ -2177,7 +2178,65 @@ def _difference(minuend, subtrahend): - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - --class CompleteDirs(ZipFile): -+class SanitizedNames: -+ """ -+ ZipFile mix-in to ensure names are sanitized. -+ """ -+ -+ def namelist(self): -+ return list(map(self._sanitize, super().namelist())) -+ -+ @staticmethod -+ def _sanitize(name): -+ r""" -+ Ensure a relative path with posix separators and no dot names. -+ Modeled after -+ https://github.com/python/cpython/blob/bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c/Lib/zipfile/__init__.py#L1799-L1813 -+ but provides consistent cross-platform behavior. -+ >>> san = SanitizedNames._sanitize -+ >>> san('/foo/bar') -+ 'foo/bar' -+ >>> san('//foo.txt') -+ 'foo.txt' -+ >>> san('foo/.././bar.txt') -+ 'foo/bar.txt' -+ >>> san('foo../.bar.txt') -+ 'foo../.bar.txt' -+ >>> san('\\foo\\bar.txt') -+ 'foo/bar.txt' -+ >>> san('D:\\foo.txt') -+ 'D/foo.txt' -+ >>> san('\\\\server\\share\\file.txt') -+ 'server/share/file.txt' -+ >>> san('\\\\?\\GLOBALROOT\\Volume3') -+ '?/GLOBALROOT/Volume3' -+ >>> san('\\\\.\\PhysicalDrive1\\root') -+ 'PhysicalDrive1/root' -+ Retain any trailing slash. -+ >>> san('abc/') -+ 'abc/' -+ Raises a ValueError if the result is empty. -+ >>> san('../..') -+ Traceback (most recent call last): -+ ... -+ ValueError: Empty filename -+ """ -+ -+ def allowed(part): -+ return part and part not in {'..', '.'} -+ -+ # Remove the drive letter. -+ # Don't use ntpath.splitdrive, because that also strips UNC paths -+ bare = re.sub('^([A-Z]):', r'\1', name, flags=re.IGNORECASE) -+ clean = bare.replace('\\', '/') -+ parts = clean.split('/') -+ joined = '/'.join(filter(allowed, parts)) -+ if not joined: -+ raise ValueError("Empty filename") -+ return joined + '/' * name.endswith('/') -+ -+ -+class CompleteDirs(SanitizedNames, ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. -diff --git a/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst -new file mode 100644 -index 0000000000..1be44c906c ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst -@@ -0,0 +1 @@ -+:class:`zipfile.Path` objects now sanitize names from the zipfile. diff --git a/python3.9.spec b/python3.9.spec index 6cc8bc9..e3df36c 100644 --- a/python3.9.spec +++ b/python3.9.spec @@ -13,11 +13,11 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.19 +%global general_version %{pybasever}.20 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 6%{?dist} +Release: 1%{?dist} License: Python @@ -385,74 +385,6 @@ Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-g # gh-99086: Fix implicit int compiler warning in configure check for PTHREAD_SCOPE_SYSTEM Patch407: 00407-gh-99086-fix-implicit-int-compiler-warning-in-configure-check-for-pthread_scope_system.patch -# 00415 # 512c60eb23a8d7b26d74824a6d7bbefb6feefb65 -# [CVE-2023-27043] gh-102988: Reject malformed addresses in email.parseaddr() (#111116) -# -# Detect email address parsing errors and return empty tuple to -# indicate the parsing error (old API). Add an optional 'strict' -# parameter to getaddresses() and parseaddr() functions. Patch by -# Thomas Dwyer. -Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch - -# 00419 # f13682530cc7e4daec2e40acd56508846fdd3aad -# gh-112769: test_zlib: Fix comparison of ZLIB_RUNTIME_VERSION with non-int suffix (GH-112771) (GH-112774) -# -# zlib-ng defines the version as "1.3.0.zlib-ng". -Patch419: 00419-gh-112769-test_zlib-fix-comparison-of-zlib_runtime_version-with-non-int-suffix-gh-112771-gh-112774.patch - -# 00431 # 1216ca6d2f3d6fc7ef7bf33b78b7a887be02e467 -# gh-113171: gh-65056: Fix "private" (non-global) IP address ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472) -# -# The _private_networks variables, used by various is_private -# implementations, were missing some ranges and at the same time had -# overly strict ranges (where there are more specific ranges considered -# globally reachable by the IANA registries). -# -# This patch updates the ranges with what was missing or otherwise -# incorrect. -# -# 100.64.0.0/10 is left alone, for now, as it's been made special in [1]. -# -# The _address_exclude_many() call returns 8 networks for IPv4, 121 -# networks for IPv6. -# -# [1] https://github.com/python/cpython/issues/61602 -# -# In 3.10 and below, is_private checks whether the network and broadcast -# address are both private. -# In later versions (where the test wss backported from), it checks -# whether they both are in the same private network. -# -# For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private, -# but one is in 0.0.0.0/8 ("This network") and the other in -# 255.255.255.255/32 ("Limited broadcast"). -# -# --------- -Patch431: 00431-gh-113171-gh-65056-fix-private-non-global-ip-address-ranges-gh-113179-gh-113186-gh-118177-gh-118472.patch - -# 00435 # f2924d30f4dd44804219c10410a57dd96764d297 -# gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233) -# -# Per RFC 2047: -# -# > [...] these encoding schemes allow the -# > encoding of arbitrary octet values, mail readers that implement this -# > decoding should also ensure that display of the decoded data on the -# > recipient's terminal will not cause unwanted side-effects -# -# It seems that the "quoted-word" scheme is a valid way to include -# a newline character in a header value, just like we already allow -# undecodable bytes or control characters. -# They do need to be properly quoted when serialized to text, though. -# -# This should fail for custom fold() implementations that aren't careful -# about newlines. -Patch435: 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch - -# 00436 # 506dd77b7132f69ada7185b8bb91eba0e1296aa8 -# [CVE-2024-8088] gh-122905: Sanitize names in zipfile.Path. -Patch436: 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch - # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1900,6 +1832,9 @@ CheckPython optimized # ====================================================== %changelog +* Mon Sep 09 2024 Tomáš Hrnčiar - 3.9.20-1 +- Update to 3.9.20 + * Fri Aug 23 2024 Charalampos Stratakis - 3.9.19-6 - Security fix for CVE-2024-8088 - Fixes: rhbz#2307466 diff --git a/sources b/sources index be10285..b5a8526 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.9.19.tar.xz) = 5577830c734e63a70bbc62cd33d263b9aa87c4381b49cb694c3559067c4c682a55506b65ec5514a8e0a5abf6294dc728e909385d449ae1c388e62f83cea9bb89 -SHA512 (Python-3.9.19.tar.xz.asc) = f7f4946243dfc56de2c84f50276b088d347f17054f50e3331d1e312e2a8e2c6ed1b4b4a807202b51137fd2af3fc9218cafa42ed348a954ace896d9a432e2defd +SHA512 (Python-3.9.20.tar.xz) = c828f33edf1704e3149499d6d34e89264cb5cdb2b09ff05561641b359716d7996f0fe928629e09f006b1fd7850fdaf937275919c7fdd83f5efc32707c64d814b +SHA512 (Python-3.9.20.tar.xz.asc) = f21c012f4f642542479ba329da9654589e5a7f7305c39fb1b6f136b578316bdb115cef9773c9a9fe4e195677af01cb80af05780613cca83f42fae131862a9584