rpms
/
python3
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update to 3.7.4rc1 

- reenable test_asyncio
 - drop merged CVE fixes (patches 320, 324)
This commit is contained in:
Miro Hrončok 2019-06-25 16:08:20 +02:00
parent 9941fbbeb9
commit dd5f80b1b8
5 changed files with 9 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
00320-fix-pre-normalization-chars-in-urlsplit.patch
View File

@ -1,42 +0,0 @@
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
index 0faf2bb..d0365ec 100644
--- a/Lib/test/test_urlparse.py
+++ b/Lib/test/test_urlparse.py
@@ -1011,6 +1011,12 @@ class UrlParseTestCase(unittest.TestCase):
self.assertIn('\u2100', denorm_chars)
self.assertIn('\uFF03', denorm_chars)
+ # bpo-36742: Verify port separators are ignored when they
+ # existed prior to decomposition
+ urllib.parse.urlsplit('http://\u30d5\u309a:80')
+ with self.assertRaises(ValueError):
+ urllib.parse.urlsplit('http://\u30d5\u309a\ufe1380')
+
for scheme in ["http", "https", "ftp"]:
for c in denorm_chars:
url = "{}://netloc{}false.netloc/path".format(scheme, c)
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index 8b6c9b1..e2f7b69 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -402,13 +402,16 @@ def _checknetloc(netloc):
# looking for characters like \u2100 that expand to 'a/c'
# IDNA uses NFKC equivalence, so normalize for this check
import unicodedata
- netloc2 = unicodedata.normalize('NFKC', netloc)
- if netloc == netloc2:
+ n = netloc.rpartition('@')[2] # ignore anything to the left of '@'
+ n = n.replace(':', '') # ignore characters already included
+ n = n.replace('#', '') # but not the surrounding text
+ n = n.replace('?', '')
+ netloc2 = unicodedata.normalize('NFKC', n)
+ if n == netloc2:
return
- _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay
for c in '/?#@:':
if c in netloc2:
- raise ValueError("netloc '" + netloc2 + "' contains invalid " +
+ raise ValueError("netloc '" + netloc + "' contains invalid " +
"characters under NFKC normalization")
def urlsplit(url, scheme='', allow_fragments=True):

150
00324-disallow-control-chars-in-http-urls.patch
View File

@ -1,150 +0,0 @@
From 7e200e0763f5b71c199aaf98bd5588f291585619 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
Date: Tue, 7 May 2019 17:28:47 +0200
Subject: [PATCH] bpo-30458: Disallow control chars in http URLs. (GH-12755)
(GH-13154)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Disallow control chars in http URLs in urllib.urlopen. This addresses a potential security problem for applications that do not sanity check their URLs where http request headers could be injected.
Disable https related urllib tests on a build without ssl (GH-13032)
These tests require an SSL enabled build. Skip these tests when python is built without SSL to fix test failures.
Use http.client.InvalidURL instead of ValueError as the new error case's exception. (GH-13044)
Backport Co-Authored-By: Miro Hrončok <miro@hroncok.cz>
---
Lib/http/client.py | 15 ++++++
Lib/test/test_urllib.py | 53 +++++++++++++++++++
Lib/test/test_xmlrpc.py | 7 ++-
.../2019-04-10-08-53-30.bpo-30458.51E-DA.rst | 1 +
4 files changed, 75 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
diff --git a/Lib/http/client.py b/Lib/http/client.py
index 1de151c38e..2afd452fe3 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -140,6 +140,16 @@ _MAXHEADERS = 100
_is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
_is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
+# These characters are not allowed within HTTP URL paths.
+# See https://tools.ietf.org/html/rfc3986#section-3.3 and the
+# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
+# Prevents CVE-2019-9740. Includes control characters such as \r\n.
+# We don't restrict chars above \x7f as putrequest() limits us to ASCII.
+_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
+# Arguably only these _should_ allowed:
+# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
+# We are more lenient for assumed real world compatibility purposes.
+
# We always set the Content-Length header for these methods because some
# servers will otherwise respond with a 411
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
@@ -1101,6 +1111,11 @@ class HTTPConnection:
self._method = method
if not url:
url = '/'
+ # Prevent CVE-2019-9740.
+ match = _contains_disallowed_url_pchar_re.search(url)
+ if match:
+ raise InvalidURL(f"URL can't contain control characters. {url!r} "
+ f"(found at least {match.group()!r})")
request = '%s %s %s' % (method, url, self._http_vsn_str)
# Non-ASCII characters should have been eliminated earlier
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 2ac73b58d8..7214492eca 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -329,6 +329,59 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin):
finally:
self.unfakehttp()
+ @unittest.skipUnless(ssl, "ssl module required")
+ def test_url_with_control_char_rejected(self):
+ for char_no in list(range(0, 0x21)) + [0x7f]:
+ char = chr(char_no)
+ schemeless_url = f"//localhost:7777/test{char}/"
+ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.")
+ try:
+ # We explicitly test urllib.request.urlopen() instead of the top
+ # level 'def urlopen()' function defined in this... (quite ugly)
+ # test suite. They use different url opening codepaths. Plain
+ # urlopen uses FancyURLOpener which goes via a codepath that
+ # calls urllib.parse.quote() on the URL which makes all of the
+ # above attempts at injection within the url _path_ safe.
+ escaped_char_repr = repr(char).replace('\\', r'\\')
+ InvalidURL = http.client.InvalidURL
+ with self.assertRaisesRegex(
+ InvalidURL, f"contain control.*{escaped_char_repr}"):
+ urllib.request.urlopen(f"http:{schemeless_url}")
+ with self.assertRaisesRegex(
+ InvalidURL, f"contain control.*{escaped_char_repr}"):
+ urllib.request.urlopen(f"https:{schemeless_url}")
+ # This code path quotes the URL so there is no injection.
+ resp = urlopen(f"http:{schemeless_url}")
+ self.assertNotIn(char, resp.geturl())
+ finally:
+ self.unfakehttp()
+
+ @unittest.skipUnless(ssl, "ssl module required")
+ def test_url_with_newline_header_injection_rejected(self):
+ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.")
+ host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123"
+ schemeless_url = "//" + host + ":8080/test/?test=a"
+ try:
+ # We explicitly test urllib.request.urlopen() instead of the top
+ # level 'def urlopen()' function defined in this... (quite ugly)
+ # test suite. They use different url opening codepaths. Plain
+ # urlopen uses FancyURLOpener which goes via a codepath that
+ # calls urllib.parse.quote() on the URL which makes all of the
+ # above attempts at injection within the url _path_ safe.
+ InvalidURL = http.client.InvalidURL
+ with self.assertRaisesRegex(
+ InvalidURL, r"contain control.*\\r.*(found at least . .)"):
+ urllib.request.urlopen(f"http:{schemeless_url}")
+ with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"):
+ urllib.request.urlopen(f"https:{schemeless_url}")
+ # This code path quotes the URL so there is no injection.
+ resp = urlopen(f"http:{schemeless_url}")
+ self.assertNotIn(' ', resp.geturl())
+ self.assertNotIn('\r', resp.geturl())
+ self.assertNotIn('\n', resp.geturl())
+ finally:
+ self.unfakehttp()
+
def test_read_0_9(self):
# "0.9" response accepted (but not "simple responses" without
# a status line)
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
index 32263f7f0b..0e002ec4ef 100644
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -945,7 +945,12 @@ class SimpleServerTestCase(BaseServerTestCase):
def test_partial_post(self):
# Check that a partial POST doesn't make the server loop: issue #14001.
conn = http.client.HTTPConnection(ADDR, PORT)
- conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
+ conn.send('POST /RPC2 HTTP/1.0\r\n'
+ 'Content-Length: 100\r\n\r\n'
+ 'bye HTTP/1.1\r\n'
+ f'Host: {ADDR}:{PORT}\r\n'
+ 'Accept-Encoding: identity\r\n'
+ 'Content-Length: 0\r\n\r\n'.encode('ascii'))
conn.close()
def test_context_manager(self):
diff --git a/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
new file mode 100644
index 0000000000..ed8027fb4d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
@@ -0,0 +1 @@
+Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised.
--
2.21.0

26
python3.spec
View File

@ -13,11 +13,11 @@ URL: https://www.python.org/
# WARNING When rebasing to a new Python version,
# remember to update the python3-docs package as well
%global general_version %{pybasever}.3
#global prerel ...
%global general_version %{pybasever}.4
%global prerel rc1
%global upstream_version %{general_version}%{?prerel}
Version: %{general_version}%{?prerel:~%{prerel}}
Release: 3%{?dist}
Release: 1%{?dist}
License: Python
@ -282,18 +282,6 @@ Patch274: 00274-fix-arch-names.patch
# So we mark the command as unsupported - and the tests are skipped
Patch316: 00316-mark-bdist_wininst-unsupported.patch
# 00320 #
# Fix handling of pre-normalization characters in urlsplit()
# This fixes a regression introduced by the fix for CVE-2019-9636
# Fixed upstream: https://bugs.python.org/issue36742
Patch320: 00320-fix-pre-normalization-chars-in-urlsplit.patch
# 00324 #
# Disallow control chars in http URLs
# Security fix for CVE-2019-9740 and CVE-2019-9947
# Fixed upstream: https://bugs.python.org/issue30458
Patch324: 00324-disallow-control-chars-in-http-urls.patch
# (New patches go here ^^^)
#
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@ -587,8 +575,6 @@ rm Lib/ensurepip/_bundled/*.whl
%patch251 -p1
%patch274 -p1
%patch316 -p1
%patch320 -p1
%patch324 -p1
# Remove files that should be generated by the build
@ -999,8 +985,6 @@ CheckPython() {
# https://bugzilla.redhat.com/show_bug.cgi?id=1196181
# test_gdb skipped on s390x:
# https://bugzilla.redhat.com/show_bug.cgi?id=1678277
# test_asyncio skipped:
# https://bugs.python.org/issue35998
LD_LIBRARY_PATH=$ConfDir $ConfDir/python -m test.regrtest \
-wW --slowest -j0 \
-x test_distutils \
@ -1011,7 +995,6 @@ CheckPython() {
%ifarch %{mips64}
-x test_ctypes \
%endif
-x test_asyncio \
echo FINISHED: CHECKING OF PYTHON FOR CONFIGURATION: $ConfName
@ -1508,6 +1491,9 @@ CheckPython optimized
# ======================================================
%changelog
* Tue Jun 25 2019 Miro Hrončok <mhroncok@redhat.com> - 3.7.4~rc1-1
- Update to 3.7.4rc1
* Tue May 07 2019 Charalampos Stratakis <cstratak@redhat.com> - 3.7.3-3
- Fix handling of pre-normalization characters in urlsplit
- Disallow control chars in http URLs (#1695572, #1700684, #1688169, #1706851)

2
sources
View File

@ -1 +1 @@
SHA512 (Python-3.7.3.tar.xz) = 6d9b7c0f1764e0f655a39430a3af6f7b5e3c9b7166c042e780677a54b17ad4ca6d0d9cba262c82b1b70bba8f7c28883dad4cc0d7cc194fc7d2c1b5f4f08a763a
SHA512 (Python-3.7.4rc1.tar.xz) = a6a901de2f70d09d16e722daf5687d0c13d9244ab3a2bb2971d6240c0c680836def3d23d517d0dc20d9badcd6583e34a14d35a8818656c422664e07c3cbd5497

4
tests/tests.yml
View File

@ -13,10 +13,10 @@
run: VERSION=3.7 ./venv.sh
- selftest:
dir: python/selftest
run: VERSION=3.7 X="-x test_socket -x test_asyncgen -x test_asyncio -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./parallel.sh
run: VERSION=3.7 X="-x test_socket -x test_asyncgen -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./parallel.sh
- debugtest:
dir: python/selftest
run: VERSION=3.7 PYTHON=python3-debug X="-x test_socket -x test_asyncgen -x test_asyncio -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./parallel.sh
run: VERSION=3.7 PYTHON=python3-debug X="-x test_socket -x test_asyncgen -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./parallel.sh
required_packages:
- gcc # for extension building in venv and selftest
- gdb # for test_gdb