From ec7d75e3d1217d306fe2feda665fef3986c8ae78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 4 Jan 2023 12:24:50 +0100 Subject: [PATCH 01/17] Opt-out from https://fedoraproject.org/wiki/Changes/fno-omit-frame-pointer See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/6TQYCHMX4FZLF27U5BCEC7IFV6XNBKJP/ for rationale, namely https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/message/ZVDEXGPU6JQFXB3XHYZ4IXVQNNR3YM3V/ Summary: Python is currently slower with frame pointers due to a slowdown in _PyEval_EvalFrameDefault, but we expect this to be solved in Python 3.12. Tracking bugzilla: https://bugzilla.redhat.com/2158729 This change does not require a release bump. It is only needed to be here to prevent the next builds from including frame pointers. --- python3.11.spec | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python3.11.spec b/python3.11.spec index e870792..5fac9d8 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -175,6 +175,12 @@ Obsoletes: python%{pybasever}%{?1:-%{1}}\ %define unversioned_obsoletes_of_python3_X_if_main() %{nil} %endif +# Opt-out from https://fedoraproject.org/wiki/Changes/fno-omit-frame-pointer +# Python is slower with frame pointers, but we expect to remove this in Python 3.12+ +# See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/6TQYCHMX4FZLF27U5BCEC7IFV6XNBKJP/ +# Tracking bugzilla: https://bugzilla.redhat.com/2158729 +%undefine _include_frame_pointers + # ======================= # Build-time requirements # ======================= From 06ef941d951e5a25f6a4acbacee286a7e97c1af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 6 Jan 2023 15:37:38 +0100 Subject: [PATCH 02/17] Fix `asyncio` subprocess losing `stderr` and `stdout` output Reported as a regression in https://bodhi.fedoraproject.org/updates/FEDORA-2022-dbb811d203 --- ...cess-losing-stderr-and-stdout-output.patch | 66 +++++++++++++++++++ python3.11.spec | 9 ++- 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch diff --git a/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch b/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch new file mode 100644 index 0000000..c480127 --- /dev/null +++ b/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch @@ -0,0 +1,66 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Wed, 21 Dec 2022 02:24:19 -0800 +Subject: [PATCH] 00395: GH-100133: fix `asyncio` subprocess losing `stderr` + and `stdout` output + +(cherry picked from commit a7715ccfba5b86ab09f86ec56ac3755c93b46b48) + +Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> +--- + Lib/asyncio/base_subprocess.py | 3 --- + Lib/test/test_asyncio/test_subprocess.py | 17 +++++++++++++++++ + ...22-12-10-08-36-07.gh-issue-100133.g-zQlp.rst | 1 + + 3 files changed, 18 insertions(+), 3 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst + +diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py +index e15bb4141f..4c9b0dd565 100644 +--- a/Lib/asyncio/base_subprocess.py ++++ b/Lib/asyncio/base_subprocess.py +@@ -215,9 +215,6 @@ def _process_exited(self, returncode): + # object. On Python 3.6, it is required to avoid a ResourceWarning. + self._proc.returncode = returncode + self._call(self._protocol.process_exited) +- for p in self._pipes.values(): +- if p is not None: +- p.pipe.close() + + self._try_finish() + +diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py +index f71ad72f99..bea2314a52 100644 +--- a/Lib/test/test_asyncio/test_subprocess.py ++++ b/Lib/test/test_asyncio/test_subprocess.py +@@ -684,6 +684,23 @@ async def execute(): + + self.assertIsNone(self.loop.run_until_complete(execute())) + ++ def test_subprocess_communicate_stdout(self): ++ # See https://github.com/python/cpython/issues/100133 ++ async def get_command_stdout(cmd, *args): ++ proc = await asyncio.create_subprocess_exec( ++ cmd, *args, stdout=asyncio.subprocess.PIPE, ++ ) ++ stdout, _ = await proc.communicate() ++ return stdout.decode().strip() ++ ++ async def main(): ++ outputs = [f'foo{i}' for i in range(10)] ++ res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', ++ f'print({out!r})') for out in outputs]) ++ self.assertEqual(res, outputs) ++ ++ self.loop.run_until_complete(main()) ++ + + if sys.platform != 'win32': + # Unix +diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst +new file mode 100644 +index 0000000000..881e6ed80f +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst +@@ -0,0 +1 @@ ++Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. diff --git a/python3.11.spec b/python3.11.spec index 5fac9d8..3c01a3b 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 2%{?dist} +Release: 3%{?dist} License: Python-2.0.1 @@ -315,6 +315,10 @@ Patch251: 00251-change-user-install-location.patch # https://github.com/GrahamDumpleton/mod_wsgi/issues/730 Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch +# 00395 # 18ff37a92c507144edf32274b356dd1dd734cf07 +# GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output +Patch395: 00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch + # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1597,6 +1601,9 @@ CheckPython optimized # ====================================================== %changelog +* Fri Jan 06 2023 Miro Hrončok - 3.11.1-3 +- Fix `asyncio` subprocess losing `stderr` and `stdout` output + * Mon Dec 19 2022 Miro Hrončok - 3.11.1-2 - No longer patch the default bytecode cache invalidation policy From 01ddee3bb650edd1e5f94296568774aff0db0ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 11 Jan 2023 15:25:19 +0100 Subject: [PATCH 03/17] Remove any deprecation warnings in asyncio.get_event_loop() The warnings were added in 3.11.1 and will be reverted in 3.11.2. They make some Fedora packages fail to build, so we backport the revert. --- ...n-warnings-in-asyncio-get_event_loop.patch | 171 ++++++++++++++++++ python3.11.spec | 7 + 2 files changed, 178 insertions(+) create mode 100644 00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch diff --git a/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch b/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch new file mode 100644 index 0000000..bebae56 --- /dev/null +++ b/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch @@ -0,0 +1,171 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Serhiy Storchaka +Date: Tue, 10 Jan 2023 22:20:09 +0200 +Subject: [PATCH] 00396: gh-100160: Remove any deprecation warnings in + asyncio.get_event_loop() + +Some deprecation warnings will reappear (in a slightly different form) in 3.12. + +Co-authored-by: Guido van Rossum +--- + Doc/library/asyncio-eventloop.rst | 14 +++++++------- + Doc/library/asyncio-policy.rst | 9 +++++---- + Doc/whatsnew/3.10.rst | 13 ------------- + Lib/asyncio/events.py | 15 --------------- + Lib/test/test_asyncio/test_events.py | 12 +++--------- + ...2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst | 2 ++ + 6 files changed, 17 insertions(+), 48 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst + +diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst +index 28b7a90058..886399e7ae 100644 +--- a/Doc/library/asyncio-eventloop.rst ++++ b/Doc/library/asyncio-eventloop.rst +@@ -48,7 +48,7 @@ an event loop: + running event loop. + + If there is no running event loop set, the function will return +- the result of ``get_event_loop_policy().get_event_loop()`` call. ++ the result of the ``get_event_loop_policy().get_event_loop()`` call. + + Because this function has rather complex behavior (especially + when custom event loop policies are in use), using the +@@ -59,15 +59,15 @@ an event loop: + instead of using these lower level functions to manually create and close an + event loop. + +- .. deprecated:: 3.10 +- Deprecation warning is emitted if there is no current event loop. +- In Python 3.12 it will be an error. +- + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function +- (and other functions which used it implicitly) emitted a ++ (and other functions which use it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if +- the current loop was set. ++ the current loop was set on the policy. ++ In Python versions 3.10.9, 3.11.1 and 3.12 they emit a ++ :exc:`DeprecationWarning` if there is no running event loop and no ++ current loop is set. ++ In some future Python release this will become an error. + + .. function:: set_event_loop(loop) + +diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst +index d0af45febd..eb043b3e5e 100644 +--- a/Doc/library/asyncio-policy.rst ++++ b/Doc/library/asyncio-policy.rst +@@ -112,10 +112,11 @@ asyncio ships with the following built-in policies: + + On Windows, :class:`ProactorEventLoop` is now used by default. + +- .. deprecated:: 3.11.1 +- :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there +- is no current event loop set and a new event loop has been implicitly +- created. In Python 3.12 it will be an error. ++ .. note:: ++ In Python versions 3.10.9, 3.11.1 and 3.12 this function emits a ++ :exc:`DeprecationWarning` if there is no running event loop and no ++ current loop is set. ++ In some future Python release this will become an error. + + + .. class:: WindowsSelectorEventLoopPolicy +diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst +index d0b436664a..38b30deff7 100644 +--- a/Doc/whatsnew/3.10.rst ++++ b/Doc/whatsnew/3.10.rst +@@ -1710,19 +1710,6 @@ Deprecated + scheduled for removal in Python 3.12. + (Contributed by Erlend E. Aasland in :issue:`42264`.) + +-* :func:`asyncio.get_event_loop` now emits a deprecation warning if there is +- no running event loop. In the future it will be an alias of +- :func:`~asyncio.get_running_loop`. +- :mod:`asyncio` functions which implicitly create :class:`~asyncio.Future` +- or :class:`~asyncio.Task` objects now emit +- a deprecation warning if there is no running event loop and no explicit +- *loop* argument is passed: :func:`~asyncio.ensure_future`, +- :func:`~asyncio.wrap_future`, :func:`~asyncio.gather`, +- :func:`~asyncio.shield`, :func:`~asyncio.as_completed` and constructors of +- :class:`~asyncio.Future`, :class:`~asyncio.Task`, +- :class:`~asyncio.StreamReader`, :class:`~asyncio.StreamReaderProtocol`. +- (Contributed by Serhiy Storchaka in :issue:`39529`.) +- + * The undocumented built-in function ``sqlite3.enable_shared_cache`` is now + deprecated, scheduled for removal in Python 3.12. Its use is strongly + discouraged by the SQLite3 documentation. See `the SQLite3 docs +diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py +index af3f9e970b..b1799320ea 100644 +--- a/Lib/asyncio/events.py ++++ b/Lib/asyncio/events.py +@@ -671,21 +671,6 @@ def get_event_loop(self): + if (self._local._loop is None and + not self._local._set_called and + threading.current_thread() is threading.main_thread()): +- stacklevel = 2 +- try: +- f = sys._getframe(1) +- except AttributeError: +- pass +- else: +- while f: +- module = f.f_globals.get('__name__') +- if not (module == 'asyncio' or module.startswith('asyncio.')): +- break +- f = f.f_back +- stacklevel += 1 +- import warnings +- warnings.warn('There is no current event loop', +- DeprecationWarning, stacklevel=stacklevel) + self.set_event_loop(self.new_event_loop()) + + if self._local._loop is None: +diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py +index c431fea401..18c4fd15d9 100644 +--- a/Lib/test/test_asyncio/test_events.py ++++ b/Lib/test/test_asyncio/test_events.py +@@ -2547,9 +2547,7 @@ def test_event_loop_policy(self): + def test_get_event_loop(self): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsNone(policy._local._loop) +- with self.assertWarns(DeprecationWarning) as cm: +- loop = policy.get_event_loop() +- self.assertEqual(cm.filename, __file__) ++ loop = policy.get_event_loop() + self.assertIsInstance(loop, asyncio.AbstractEventLoop) + + self.assertIs(policy._local._loop, loop) +@@ -2563,10 +2561,8 @@ def test_get_event_loop_calls_set_event_loop(self): + policy, "set_event_loop", + wraps=policy.set_event_loop) as m_set_event_loop: + +- with self.assertWarns(DeprecationWarning) as cm: +- loop = policy.get_event_loop() ++ loop = policy.get_event_loop() + self.addCleanup(loop.close) +- self.assertEqual(cm.filename, __file__) + + # policy._local._loop must be set through .set_event_loop() + # (the unix DefaultEventLoopPolicy needs this call to attach +@@ -2755,10 +2751,8 @@ def test_get_event_loop_returns_running_loop2(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + +- with self.assertWarns(DeprecationWarning) as cm: +- loop2 = asyncio.get_event_loop() ++ loop2 = asyncio.get_event_loop() + self.addCleanup(loop2.close) +- self.assertEqual(cm.filename, __file__) + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() +diff --git a/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst b/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst +new file mode 100644 +index 0000000000..c3b518ca85 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst +@@ -0,0 +1,2 @@ ++Remove any deprecation warnings in :func:`asyncio.get_event_loop`. They are ++deferred to Python 3.12. diff --git a/python3.11.spec b/python3.11.spec index 3c01a3b..96dae6d 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -319,6 +319,12 @@ Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-g # GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output Patch395: 00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch +# 00396 # 4c775fbed65016fec5dfd66316559024d2af9135 +# gh-100160: Remove any deprecation warnings in asyncio.get_event_loop() +# +# Some deprecation warnings will reappear (in a slightly different form) in 3.12. +Patch396: 00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch + # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1603,6 +1609,7 @@ CheckPython optimized %changelog * Fri Jan 06 2023 Miro Hrončok - 3.11.1-3 - Fix `asyncio` subprocess losing `stderr` and `stdout` output +- Remove any deprecation warnings in asyncio.get_event_loop() * Mon Dec 19 2022 Miro Hrončok - 3.11.1-2 - No longer patch the default bytecode cache invalidation policy From 85c6584975cba261de7ea3c45f7ca3d48bff481b Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 20 Jan 2023 18:45:09 +0000 Subject: [PATCH 04/17] Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python3.11.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python3.11.spec b/python3.11.spec index 96dae6d..5d59c3d 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 3%{?dist} +Release: 4%{?dist} License: Python-2.0.1 @@ -1607,6 +1607,9 @@ CheckPython optimized # ====================================================== %changelog +* Fri Jan 20 2023 Fedora Release Engineering - 3.11.1-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + * Fri Jan 06 2023 Miro Hrončok - 3.11.1-3 - Fix `asyncio` subprocess losing `stderr` and `stdout` output - Remove any deprecation warnings in asyncio.get_event_loop() From db4f00169267a208de762af949ff98fb33f34838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 12 Jan 2023 16:39:34 +0000 Subject: [PATCH 05/17] Don't require pyproject-rpm-macros on RHEL See also https://src.fedoraproject.org/rpms/pyproject-rpm-macros/pull-request/345 --- python3.11.spec | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python3.11.spec b/python3.11.spec index 5d59c3d..f9e3975 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -484,7 +484,11 @@ Requires: %{pkgname}-libs%{?_isa} = %{version}-%{release} # But we want them when packages BuildRequire python3-devel Requires: (python-rpm-macros if rpm-build) Requires: (python3-rpm-macros if rpm-build) -Requires: (pyproject-rpm-macros if rpm-build) +# We omit this dependency on RHEL to avoid pulling the macros to AppStream: +# RHEL users can use the minimal implementation of %%pyproject_buildrequires +# from pyproject-srpm-macros instead. +# On Fedora, we keep this to avoid one additional round of %%generate_buildrequires. +%{!?rhel:Requires: (pyproject-rpm-macros if rpm-build)} %unversioned_obsoletes_of_python3_X_if_main devel From 0331299c7d019060c2dab6ab56c82e5f60aa4d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 8 Feb 2023 16:08:24 +0100 Subject: [PATCH 06/17] Update to 3.11.2 --- ...cess-losing-stderr-and-stdout-output.patch | 66 ------- ...n-warnings-in-asyncio-get_event_loop.patch | 171 ------------------ python3.11.spec | 19 +- sources | 4 +- 4 files changed, 9 insertions(+), 251 deletions(-) delete mode 100644 00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch delete mode 100644 00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch diff --git a/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch b/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch deleted file mode 100644 index c480127..0000000 --- a/00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch +++ /dev/null @@ -1,66 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: "Miss Islington (bot)" - <31488909+miss-islington@users.noreply.github.com> -Date: Wed, 21 Dec 2022 02:24:19 -0800 -Subject: [PATCH] 00395: GH-100133: fix `asyncio` subprocess losing `stderr` - and `stdout` output - -(cherry picked from commit a7715ccfba5b86ab09f86ec56ac3755c93b46b48) - -Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> ---- - Lib/asyncio/base_subprocess.py | 3 --- - Lib/test/test_asyncio/test_subprocess.py | 17 +++++++++++++++++ - ...22-12-10-08-36-07.gh-issue-100133.g-zQlp.rst | 1 + - 3 files changed, 18 insertions(+), 3 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst - -diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py -index e15bb4141f..4c9b0dd565 100644 ---- a/Lib/asyncio/base_subprocess.py -+++ b/Lib/asyncio/base_subprocess.py -@@ -215,9 +215,6 @@ def _process_exited(self, returncode): - # object. On Python 3.6, it is required to avoid a ResourceWarning. - self._proc.returncode = returncode - self._call(self._protocol.process_exited) -- for p in self._pipes.values(): -- if p is not None: -- p.pipe.close() - - self._try_finish() - -diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py -index f71ad72f99..bea2314a52 100644 ---- a/Lib/test/test_asyncio/test_subprocess.py -+++ b/Lib/test/test_asyncio/test_subprocess.py -@@ -684,6 +684,23 @@ async def execute(): - - self.assertIsNone(self.loop.run_until_complete(execute())) - -+ def test_subprocess_communicate_stdout(self): -+ # See https://github.com/python/cpython/issues/100133 -+ async def get_command_stdout(cmd, *args): -+ proc = await asyncio.create_subprocess_exec( -+ cmd, *args, stdout=asyncio.subprocess.PIPE, -+ ) -+ stdout, _ = await proc.communicate() -+ return stdout.decode().strip() -+ -+ async def main(): -+ outputs = [f'foo{i}' for i in range(10)] -+ res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', -+ f'print({out!r})') for out in outputs]) -+ self.assertEqual(res, outputs) -+ -+ self.loop.run_until_complete(main()) -+ - - if sys.platform != 'win32': - # Unix -diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst -new file mode 100644 -index 0000000000..881e6ed80f ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst -@@ -0,0 +1 @@ -+Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. diff --git a/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch b/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch deleted file mode 100644 index bebae56..0000000 --- a/00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch +++ /dev/null @@ -1,171 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Serhiy Storchaka -Date: Tue, 10 Jan 2023 22:20:09 +0200 -Subject: [PATCH] 00396: gh-100160: Remove any deprecation warnings in - asyncio.get_event_loop() - -Some deprecation warnings will reappear (in a slightly different form) in 3.12. - -Co-authored-by: Guido van Rossum ---- - Doc/library/asyncio-eventloop.rst | 14 +++++++------- - Doc/library/asyncio-policy.rst | 9 +++++---- - Doc/whatsnew/3.10.rst | 13 ------------- - Lib/asyncio/events.py | 15 --------------- - Lib/test/test_asyncio/test_events.py | 12 +++--------- - ...2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst | 2 ++ - 6 files changed, 17 insertions(+), 48 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst - -diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst -index 28b7a90058..886399e7ae 100644 ---- a/Doc/library/asyncio-eventloop.rst -+++ b/Doc/library/asyncio-eventloop.rst -@@ -48,7 +48,7 @@ an event loop: - running event loop. - - If there is no running event loop set, the function will return -- the result of ``get_event_loop_policy().get_event_loop()`` call. -+ the result of the ``get_event_loop_policy().get_event_loop()`` call. - - Because this function has rather complex behavior (especially - when custom event loop policies are in use), using the -@@ -59,15 +59,15 @@ an event loop: - instead of using these lower level functions to manually create and close an - event loop. - -- .. deprecated:: 3.10 -- Deprecation warning is emitted if there is no current event loop. -- In Python 3.12 it will be an error. -- - .. note:: - In Python versions 3.10.0--3.10.8 and 3.11.0 this function -- (and other functions which used it implicitly) emitted a -+ (and other functions which use it implicitly) emitted a - :exc:`DeprecationWarning` if there was no running event loop, even if -- the current loop was set. -+ the current loop was set on the policy. -+ In Python versions 3.10.9, 3.11.1 and 3.12 they emit a -+ :exc:`DeprecationWarning` if there is no running event loop and no -+ current loop is set. -+ In some future Python release this will become an error. - - .. function:: set_event_loop(loop) - -diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst -index d0af45febd..eb043b3e5e 100644 ---- a/Doc/library/asyncio-policy.rst -+++ b/Doc/library/asyncio-policy.rst -@@ -112,10 +112,11 @@ asyncio ships with the following built-in policies: - - On Windows, :class:`ProactorEventLoop` is now used by default. - -- .. deprecated:: 3.11.1 -- :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there -- is no current event loop set and a new event loop has been implicitly -- created. In Python 3.12 it will be an error. -+ .. note:: -+ In Python versions 3.10.9, 3.11.1 and 3.12 this function emits a -+ :exc:`DeprecationWarning` if there is no running event loop and no -+ current loop is set. -+ In some future Python release this will become an error. - - - .. class:: WindowsSelectorEventLoopPolicy -diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst -index d0b436664a..38b30deff7 100644 ---- a/Doc/whatsnew/3.10.rst -+++ b/Doc/whatsnew/3.10.rst -@@ -1710,19 +1710,6 @@ Deprecated - scheduled for removal in Python 3.12. - (Contributed by Erlend E. Aasland in :issue:`42264`.) - --* :func:`asyncio.get_event_loop` now emits a deprecation warning if there is -- no running event loop. In the future it will be an alias of -- :func:`~asyncio.get_running_loop`. -- :mod:`asyncio` functions which implicitly create :class:`~asyncio.Future` -- or :class:`~asyncio.Task` objects now emit -- a deprecation warning if there is no running event loop and no explicit -- *loop* argument is passed: :func:`~asyncio.ensure_future`, -- :func:`~asyncio.wrap_future`, :func:`~asyncio.gather`, -- :func:`~asyncio.shield`, :func:`~asyncio.as_completed` and constructors of -- :class:`~asyncio.Future`, :class:`~asyncio.Task`, -- :class:`~asyncio.StreamReader`, :class:`~asyncio.StreamReaderProtocol`. -- (Contributed by Serhiy Storchaka in :issue:`39529`.) -- - * The undocumented built-in function ``sqlite3.enable_shared_cache`` is now - deprecated, scheduled for removal in Python 3.12. Its use is strongly - discouraged by the SQLite3 documentation. See `the SQLite3 docs -diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py -index af3f9e970b..b1799320ea 100644 ---- a/Lib/asyncio/events.py -+++ b/Lib/asyncio/events.py -@@ -671,21 +671,6 @@ def get_event_loop(self): - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): -- stacklevel = 2 -- try: -- f = sys._getframe(1) -- except AttributeError: -- pass -- else: -- while f: -- module = f.f_globals.get('__name__') -- if not (module == 'asyncio' or module.startswith('asyncio.')): -- break -- f = f.f_back -- stacklevel += 1 -- import warnings -- warnings.warn('There is no current event loop', -- DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - - if self._local._loop is None: -diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py -index c431fea401..18c4fd15d9 100644 ---- a/Lib/test/test_asyncio/test_events.py -+++ b/Lib/test/test_asyncio/test_events.py -@@ -2547,9 +2547,7 @@ def test_event_loop_policy(self): - def test_get_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() - self.assertIsNone(policy._local._loop) -- with self.assertWarns(DeprecationWarning) as cm: -- loop = policy.get_event_loop() -- self.assertEqual(cm.filename, __file__) -+ loop = policy.get_event_loop() - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - - self.assertIs(policy._local._loop, loop) -@@ -2563,10 +2561,8 @@ def test_get_event_loop_calls_set_event_loop(self): - policy, "set_event_loop", - wraps=policy.set_event_loop) as m_set_event_loop: - -- with self.assertWarns(DeprecationWarning) as cm: -- loop = policy.get_event_loop() -+ loop = policy.get_event_loop() - self.addCleanup(loop.close) -- self.assertEqual(cm.filename, __file__) - - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach -@@ -2755,10 +2751,8 @@ def test_get_event_loop_returns_running_loop2(self): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - -- with self.assertWarns(DeprecationWarning) as cm: -- loop2 = asyncio.get_event_loop() -+ loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) -- self.assertEqual(cm.filename, __file__) - asyncio.set_event_loop(None) - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() -diff --git a/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst b/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst -new file mode 100644 -index 0000000000..c3b518ca85 ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2022-12-21-18-29-24.gh-issue-100160.isBmL5.rst -@@ -0,0 +1,2 @@ -+Remove any deprecation warnings in :func:`asyncio.get_event_loop`. They are -+deferred to Python 3.12. diff --git a/python3.11.spec b/python3.11.spec index f9e3975..b234c8b 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -13,11 +13,11 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.1 +%global general_version %{pybasever}.2 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 4%{?dist} +Release: 1%{?dist} License: Python-2.0.1 @@ -315,16 +315,6 @@ Patch251: 00251-change-user-install-location.patch # https://github.com/GrahamDumpleton/mod_wsgi/issues/730 Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch -# 00395 # 18ff37a92c507144edf32274b356dd1dd734cf07 -# GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output -Patch395: 00395-gh-100133-fix-asyncio-subprocess-losing-stderr-and-stdout-output.patch - -# 00396 # 4c775fbed65016fec5dfd66316559024d2af9135 -# gh-100160: Remove any deprecation warnings in asyncio.get_event_loop() -# -# Some deprecation warnings will reappear (in a slightly different form) in 3.12. -Patch396: 00396-gh-100160-remove-any-deprecation-warnings-in-asyncio-get_event_loop.patch - # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1453,6 +1443,7 @@ CheckPython optimized %{dynload_dir}/_ctypes_test.%{SOABI_optimized}.so %{dynload_dir}/_testbuffer.%{SOABI_optimized}.so %{dynload_dir}/_testcapi.%{SOABI_optimized}.so +%{dynload_dir}/_testclinic.%{SOABI_optimized}.so %{dynload_dir}/_testimportmultiple.%{SOABI_optimized}.so %{dynload_dir}/_testinternalcapi.%{SOABI_optimized}.so %{dynload_dir}/_testmultiphase.%{SOABI_optimized}.so @@ -1581,6 +1572,7 @@ CheckPython optimized %{dynload_dir}/_ctypes_test.%{SOABI_debug}.so %{dynload_dir}/_testbuffer.%{SOABI_debug}.so %{dynload_dir}/_testcapi.%{SOABI_debug}.so +%{dynload_dir}/_testclinic.%{SOABI_debug}.so %{dynload_dir}/_testimportmultiple.%{SOABI_debug}.so %{dynload_dir}/_testinternalcapi.%{SOABI_debug}.so %{dynload_dir}/_testmultiphase.%{SOABI_debug}.so @@ -1611,6 +1603,9 @@ CheckPython optimized # ====================================================== %changelog +* Wed Feb 08 2023 Tomáš Hrnčiar - 3.11.2-1 +- Update to 3.11.2 + * Fri Jan 20 2023 Fedora Release Engineering - 3.11.1-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild diff --git a/sources b/sources index 327dc94..4ed0e1f 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.11.1.tar.xz) = 5edd70c881e083c96199c60471f18f9ebc4c97a2d45dc66f89e16d7c3638d8a5d2cbf2e84b1be3d7f1178ce9f7fa4197884385c1ee3618ff66a538f872f318ed -SHA512 (Python-3.11.1.tar.xz.asc) = 81ed05c2adf38552bdc5ac761704f2720a646d56681a919a6bfa51f1a4b42cd14edb9c84d58664dbc8e7b561cd78d82ae6b10dda423e1fae543bc7fa4bf3f78e +SHA512 (Python-3.11.2.tar.xz) = 5684ec7eae2dce26facc54d448ccdb6901bbfa1cab03abbe8fd34e4268a2b701daa13df15903349492447035be78380d473389e8703b4e910a65b088d2462e8b +SHA512 (Python-3.11.2.tar.xz.asc) = 9d9d1c6dd6e56a916c6861cd0e7e623a165b0845bafe0acfcae27b4fe10a6b7015844e6b3f8deded26a763c935e32f565b2e12beb20c43fda16c11eba8d282a1 From 7b688dbf924e1bbbc154a79857de1040131a49da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 5 Apr 2023 13:51:34 +0200 Subject: [PATCH 07/17] Update to 3.11.3 --- python3.11.spec | 5 ++++- sources | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python3.11.spec b/python3.11.spec index b234c8b..367ccc4 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -13,7 +13,7 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.2 +%global general_version %{pybasever}.3 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} @@ -1603,6 +1603,9 @@ CheckPython optimized # ====================================================== %changelog +* Wed Apr 05 2023 Tomáš Hrnčiar - 3.11.3-1 +- Update to 3.11.3 + * Wed Feb 08 2023 Tomáš Hrnčiar - 3.11.2-1 - Update to 3.11.2 diff --git a/sources b/sources index 4ed0e1f..ecff347 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.11.2.tar.xz) = 5684ec7eae2dce26facc54d448ccdb6901bbfa1cab03abbe8fd34e4268a2b701daa13df15903349492447035be78380d473389e8703b4e910a65b088d2462e8b -SHA512 (Python-3.11.2.tar.xz.asc) = 9d9d1c6dd6e56a916c6861cd0e7e623a165b0845bafe0acfcae27b4fe10a6b7015844e6b3f8deded26a763c935e32f565b2e12beb20c43fda16c11eba8d282a1 +SHA512 (Python-3.11.3.tar.xz) = a3bba4b69322a47bfeefe42ba0fd7331b5b67fd2ab41441e2219d16ef8c6f307f1a48977afd073c18cfd24ec6cb1bfe0c4bb4b273031eb524dc7e9fb5fbcc3b6 +SHA512 (Python-3.11.3.tar.xz.asc) = 34bfb6e74e2bbec15bf3f653e32a3d705961b5f724e3f4713cf9b6b530f9c9e7e94f5cf855798bdf9763235b97a60ec9ad554d24cdc793e21a0c39540b5818d8 From 68ad779a6cba886f9f92ebb8eb22e192710bbcb1 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 24 May 2023 13:58:20 +0200 Subject: [PATCH 08/17] Fix for CVE-2023-24329 --- ...e-chars-in-urlsplit-gh-102508-104575.patch | 230 ++++++++++++++++++ python3.11.spec | 17 +- 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch diff --git a/00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch b/00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch new file mode 100644 index 0000000..3eb1d6d --- /dev/null +++ b/00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch @@ -0,0 +1,230 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Wed, 17 May 2023 14:41:25 -0700 +Subject: [PATCH] 00399: gh-102153: Start stripping C0 control and space chars + in `urlsplit` (GH-102508) (#104575) + +* gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) + +`urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595. + +This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%20any%20leading%20and%20trailing%20C0%20control%20or%20space%20from%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). + +--------- + +(cherry picked from commit 2f630e1ce18ad2e07428296532a68b11dc66ad10) + +Co-authored-by: Illia Volochii +Co-authored-by: Gregory P. Smith [Google] +--- + Doc/library/urllib.parse.rst | 46 +++++++++++++- + Lib/test/test_urlparse.py | 61 ++++++++++++++++++- + Lib/urllib/parse.py | 12 ++++ + ...-03-07-20-59-17.gh-issue-102153.14CLSZ.rst | 3 + + 4 files changed, 119 insertions(+), 3 deletions(-) + create mode 100644 Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst + +diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst +index 96b3965107..a326e82e30 100644 +--- a/Doc/library/urllib.parse.rst ++++ b/Doc/library/urllib.parse.rst +@@ -159,6 +159,10 @@ or on combining URL components into a URL string. + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + ++ .. warning:: ++ ++ :func:`urlparse` does not perform validation. See :ref:`URL parsing ++ security ` for details. + + .. versionchanged:: 3.2 + Added IPv6 URL parsing capabilities. +@@ -324,8 +328,14 @@ or on combining URL components into a URL string. + ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is + decomposed before parsing, no error will be raised. + +- Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline +- ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. ++ Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 ++ control and space characters are stripped from the URL. ``\n``, ++ ``\r`` and tab ``\t`` characters are removed from the URL at any position. ++ ++ .. warning:: ++ ++ :func:`urlsplit` does not perform validation. See :ref:`URL parsing ++ security ` for details. + + .. versionchanged:: 3.6 + Out-of-range port numbers now raise :exc:`ValueError`, instead of +@@ -338,6 +348,9 @@ or on combining URL components into a URL string. + .. versionchanged:: 3.10 + ASCII newline and tab characters are stripped from the URL. + ++ .. versionchanged:: 3.11.4 ++ Leading WHATWG C0 control and space characters are stripped from the URL. ++ + .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser + + .. function:: urlunsplit(parts) +@@ -414,6 +427,35 @@ or on combining URL components into a URL string. + or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned + without changes. + ++.. _url-parsing-security: ++ ++URL parsing security ++-------------------- ++ ++The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of ++inputs. They may not raise errors on inputs that other applications consider ++invalid. They may also succeed on some inputs that might not be considered ++URLs elsewhere. Their purpose is for practical functionality rather than ++purity. ++ ++Instead of raising an exception on unusual input, they may instead return some ++component parts as empty strings. Or components may contain more than perhaps ++they should. ++ ++We recommend that users of these APIs where the values may be used anywhere ++with security implications code defensively. Do some verification within your ++code before trusting a returned component part. Does that ``scheme`` make ++sense? Is that a sensible ``path``? Is there anything strange about that ++``hostname``? etc. ++ ++What constitutes a URL is not universally well defined. Different applications ++have different needs and desired constraints. For instance the living `WHATWG ++spec`_ describes what user facing web clients such as a web browser require. ++While :rfc:`3986` is more general. These functions incorporate some aspects of ++both, but cannot be claimed compliant with either. The APIs and existing user ++code with expectations on specific behaviors predate both standards leading us ++to be very cautious about making API behavior changes. ++ + .. _parsing-ascii-encoded-bytes: + + Parsing ASCII Encoded Bytes +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index b426110723..40f13d631c 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -649,6 +649,65 @@ def test_urlsplit_remove_unsafe_bytes(self): + self.assertEqual(p.scheme, "http") + self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment") + ++ def test_urlsplit_strip_url(self): ++ noise = bytes(range(0, 0x20 + 1)) ++ base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" ++ ++ url = noise.decode("utf-8") + base_url ++ p = urllib.parse.urlsplit(url) ++ self.assertEqual(p.scheme, "http") ++ self.assertEqual(p.netloc, "User:Pass@www.python.org:080") ++ self.assertEqual(p.path, "/doc/") ++ self.assertEqual(p.query, "query=yes") ++ self.assertEqual(p.fragment, "frag") ++ self.assertEqual(p.username, "User") ++ self.assertEqual(p.password, "Pass") ++ self.assertEqual(p.hostname, "www.python.org") ++ self.assertEqual(p.port, 80) ++ self.assertEqual(p.geturl(), base_url) ++ ++ url = noise + base_url.encode("utf-8") ++ p = urllib.parse.urlsplit(url) ++ self.assertEqual(p.scheme, b"http") ++ self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") ++ self.assertEqual(p.path, b"/doc/") ++ self.assertEqual(p.query, b"query=yes") ++ self.assertEqual(p.fragment, b"frag") ++ self.assertEqual(p.username, b"User") ++ self.assertEqual(p.password, b"Pass") ++ self.assertEqual(p.hostname, b"www.python.org") ++ self.assertEqual(p.port, 80) ++ self.assertEqual(p.geturl(), base_url.encode("utf-8")) ++ ++ # Test that trailing space is preserved as some applications rely on ++ # this within query strings. ++ query_spaces_url = "https://www.python.org:88/doc/?query= " ++ p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) ++ self.assertEqual(p.scheme, "https") ++ self.assertEqual(p.netloc, "www.python.org:88") ++ self.assertEqual(p.path, "/doc/") ++ self.assertEqual(p.query, "query= ") ++ self.assertEqual(p.port, 88) ++ self.assertEqual(p.geturl(), query_spaces_url) ++ ++ p = urllib.parse.urlsplit("www.pypi.org ") ++ # That "hostname" gets considered a "path" due to the ++ # trailing space and our existing logic... YUCK... ++ # and re-assembles via geturl aka unurlsplit into the original. ++ # django.core.validators.URLValidator (at least through v3.2) relies on ++ # this, for better or worse, to catch it in a ValidationError via its ++ # regular expressions. ++ # Here we test the basic round trip concept of such a trailing space. ++ self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") ++ ++ # with scheme as cache-key ++ url = "//www.python.org/" ++ scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") ++ for _ in range(2): ++ p = urllib.parse.urlsplit(url, scheme=scheme) ++ self.assertEqual(p.scheme, "https") ++ self.assertEqual(p.geturl(), "https://www.python.org/") ++ + def test_attributes_bad_port(self): + """Check handling of invalid ports.""" + for bytes in (False, True): +@@ -656,7 +715,7 @@ def test_attributes_bad_port(self): + for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port +- url = "http://" + netloc ++ url = "http://" + netloc + "/" + if bytes: + if netloc.isascii() and port.isascii(): + netloc = netloc.encode("ascii") +diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py +index 69631cbb81..4f06fd509e 100644 +--- a/Lib/urllib/parse.py ++++ b/Lib/urllib/parse.py +@@ -25,6 +25,10 @@ + scenarios for parsing, and for backward compatibility purposes, some + parsing quirks from older RFCs are retained. The testcases in + test_urlparse.py provides a good indicator of parsing behavior. ++ ++The WHATWG URL Parser spec should also be considered. We are not compliant with ++it either due to existing user code API behavior expectations (Hyrum's Law). ++It serves as a useful guide when making changes. + """ + + from collections import namedtuple +@@ -79,6 +83,10 @@ + '0123456789' + '+-.') + ++# Leading and trailing C0 control and space to be stripped per WHATWG spec. ++# == "".join([chr(i) for i in range(0, 0x20 + 1)]) ++_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' ++ + # Unsafe bytes to be removed per WHATWG spec + _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] + +@@ -452,6 +460,10 @@ def urlsplit(url, scheme='', allow_fragments=True): + """ + + url, scheme, _coerce_result = _coerce_args(url, scheme) ++ # Only lstrip url as some applications rely on preserving trailing space. ++ # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) ++ url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) ++ scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") +diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst +new file mode 100644 +index 0000000000..e57ac4ed3a +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst +@@ -0,0 +1,3 @@ ++:func:`urllib.parse.urlsplit` now strips leading C0 control and space ++characters following the specification for URLs defined by WHATWG in ++response to CVE-2023-24329. Patch by Illia Volochii. diff --git a/python3.11.spec b/python3.11.spec index 367ccc4..357ce57 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 1%{?dist} +Release: 2%{?dist} License: Python-2.0.1 @@ -315,6 +315,18 @@ Patch251: 00251-change-user-install-location.patch # https://github.com/GrahamDumpleton/mod_wsgi/issues/730 Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch +# 00399 # 62614243969f1c717a02a1c65e55ef173ad9a6dd +# gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) (#104575) +# +# * gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) +# +# `urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595. +# +# This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%%20any%%20leading%%20and%%20trailing%%20C0%%20control%%20or%%20space%%20from%%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). +# +# --------- +Patch399: 00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch + # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1603,6 +1615,9 @@ CheckPython optimized # ====================================================== %changelog +* Wed May 24 2023 Lumír Balhar - 3.11.3-2 +- Fix for CVE-2023-24329 + * Wed Apr 05 2023 Tomáš Hrnčiar - 3.11.3-1 - Update to 3.11.3 From 64d93401a313b91ae51de383313970835e388d8d Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Sat, 27 May 2023 00:55:16 +0200 Subject: [PATCH 09/17] Fixup for CVE-2023-24329 patch name --- ...split-gh-102508-104575.patch => 00399-cve-2023-24329.patch | 3 +-- python3.11.spec | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) rename 00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch => 00399-cve-2023-24329.patch (98%) diff --git a/00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch b/00399-cve-2023-24329.patch similarity index 98% rename from 00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch rename to 00399-cve-2023-24329.patch index 3eb1d6d..e7d5159 100644 --- a/00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch +++ b/00399-cve-2023-24329.patch @@ -2,8 +2,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 17 May 2023 14:41:25 -0700 -Subject: [PATCH] 00399: gh-102153: Start stripping C0 control and space chars - in `urlsplit` (GH-102508) (#104575) +Subject: [PATCH] 00399: CVE-2023-24329 * gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) diff --git a/python3.11.spec b/python3.11.spec index 357ce57..aa37a07 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -316,7 +316,7 @@ Patch251: 00251-change-user-install-location.patch Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch # 00399 # 62614243969f1c717a02a1c65e55ef173ad9a6dd -# gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) (#104575) +# CVE-2023-24329 # # * gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) # @@ -325,7 +325,7 @@ Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-g # This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%%20any%%20leading%%20and%%20trailing%%20C0%%20control%%20or%%20space%%20from%%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). # # --------- -Patch399: 00399-gh-102153-start-stripping-c0-control-and-space-chars-in-urlsplit-gh-102508-104575.patch +Patch399: 00399-cve-2023-24329.patch # (New patches go here ^^^) # From 8ca0a0a2d8cf70fc2d644aea41030c8dc3d81035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 7 Jun 2023 22:32:30 +0200 Subject: [PATCH 10/17] Update to 3.11.4 --- 00399-cve-2023-24329.patch | 229 ------------------------------------- python3.11.spec | 21 +--- sources | 4 +- 3 files changed, 8 insertions(+), 246 deletions(-) delete mode 100644 00399-cve-2023-24329.patch diff --git a/00399-cve-2023-24329.patch b/00399-cve-2023-24329.patch deleted file mode 100644 index e7d5159..0000000 --- a/00399-cve-2023-24329.patch +++ /dev/null @@ -1,229 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: "Miss Islington (bot)" - <31488909+miss-islington@users.noreply.github.com> -Date: Wed, 17 May 2023 14:41:25 -0700 -Subject: [PATCH] 00399: CVE-2023-24329 - -* gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) - -`urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595. - -This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%20any%20leading%20and%20trailing%20C0%20control%20or%20space%20from%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). - ---------- - -(cherry picked from commit 2f630e1ce18ad2e07428296532a68b11dc66ad10) - -Co-authored-by: Illia Volochii -Co-authored-by: Gregory P. Smith [Google] ---- - Doc/library/urllib.parse.rst | 46 +++++++++++++- - Lib/test/test_urlparse.py | 61 ++++++++++++++++++- - Lib/urllib/parse.py | 12 ++++ - ...-03-07-20-59-17.gh-issue-102153.14CLSZ.rst | 3 + - 4 files changed, 119 insertions(+), 3 deletions(-) - create mode 100644 Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst - -diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst -index 96b3965107..a326e82e30 100644 ---- a/Doc/library/urllib.parse.rst -+++ b/Doc/library/urllib.parse.rst -@@ -159,6 +159,10 @@ or on combining URL components into a URL string. - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - -+ .. warning:: -+ -+ :func:`urlparse` does not perform validation. See :ref:`URL parsing -+ security ` for details. - - .. versionchanged:: 3.2 - Added IPv6 URL parsing capabilities. -@@ -324,8 +328,14 @@ or on combining URL components into a URL string. - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - -- Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline -- ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. -+ Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 -+ control and space characters are stripped from the URL. ``\n``, -+ ``\r`` and tab ``\t`` characters are removed from the URL at any position. -+ -+ .. warning:: -+ -+ :func:`urlsplit` does not perform validation. See :ref:`URL parsing -+ security ` for details. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of -@@ -338,6 +348,9 @@ or on combining URL components into a URL string. - .. versionchanged:: 3.10 - ASCII newline and tab characters are stripped from the URL. - -+ .. versionchanged:: 3.11.4 -+ Leading WHATWG C0 control and space characters are stripped from the URL. -+ - .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser - - .. function:: urlunsplit(parts) -@@ -414,6 +427,35 @@ or on combining URL components into a URL string. - or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned - without changes. - -+.. _url-parsing-security: -+ -+URL parsing security -+-------------------- -+ -+The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of -+inputs. They may not raise errors on inputs that other applications consider -+invalid. They may also succeed on some inputs that might not be considered -+URLs elsewhere. Their purpose is for practical functionality rather than -+purity. -+ -+Instead of raising an exception on unusual input, they may instead return some -+component parts as empty strings. Or components may contain more than perhaps -+they should. -+ -+We recommend that users of these APIs where the values may be used anywhere -+with security implications code defensively. Do some verification within your -+code before trusting a returned component part. Does that ``scheme`` make -+sense? Is that a sensible ``path``? Is there anything strange about that -+``hostname``? etc. -+ -+What constitutes a URL is not universally well defined. Different applications -+have different needs and desired constraints. For instance the living `WHATWG -+spec`_ describes what user facing web clients such as a web browser require. -+While :rfc:`3986` is more general. These functions incorporate some aspects of -+both, but cannot be claimed compliant with either. The APIs and existing user -+code with expectations on specific behaviors predate both standards leading us -+to be very cautious about making API behavior changes. -+ - .. _parsing-ascii-encoded-bytes: - - Parsing ASCII Encoded Bytes -diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py -index b426110723..40f13d631c 100644 ---- a/Lib/test/test_urlparse.py -+++ b/Lib/test/test_urlparse.py -@@ -649,6 +649,65 @@ def test_urlsplit_remove_unsafe_bytes(self): - self.assertEqual(p.scheme, "http") - self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment") - -+ def test_urlsplit_strip_url(self): -+ noise = bytes(range(0, 0x20 + 1)) -+ base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" -+ -+ url = noise.decode("utf-8") + base_url -+ p = urllib.parse.urlsplit(url) -+ self.assertEqual(p.scheme, "http") -+ self.assertEqual(p.netloc, "User:Pass@www.python.org:080") -+ self.assertEqual(p.path, "/doc/") -+ self.assertEqual(p.query, "query=yes") -+ self.assertEqual(p.fragment, "frag") -+ self.assertEqual(p.username, "User") -+ self.assertEqual(p.password, "Pass") -+ self.assertEqual(p.hostname, "www.python.org") -+ self.assertEqual(p.port, 80) -+ self.assertEqual(p.geturl(), base_url) -+ -+ url = noise + base_url.encode("utf-8") -+ p = urllib.parse.urlsplit(url) -+ self.assertEqual(p.scheme, b"http") -+ self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") -+ self.assertEqual(p.path, b"/doc/") -+ self.assertEqual(p.query, b"query=yes") -+ self.assertEqual(p.fragment, b"frag") -+ self.assertEqual(p.username, b"User") -+ self.assertEqual(p.password, b"Pass") -+ self.assertEqual(p.hostname, b"www.python.org") -+ self.assertEqual(p.port, 80) -+ self.assertEqual(p.geturl(), base_url.encode("utf-8")) -+ -+ # Test that trailing space is preserved as some applications rely on -+ # this within query strings. -+ query_spaces_url = "https://www.python.org:88/doc/?query= " -+ p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) -+ self.assertEqual(p.scheme, "https") -+ self.assertEqual(p.netloc, "www.python.org:88") -+ self.assertEqual(p.path, "/doc/") -+ self.assertEqual(p.query, "query= ") -+ self.assertEqual(p.port, 88) -+ self.assertEqual(p.geturl(), query_spaces_url) -+ -+ p = urllib.parse.urlsplit("www.pypi.org ") -+ # That "hostname" gets considered a "path" due to the -+ # trailing space and our existing logic... YUCK... -+ # and re-assembles via geturl aka unurlsplit into the original. -+ # django.core.validators.URLValidator (at least through v3.2) relies on -+ # this, for better or worse, to catch it in a ValidationError via its -+ # regular expressions. -+ # Here we test the basic round trip concept of such a trailing space. -+ self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") -+ -+ # with scheme as cache-key -+ url = "//www.python.org/" -+ scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") -+ for _ in range(2): -+ p = urllib.parse.urlsplit(url, scheme=scheme) -+ self.assertEqual(p.scheme, "https") -+ self.assertEqual(p.geturl(), "https://www.python.org/") -+ - def test_attributes_bad_port(self): - """Check handling of invalid ports.""" - for bytes in (False, True): -@@ -656,7 +715,7 @@ def test_attributes_bad_port(self): - for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port -- url = "http://" + netloc -+ url = "http://" + netloc + "/" - if bytes: - if netloc.isascii() and port.isascii(): - netloc = netloc.encode("ascii") -diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py -index 69631cbb81..4f06fd509e 100644 ---- a/Lib/urllib/parse.py -+++ b/Lib/urllib/parse.py -@@ -25,6 +25,10 @@ - scenarios for parsing, and for backward compatibility purposes, some - parsing quirks from older RFCs are retained. The testcases in - test_urlparse.py provides a good indicator of parsing behavior. -+ -+The WHATWG URL Parser spec should also be considered. We are not compliant with -+it either due to existing user code API behavior expectations (Hyrum's Law). -+It serves as a useful guide when making changes. - """ - - from collections import namedtuple -@@ -79,6 +83,10 @@ - '0123456789' - '+-.') - -+# Leading and trailing C0 control and space to be stripped per WHATWG spec. -+# == "".join([chr(i) for i in range(0, 0x20 + 1)]) -+_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' -+ - # Unsafe bytes to be removed per WHATWG spec - _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] - -@@ -452,6 +460,10 @@ def urlsplit(url, scheme='', allow_fragments=True): - """ - - url, scheme, _coerce_result = _coerce_args(url, scheme) -+ # Only lstrip url as some applications rely on preserving trailing space. -+ # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) -+ url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) -+ scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) - - for b in _UNSAFE_URL_BYTES_TO_REMOVE: - url = url.replace(b, "") -diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst -new file mode 100644 -index 0000000000..e57ac4ed3a ---- /dev/null -+++ b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst -@@ -0,0 +1,3 @@ -+:func:`urllib.parse.urlsplit` now strips leading C0 control and space -+characters following the specification for URLs defined by WHATWG in -+response to CVE-2023-24329. Patch by Illia Volochii. diff --git a/python3.11.spec b/python3.11.spec index aa37a07..83123a7 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -13,11 +13,11 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.3 +%global general_version %{pybasever}.4 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 2%{?dist} +Release: 1%{?dist} License: Python-2.0.1 @@ -67,7 +67,7 @@ License: Python-2.0.1 # If the rpmwheels condition is disabled, we use the bundled wheel packages # from Python with the versions below. # This needs to be manually updated when we update Python. -%global pip_version 22.3.1 +%global pip_version 23.1.2 %global setuptools_version 65.5.0 # Expensive optimizations (mainly, profile-guided optimizations) @@ -315,18 +315,6 @@ Patch251: 00251-change-user-install-location.patch # https://github.com/GrahamDumpleton/mod_wsgi/issues/730 Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch -# 00399 # 62614243969f1c717a02a1c65e55ef173ad9a6dd -# CVE-2023-24329 -# -# * gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508) -# -# `urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595. -# -# This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%%20any%%20leading%%20and%%20trailing%%20C0%%20control%%20or%%20space%%20from%%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). -# -# --------- -Patch399: 00399-cve-2023-24329.patch - # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -1615,6 +1603,9 @@ CheckPython optimized # ====================================================== %changelog +* Wed Jun 07 2023 Tomáš Hrnčiar - 3.11.4-1 +- Update to 3.11.4 + * Wed May 24 2023 Lumír Balhar - 3.11.3-2 - Fix for CVE-2023-24329 diff --git a/sources b/sources index ecff347..6b8930e 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.11.3.tar.xz) = a3bba4b69322a47bfeefe42ba0fd7331b5b67fd2ab41441e2219d16ef8c6f307f1a48977afd073c18cfd24ec6cb1bfe0c4bb4b273031eb524dc7e9fb5fbcc3b6 -SHA512 (Python-3.11.3.tar.xz.asc) = 34bfb6e74e2bbec15bf3f653e32a3d705961b5f724e3f4713cf9b6b530f9c9e7e94f5cf855798bdf9763235b97a60ec9ad554d24cdc793e21a0c39540b5818d8 +SHA512 (Python-3.11.4.tar.xz) = 7eb14fecbf60824d10c22a9057584c3a142c2866f4af6caa2525c10c8bcb24e6e7afb32a44a0e118df0a2b2543d578c3b422ffd4a5fa317dfe6ea371cc7ee1ee +SHA512 (Python-3.11.4.tar.xz.asc) = 8ee82bf116b2cc7407e260eccf53e7fee4d7497165d0b9c3e59931c73f3b419bc0299b459eee9544a6e51e323ff0a6aa07827efd89f9c320b54556feeea04a78 From 5f98fe006c626e18849c6243ac1648caafe6004e Mon Sep 17 00:00:00 2001 From: Python Maint Date: Tue, 13 Jun 2023 15:09:02 +0200 Subject: [PATCH 11/17] Rebuilt for Python 3.12 --- python3.11.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python3.11.spec b/python3.11.spec index 83123a7..cf60fab 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 1%{?dist} +Release: 2%{?dist} License: Python-2.0.1 @@ -1603,6 +1603,9 @@ CheckPython optimized # ====================================================== %changelog +* Tue Jun 13 2023 Python Maint - 3.11.4-2 +- Rebuilt for Python 3.12 + * Wed Jun 07 2023 Tomáš Hrnčiar - 3.11.4-1 - Update to 3.11.4 From 458f7d2d2438abe3e438e4735e5943a3aa2a326f Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 21 Jul 2023 15:42:14 +0000 Subject: [PATCH 12/17] Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python3.11.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python3.11.spec b/python3.11.spec index cf60fab..5488d13 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 2%{?dist} +Release: 3%{?dist} License: Python-2.0.1 @@ -1603,6 +1603,9 @@ CheckPython optimized # ====================================================== %changelog +* Fri Jul 21 2023 Fedora Release Engineering - 3.11.4-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + * Tue Jun 13 2023 Python Maint - 3.11.4-2 - Rebuilt for Python 3.12 From ee27ca470f1db077c387c62671469d83df13e23d Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Thu, 3 Aug 2023 04:49:35 +0200 Subject: [PATCH 13/17] Remove extra distro-applied CFLAGS passed to user-built C extensions Only -fexceptions and -fcf-protection are preserved for binary compatibility with user-built python C extension. https://fedoraproject.org/wiki/Changes/Python_Extension_Flags_Reduction --- python3.11.spec | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python3.11.spec b/python3.11.spec index 5488d13..b071fef 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 3%{?dist} +Release: 4%{?dist} License: Python-2.0.1 @@ -670,14 +670,15 @@ topdir=$(pwd) # Standard library built here will still use the %%build_...flags, # Fedora packages utilizing %%py3_build will use them as well # https://fedoraproject.org/wiki/Changes/Python_Extension_Flags -export CFLAGS="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv" +# https://fedoraproject.org/wiki/Changes/Python_Extension_Flags_Reduction +export CFLAGS="%{extension_cflags}" export CFLAGS_NODIST="%{build_cflags} -D_GNU_SOURCE -fPIC -fwrapv" -export CXXFLAGS="%{extension_cxxflags} -D_GNU_SOURCE -fPIC -fwrapv" +export CXXFLAGS="%{extension_cxxflags}" export CPPFLAGS="$(pkg-config --cflags-only-I libffi)" -export OPT="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv" +export OPT="%{extension_cflags}" export LINKCC="gcc" export CFLAGS="$CFLAGS $(pkg-config --cflags openssl)" -export LDFLAGS="%{extension_ldflags} -g $(pkg-config --libs-only-L openssl)" +export LDFLAGS="%{extension_ldflags} $(pkg-config --libs-only-L openssl)" export LDFLAGS_NODIST="%{build_ldflags} -g $(pkg-config --libs-only-L openssl)" # We can build several different configurations of Python: regular and debug. @@ -1603,6 +1604,10 @@ CheckPython optimized # ====================================================== %changelog +* Wed Aug 02 2023 Charalampos Stratakis - 3.11.4-4 +- Remove extra distro-applied CFLAGS passed to user built C extensions +- https://fedoraproject.org/wiki/Changes/Python_Extension_Flags_Reduction + * Fri Jul 21 2023 Fedora Release Engineering - 3.11.4-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild From 32c38941f9bdcd5e89c207444b20ab568bbd48cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Tue, 29 Aug 2023 15:50:12 +0200 Subject: [PATCH 14/17] Update to 3.11.5 --- python3.11.spec | 13 ++++++++++--- sources | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python3.11.spec b/python3.11.spec index b071fef..d4a9fb8 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -13,11 +13,11 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.4 +%global general_version %{pybasever}.5 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 4%{?dist} +Release: 1%{?dist} License: Python-2.0.1 @@ -67,7 +67,7 @@ License: Python-2.0.1 # If the rpmwheels condition is disabled, we use the bundled wheel packages # from Python with the versions below. # This needs to be manually updated when we update Python. -%global pip_version 23.1.2 +%global pip_version 23.2.1 %global setuptools_version 65.5.0 # Expensive optimizations (mainly, profile-guided optimizations) @@ -1079,10 +1079,14 @@ CheckPython() { # test_freeze_simple_script is skipped, because it fails when bundled wheels # are removed in Fedora. # upstream report: https://bugs.python.org/issue45783 + # test_check_probes is failing since it was introduced in 3.12.0rc1, + # the test is skipped until it is fixed in upstream. + # see: https://github.com/python/cpython/issues/104280#issuecomment-1669249980 LD_LIBRARY_PATH=$ConfDir $ConfDir/python -m test.regrtest \ -wW --slowest -j0 --timeout=1800 \ -i test_freeze_simple_script \ + -i test_check_probes \ %if %{with bootstrap} -x test_distutils \ %endif @@ -1604,6 +1608,9 @@ CheckPython optimized # ====================================================== %changelog +* Mon Aug 28 2023 Tomáš Hrnčiar - 3.11.5-1 +- Update to 3.11.5 + * Wed Aug 02 2023 Charalampos Stratakis - 3.11.4-4 - Remove extra distro-applied CFLAGS passed to user built C extensions - https://fedoraproject.org/wiki/Changes/Python_Extension_Flags_Reduction diff --git a/sources b/sources index 6b8930e..be0708a 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.11.4.tar.xz) = 7eb14fecbf60824d10c22a9057584c3a142c2866f4af6caa2525c10c8bcb24e6e7afb32a44a0e118df0a2b2543d578c3b422ffd4a5fa317dfe6ea371cc7ee1ee -SHA512 (Python-3.11.4.tar.xz.asc) = 8ee82bf116b2cc7407e260eccf53e7fee4d7497165d0b9c3e59931c73f3b419bc0299b459eee9544a6e51e323ff0a6aa07827efd89f9c320b54556feeea04a78 +SHA512 (Python-3.11.5.tar.xz) = 93fa640bedcea449060caac8aa691aa315a19f172fd9f0422183d17749c3512d4ecac60e7599f9ef14e3cdb3c8b4b060e484c9061b1e7ee8d958200d6041e408 +SHA512 (Python-3.11.5.tar.xz.asc) = 5a8e1b1cabe89de03c050d581bbd3ec917d93ec943b2e8241db05c245809cf80294022c4cfc1bea3b90aa0570176109aac90455057256c025e2596aa136375fc From 7d571986dc19437d37bae51f4b32c44b320fd9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Tue, 29 Aug 2023 15:54:05 +0200 Subject: [PATCH 15/17] Temporarily skip test_check_probes in CI tests See: https://github.com/python/cpython/issues/104280#issuecomment-1669249980 --- tests/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.yml b/tests/tests.yml index 3f9165b..07e31cb 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -30,10 +30,10 @@ run: "PYTHON=python{{ pybasever }}d TOX=false VERSION={{ pybasever }} ./venv.sh" - selftest: dir: python/selftest - run: "VERSION={{ pybasever }} X='' ./parallel.sh" + run: "VERSION={{ pybasever }} X='-i test_check_probes' ./parallel.sh" - debugtest: dir: python/selftest - run: "VERSION={{ pybasever }} PYTHON=python{{ pybasever }}d X='' ./parallel.sh" + run: "VERSION={{ pybasever }} PYTHON=python{{ pybasever }}d X='-i test_check_probes' ./parallel.sh" - debugflags: dir: python/flags run: "python{{ pybasever }}d ./assertflags.py -O0" From 3254311fe9359731bd5c6721852d0043dfe1be6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Tue, 3 Oct 2023 10:14:10 +0200 Subject: [PATCH 16/17] Update to 3.11.6 --- check-pyc-timestamps.py | 6 +++--- python3.11.spec | 5 ++++- sources | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/check-pyc-timestamps.py b/check-pyc-timestamps.py index e421fca..0497eca 100644 --- a/check-pyc-timestamps.py +++ b/check-pyc-timestamps.py @@ -16,9 +16,9 @@ LEVELS = (None, 1, 2) # list of globs of test and other files that we expect not to have bytecode not_compiled = [ '/usr/bin/*', - '*/test/bad_coding.py', - '*/test/bad_coding2.py', - '*/test/badsyntax_*.py', + '*/test/*/bad_coding.py', + '*/test/*/bad_coding2.py', + '*/test/*/badsyntax_*.py', '*/lib2to3/tests/data/*.py', '*/lib2to3/tests/data/*/*.py', '*/lib2to3/tests/data/*/*/*.py', diff --git a/python3.11.spec b/python3.11.spec index d4a9fb8..baea8dc 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -13,7 +13,7 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well -%global general_version %{pybasever}.5 +%global general_version %{pybasever}.6 #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} @@ -1608,6 +1608,9 @@ CheckPython optimized # ====================================================== %changelog +* Tue Oct 03 2023 Tomáš Hrnčiar - 3.11.6-1 +- Update to 3.11.6 + * Mon Aug 28 2023 Tomáš Hrnčiar - 3.11.5-1 - Update to 3.11.5 diff --git a/sources b/sources index be0708a..05a350f 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (Python-3.11.5.tar.xz) = 93fa640bedcea449060caac8aa691aa315a19f172fd9f0422183d17749c3512d4ecac60e7599f9ef14e3cdb3c8b4b060e484c9061b1e7ee8d958200d6041e408 -SHA512 (Python-3.11.5.tar.xz.asc) = 5a8e1b1cabe89de03c050d581bbd3ec917d93ec943b2e8241db05c245809cf80294022c4cfc1bea3b90aa0570176109aac90455057256c025e2596aa136375fc +SHA512 (Python-3.11.6.tar.xz) = 94b1038f6f53de0c44f99f72ed0f2e0791fd9d2a325ae00ba145b2b2c332c27b300b3ea3473017518089478f15e01867b1bb203c16610039cce36f8366de341a +SHA512 (Python-3.11.6.tar.xz.asc) = 510f0b1393948c1490f81fbd90987e6f6b048b9f4d9df5814168097f5d9ac96e3682ff9bdc82d35b351eff5a4cc75015c28253b1dbbb2d94780411157c8beb25 From 1b480eac01d93d0383eeb6a79fcc18048d34b950 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Thu, 7 Sep 2023 22:54:51 -0400 Subject: [PATCH 17/17] Use bundled libb2 in RHEL builds Standalone libb2 is unwanted in RHEL. --- python3.11.spec | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python3.11.spec b/python3.11.spec index baea8dc..0ea7d64 100644 --- a/python3.11.spec +++ b/python3.11.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 1%{?dist} +Release: 2%{?dist} License: Python-2.0.1 @@ -205,7 +205,9 @@ BuildRequires: glibc-devel BuildRequires: gmp-devel BuildRequires: gnupg2 BuildRequires: libappstream-glib +%if %{undefined rhel} BuildRequires: libb2-devel +%endif BuildRequires: libffi-devel BuildRequires: libnsl2-devel BuildRequires: libtirpc-devel @@ -434,6 +436,8 @@ This package contains /usr/bin/python - the "python" command that runs Python 3. %package -n %{pkgname}-libs Summary: Python runtime libraries +# Bundled libb2 is CC0, covered by grandfathering exception +License: Python-2.0.1 AND CC0-1.0 %if %{with rpmwheels} Requires: %{python_wheel_pkg_prefix}-setuptools-wheel @@ -445,6 +449,10 @@ Provides: bundled(python3dist(setuptools)) = %{setuptools_version} %unversioned_obsoletes_of_python3_X_if_main libs +# Bundled internal headers are used even when building with system libb2 +# last updated by https://github.com/python/cpython/pull/6286 +Provides: bundled(libb2) = 0.98.1 + # There are files in the standard library that have python shebang. # We've filtered the automatic requirement out so libs are installable without # the main package. This however makes it pulled in by default. @@ -1608,6 +1616,9 @@ CheckPython optimized # ====================================================== %changelog +* Tue Oct 03 2023 Yaakov Selkowitz - 3.11.6-2 +- Use bundled libb2 in RHEL builds + * Tue Oct 03 2023 Tomáš Hrnčiar - 3.11.6-1 - Update to 3.11.6