From f5b3292c6845e91dfcf02b9fc3aab6a3a3e9bdd6 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Tue, 10 Jul 2018 01:54:54 -0500 Subject: [PATCH 01/10] Remove needless use of %defattr --- python2.spec | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python2.spec b/python2.spec index 8b614d7..2bf84ad 100644 --- a/python2.spec +++ b/python2.spec @@ -1583,7 +1583,6 @@ CheckPython \ %files -%defattr(-, root, root, -) %{!?_licensedir:%global license %%doc} %license LICENSE %doc README @@ -1597,7 +1596,6 @@ CheckPython \ %{_mandir}/*/python.1.* %files libs -%defattr(-,root,root,-) %{!?_licensedir:%global license %%doc} %license LICENSE %doc README @@ -1757,7 +1755,6 @@ CheckPython \ %files devel -%defattr(-,root,root,-) %{_libdir}/pkgconfig/python-%{pybasever}.pc %{_libdir}/pkgconfig/python.pc %{_libdir}/pkgconfig/python2.pc @@ -1773,7 +1770,6 @@ CheckPython \ %{_libdir}/libpython%{pybasever}.so %files tools -%defattr(-,root,root,755) %doc Tools/pynche/README.pynche %{site_packages}/pynche %{_bindir}/smtpd*.py* @@ -1790,14 +1786,12 @@ CheckPython \ %{pylibdir}/Doc %files tkinter -%defattr(-,root,root,755) %{pylibdir}/lib-tk %if ! 0%{?_module_build} %{dynload_dir}/_tkinter.so %endif %files test -%defattr(-, root, root, -) %{pylibdir}/bsddb/test %{pylibdir}/ctypes/test %{pylibdir}/distutils/tests @@ -1826,7 +1820,6 @@ CheckPython \ %if %{with debug_build} %files debug -%defattr(-,root,root,-) # Analog of the core subpackage's files: %{_bindir}/python-debug From fb398608462610bd519556a9d4e6b10ead41d696 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 14 Jul 2018 02:48:28 +0000 Subject: [PATCH 02/10] - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python2.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python2.spec b/python2.spec index 2bf84ad..86003b1 100644 --- a/python2.spec +++ b/python2.spec @@ -112,7 +112,7 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 5%{?dist} +Release: 6%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} @@ -1962,6 +1962,9 @@ CheckPython \ # ====================================================== %changelog +* Sat Jul 14 2018 Fedora Release Engineering - 2.7.15-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild + * Thu Jun 14 2018 Miro Hrončok - 2.7.15-5 - Move /usr/bin/python into a separate package https://fedoraproject.org/wiki/Changes/Move_usr_bin_python_into_separate_package From 8b06c12ed7dfcbe634bcd9c5642500ee1488aec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sun, 5 Aug 2018 20:28:48 +0000 Subject: [PATCH 03/10] Temporarily use compat-openssl10-devel to workaround #1609291 --- python2.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python2.spec b/python2.spec index 86003b1..11af0f1 100644 --- a/python2.spec +++ b/python2.spec @@ -136,12 +136,14 @@ BuildRequires: gmp-devel BuildRequires: libdb-devel BuildRequires: libffi-devel BuildRequires: ncurses-devel -BuildRequires: openssl-devel BuildRequires: pkgconfig BuildRequires: readline-devel BuildRequires: sqlite-devel BuildRequires: tcl-devel +# https://bugzilla.redhat.com/show_bug.cgi?id=1609291 +BuildRequires: compat-openssl10-devel + # For the nis module BuildRequires: libnsl2-devel BuildRequires: libtirpc-devel From d60e5a4fc915bd68ef372d5ad8b5851179d96d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 21 Aug 2018 15:47:27 +0200 Subject: [PATCH 04/10] Move patch 189 to 190 to make room for standardized 189 https://fedoraproject.org/wiki/SIGs/Python/PythonPatches --- ... => 00190-gdb-py-bt-dont-raise-exception-from-eval.patch | 0 python2.spec | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename 00189-gdb-py-bt-dont-raise-exception-from-eval.patch => 00190-gdb-py-bt-dont-raise-exception-from-eval.patch (100%) diff --git a/00189-gdb-py-bt-dont-raise-exception-from-eval.patch b/00190-gdb-py-bt-dont-raise-exception-from-eval.patch similarity index 100% rename from 00189-gdb-py-bt-dont-raise-exception-from-eval.patch rename to 00190-gdb-py-bt-dont-raise-exception-from-eval.patch diff --git a/python2.spec b/python2.spec index 11af0f1..78fe8cb 100644 --- a/python2.spec +++ b/python2.spec @@ -719,11 +719,11 @@ Patch185: 00185-urllib2-honors-noproxy-for-ftp.patch # symbol) Patch187: 00187-add-RPATH-to-pyexpat.patch -# 00189 # +# 00190 # # Fixes gdb py-bt command not to raise exception while processing # statements from eval # rhbz#1008154 (patch by Attila Fazekas) -Patch189: 00189-gdb-py-bt-dont-raise-exception-from-eval.patch +Patch190: 00190-gdb-py-bt-dont-raise-exception-from-eval.patch # 00191 # # Disabling NOOP test as it fails without internet connection @@ -1058,7 +1058,7 @@ mv Modules/cryptmodule.c Modules/_cryptmodule.c %patch181 -p1 %patch185 -p1 %patch187 -p1 -%patch189 -p1 +%patch190 -p1 %patch191 -p1 %patch193 -p1 %if %{with rewheel} From 4d74b7c9bf6d6aa13cf6d7099ced0ebe3768e73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 21 Aug 2018 15:58:19 +0200 Subject: [PATCH 05/10] Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch --- 00189-use-rpm-wheels.patch | 71 ++++++++++ 00198-add-rewheel-module.patch | 249 --------------------------------- python2.spec | 62 +++++--- 3 files changed, 111 insertions(+), 271 deletions(-) create mode 100644 00189-use-rpm-wheels.patch delete mode 100644 00198-add-rewheel-module.patch diff --git a/00189-use-rpm-wheels.patch b/00189-use-rpm-wheels.patch new file mode 100644 index 0000000..d7428f8 --- /dev/null +++ b/00189-use-rpm-wheels.patch @@ -0,0 +1,71 @@ +diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py +index 89ed1ef..8008222 100644 +--- a/Lib/ensurepip/__init__.py ++++ b/Lib/ensurepip/__init__.py +@@ -1,9 +1,10 @@ + #!/usr/bin/env python2 + from __future__ import print_function + ++import distutils.version ++import glob + import os + import os.path +-import pkgutil + import shutil + import sys + import tempfile +@@ -11,10 +12,20 @@ import tempfile + + __all__ = ["version", "bootstrap"] + ++_WHEEL_DIR = "/usr/share/python-wheels/" + +-_SETUPTOOLS_VERSION = "39.0.1" + +-_PIP_VERSION = "9.0.3" ++def _get_most_recent_wheel_version(pkg): ++ prefix = os.path.join(_WHEEL_DIR, "{}-".format(pkg)) ++ suffix = "-py2.py3-none-any.whl" ++ pattern = "{}*{}".format(prefix, suffix) ++ versions = (p[len(prefix):-len(suffix)] for p in glob.glob(pattern)) ++ return str(max(versions, key=distutils.version.LooseVersion)) ++ ++ ++_SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools") ++ ++_PIP_VERSION = _get_most_recent_wheel_version("pip") + + _PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), +@@ -28,8 +39,13 @@ def _run_pip(args, additional_paths=None): + sys.path = additional_paths + sys.path + + # Install the bundled software +- import pip +- return pip.main(args) ++ try: ++ # pip 10 ++ from pip._internal import main ++ except ImportError: ++ # pip 9 ++ from pip import main ++ return main(args) + + + def version(): +@@ -100,12 +116,9 @@ def _bootstrap(root=None, upgrade=False, user=False, + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) +- whl = pkgutil.get_data( +- "ensurepip", +- "_bundled/{}".format(wheel_name), +- ) +- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: +- fp.write(whl) ++ with open(os.path.join(_WHEEL_DIR, wheel_name), "rb") as sfp: ++ with open(os.path.join(tmpdir, wheel_name), "wb") as fp: ++ fp.write(sfp.read()) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + diff --git a/00198-add-rewheel-module.patch b/00198-add-rewheel-module.patch deleted file mode 100644 index 0273f1b..0000000 --- a/00198-add-rewheel-module.patch +++ /dev/null @@ -1,249 +0,0 @@ -diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py -index 89ed1ef..e2a4c25 100644 ---- a/Lib/ensurepip/__init__.py -+++ b/Lib/ensurepip/__init__.py -@@ -7,6 +7,7 @@ import pkgutil - import shutil - import sys - import tempfile -+from ensurepip import rewheel - - - __all__ = ["version", "bootstrap"] -@@ -29,6 +30,8 @@ def _run_pip(args, additional_paths=None): - - # Install the bundled software - import pip -+ if args[0] in ["install", "list", "wheel"]: -+ args.append('--pre') - return pip.main(args) - - -@@ -93,21 +96,40 @@ def _bootstrap(root=None, upgrade=False, user=False, - # omit pip and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "install" - -+ whls = [] -+ rewheel_dir = None -+ # try to see if we have system-wide versions of _PROJECTS -+ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS]) -+ # TODO: check if system-wide versions are the newest ones -+ # if --upgrade is used? -+ if all(dep_records): -+ # if we have all _PROJECTS installed system-wide, we'll recreate -+ # wheels from them and install those -+ rewheel_dir = tempfile.mkdtemp() -+ for dr in dep_records: -+ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir) -+ whls.append(os.path.join(rewheel_dir, new_whl)) -+ else: -+ # if we don't have all the _PROJECTS installed system-wide, -+ # let's just fall back to bundled wheels -+ for project, version in _PROJECTS: -+ whl = os.path.join( -+ os.path.dirname(__file__), -+ "_bundled", -+ "{}-{}-py2.py3-none-any.whl".format(project, version) -+ ) -+ whls.append(whl) -+ - tmpdir = tempfile.mkdtemp() - try: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] -- for project, version in _PROJECTS: -- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) -- whl = pkgutil.get_data( -- "ensurepip", -- "_bundled/{}".format(wheel_name), -- ) -- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: -- fp.write(whl) -- -- additional_paths.append(os.path.join(tmpdir, wheel_name)) -+ for whl in whls: -+ shutil.copy(whl, tmpdir) -+ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl))) -+ if rewheel_dir: -+ shutil.rmtree(rewheel_dir) - - # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] -diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py -new file mode 100644 -index 0000000..75c2094 ---- /dev/null -+++ b/Lib/ensurepip/rewheel/__init__.py -@@ -0,0 +1,158 @@ -+import argparse -+import codecs -+import csv -+import email.parser -+import os -+import io -+import re -+import site -+import subprocess -+import sys -+import zipfile -+ -+def run(): -+ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.') -+ parser.add_argument('record_path', -+ help='Path to RECORD file') -+ parser.add_argument('-o', '--output-dir', -+ help='Dir where to place the wheel, defaults to current working dir.', -+ dest='outdir', -+ default=os.path.curdir) -+ -+ ns = parser.parse_args() -+ retcode = 0 -+ try: -+ print(rewheel_from_record(**vars(ns))) -+ except BaseException as e: -+ print('Failed: {}'.format(e)) -+ retcode = 1 -+ sys.exit(1) -+ -+def find_system_records(projects): -+ """Return list of paths to RECORD files for system-installed projects. -+ -+ If a project is not installed, the resulting list contains None instead -+ of a path to its RECORD -+ """ -+ records = [] -+ # get system site-packages dirs -+ if hasattr(sys, 'real_prefix'): -+ #we are in python2 virtualenv and sys.real_prefix is the original sys.prefix -+ _orig_prefixes = site.PREFIXES -+ setattr(site, 'PREFIXES', [sys.real_prefix]*2) -+ sys_sitepack = site.getsitepackages() -+ setattr(site, 'PREFIXES', _orig_prefixes) -+ elif hasattr(sys, 'base_prefix'): # python3 venv doesn't inject real_prefix to sys -+ # we are on python3 and base(_exec)_prefix is unchanged in venv -+ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix]) -+ else: -+ # we are in python2 without virtualenv -+ sys_sitepack = site.getsitepackages() -+ -+ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)] -+ # try to find all projects in all system site-packages -+ for project in projects: -+ path = None -+ for sp in sys_sitepack: -+ dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep) -+ candidates = [os.path.join(sp, p) for p in os.listdir(sp)] -+ # filter out candidate dirs based on the above regexp -+ filtered = [c for c in candidates if re.match(dist_info_re, c)] -+ # if we have 0 or 2 or more dirs, something is wrong... -+ if len(filtered) == 1: -+ path = filtered[0] -+ if path is not None: -+ records.append(os.path.join(path, 'RECORD')) -+ else: -+ records.append(None) -+ return records -+ -+def rewheel_from_record(record_path, outdir): -+ """Recreates a whee of package with given record_path and returns path -+ to the newly created wheel.""" -+ site_dir = os.path.dirname(os.path.dirname(record_path)) -+ record_relpath = record_path[len(site_dir):].strip(os.path.sep) -+ to_write, to_omit = get_records_to_pack(site_dir, record_relpath) -+ new_wheel_name = get_wheel_name(record_path) -+ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl') -+ -+ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED) -+ # we need to write a new record with just the files that we will write, -+ # e.g. not binaries and *.pyc/*.pyo files -+ if sys.version_info[0] < 3: -+ new_record = io.BytesIO() -+ else: -+ new_record = io.StringIO() -+ writer = csv.writer(new_record) -+ -+ # handle files that we can write straight away -+ for f, sha_hash, size in to_write: -+ new_wheel.write(os.path.join(site_dir, f), arcname=f) -+ writer.writerow([f, sha_hash,size]) -+ -+ # rewrite the old wheel file with a new computed one -+ writer.writerow([record_relpath, '', '']) -+ new_wheel.writestr(record_relpath, new_record.getvalue()) -+ -+ new_wheel.close() -+ -+ return new_wheel.filename -+ -+def get_wheel_name(record_path): -+ """Return proper name of the wheel, without .whl.""" -+ -+ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL') -+ with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file: -+ wheel_info = email.parser.Parser().parsestr(wheel_info_file.read().encode('utf-8')) -+ -+ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA') -+ with codecs.open(metadata_path, encoding='utf-8') as metadata_file: -+ metadata = email.parser.Parser().parsestr(metadata_file.read().encode('utf-8')) -+ -+ # construct name parts according to wheel spec -+ distribution = metadata.get('Name') -+ version = metadata.get('Version') -+ build_tag = '' # nothing for now -+ lang_tag = [] -+ for t in wheel_info.get_all('Tag'): -+ lang_tag.append(t.split('-')[0]) -+ lang_tag = '.'.join(lang_tag) -+ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3] -+ # leave out build tag, if it is empty -+ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag]) -+ return '-'.join(list(to_join)) -+ -+def get_records_to_pack(site_dir, record_relpath): -+ """Accepts path of sitedir and path of RECORD file relative to it. -+ Returns two lists: -+ - list of files that can be written to new RECORD straight away -+ - list of files that shouldn't be written or need some processing -+ (pyc and pyo files, scripts) -+ """ -+ record_file_path = os.path.join(site_dir, record_relpath) -+ with codecs.open(record_file_path, encoding='utf-8') as record_file: -+ record_contents = record_file.read() -+ # temporary fix for https://github.com/pypa/pip/issues/1376 -+ # we need to ignore files under ".data" directory -+ data_dir = os.path.dirname(record_relpath).strip(os.path.sep) -+ data_dir = data_dir[:-len('dist-info')] + 'data' -+ -+ to_write = [] -+ to_omit = [] -+ for l in record_contents.splitlines(): -+ spl = l.split(',') -+ if len(spl) == 3: -+ # new record will omit (or write differently): -+ # - abs paths, paths with ".." (entry points), -+ # - pyc+pyo files -+ # - the old RECORD file -+ # TODO: is there any better way to recognize an entry point? -+ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \ -+ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \ -+ spl[0] == record_relpath or spl[0].startswith(data_dir): -+ to_omit.append(spl) -+ else: -+ to_write.append(spl) -+ else: -+ pass # bad RECORD or empty line -+ return to_write, to_omit -diff --git a/Makefile.pre.in b/Makefile.pre.in -index 877698c..2c43611 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -1065,7 +1065,7 @@ LIBSUBDIRS= lib-tk lib-tk/test lib-tk/test/test_tkinter \ - test/tracedmodules \ - encodings compiler hotshot \ - email email/mime email/test email/test/data \ -- ensurepip ensurepip/_bundled \ -+ ensurepip ensurepip/_bundled ensurepip/rewheel\ - json json/tests \ - sqlite3 sqlite3/test \ - logging bsddb bsddb/test csv importlib wsgiref \ diff --git a/python2.spec b/python2.spec index 78fe8cb..c93f93c 100644 --- a/python2.spec +++ b/python2.spec @@ -5,8 +5,9 @@ # Note that the bcond macros are named for the CLI option they create. # "%%bcond_without" means "ENABLE by default and create a --without option" -# Ability to reuse RPM-installed pip using rewheel -%bcond_without rewheel +# Whether to use RPM build wheels from the python-{pip,setuptools}-wheel package +# Uses upstream bundled prebuilt wheels otherwise +%bcond_without rpmwheels # Extra build for debugging the interpreter or C-API extensions # (the -debug subpackages) @@ -112,7 +113,7 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 6%{?dist} +Release: 7%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} @@ -182,15 +183,10 @@ BuildRequires: valgrind-devel BuildRequires: zlib-devel -%if %{with rewheel} -BuildRequires: python2-setuptools -Requires: python2-setuptools - -%if ! 0%{?_module_build} -BuildRequires: python2-pip -Requires: python2-pip -%endif # !module_build -%endif # rewheel +%if %{with rpmwheels} +BuildRequires: python-setuptools-wheel +BuildRequires: python-pip-wheel +%endif # Providing python27 as now multiple interpreters exist in Fedora # alongside the system one e.g. python26, python33 etc @@ -202,6 +198,12 @@ Provides: python27 = %{version}-%{release} # python2 back with fixed version, so we just use the path here: Recommends: %{_bindir}/python +# Previously, this was required for our rewheel patch to work. +# This is technically no longer needed, but we keep it recommended +# for the developer experience. +Recommends: python2-setuptools +Recommends: python2-pip + # ======================= # Source code and patches @@ -719,6 +721,11 @@ Patch185: 00185-urllib2-honors-noproxy-for-ftp.patch # symbol) Patch187: 00187-add-RPATH-to-pyexpat.patch +# 00189 # +# Instead of bundled wheels, use our RPM packaged wheels from +# /usr/share/python-wheels +Patch189: 00189-use-rpm-wheels.patch + # 00190 # # Fixes gdb py-bt command not to raise exception while processing # statements from eval @@ -736,9 +743,6 @@ Patch191: 00191-disable-NOOP.patch # Patch provided by John C. Peterson Patch193: 00193-enable-loading-sqlite-extensions.patch -# 00198 # -Patch198: 00198-add-rewheel-module.patch - # 00289 # # Disable automatic detection for the nis module # (we handle it it in Setup.dist, see Patch0) @@ -818,6 +822,14 @@ Requires: glibc%{?_isa} >= 2.24.90-26 Requires: gdbm%{?_isa} >= 1:1.13 %endif +%if %{with rpmwheels} +Requires: python-setuptools-wheel +Requires: python-pip-wheel +%else +Provides: bundled(python2-pip) = 9.0.3 +Provides: bundled(python2-setuptools) = 39.0.1 +%endif + Provides: python-libs = %{version}-%{release} Provides: python-libs%{?_isa} = %{version}-%{release} @@ -1058,12 +1070,15 @@ mv Modules/cryptmodule.c Modules/_cryptmodule.c %patch181 -p1 %patch185 -p1 %patch187 -p1 + +%if %{with rpmwheels} +%patch189 -p1 +rm Lib/ensurepip/_bundled/*.whl +%endif + %patch190 -p1 %patch191 -p1 %patch193 -p1 -%if %{with rewheel} -%patch198 -p1 -%endif %patch289 -p1 @@ -1748,11 +1763,11 @@ CheckPython \ %dir %{pylibdir}/ensurepip/ %{pylibdir}/ensurepip/*.py* +%if %{with rpmwheels} %exclude %{pylibdir}/ensurepip/_bundled - -%if %{with rewheel} -%dir %{pylibdir}/ensurepip/rewheel/ -%{pylibdir}/ensurepip/rewheel/*.py* +%else +%dir %{pylibdir}/ensurepip/_bundled +%{pylibdir}/ensurepip/_bundled/*.whl %endif @@ -1964,6 +1979,9 @@ CheckPython \ # ====================================================== %changelog +* Tue Aug 21 2018 Miro Hrončok - 2.7.15-7 +- Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch + * Sat Jul 14 2018 Fedora Release Engineering - 2.7.15-6 - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild From e8ef86e1cf0e8b37771e5053c9124ae55b0026c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 30 Aug 2018 15:46:34 +0200 Subject: [PATCH 06/10] Require python2-setuptools from python2-devel to prevent packaging errors (#1623922) --- python2.spec | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python2.spec b/python2.spec index c93f93c..4b1eff0 100644 --- a/python2.spec +++ b/python2.spec @@ -113,7 +113,7 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 7%{?dist} +Release: 8%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} @@ -851,6 +851,15 @@ Requires: pkgconfig Requires: python3-rpm-generators %endif +# This is not "API" (packages that need setuptools should still BuildRequire it) +# However some packages apparently can build both with and without setuptools +# producing egg-info as file or directory (depending on setuptools presence). +# Directory-to-file updates are problematic in RPM, so we ensure setuptools is +# installed when -devel is required. +# See https://bugzilla.redhat.com/show_bug.cgi?id=1623922 +# See https://fedoraproject.org/wiki/Packaging:Directory_Replacement +Requires: python2-setuptools + # https://bugzilla.redhat.com/show_bug.cgi?id=1217376 # https://bugzilla.redhat.com/show_bug.cgi?id=1496757 # https://bugzilla.redhat.com/show_bug.cgi?id=1218294 @@ -1979,6 +1988,9 @@ CheckPython \ # ====================================================== %changelog +* Thu Aug 30 2018 Miro Hrončok - 2.7.15-8 +- Require python2-setuptools from python2-devel to prevent packaging errors (#1623922) + * Tue Aug 21 2018 Miro Hrončok - 2.7.15-7 - Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch From ae09a791255cd6f6ee172efa8c77bb83746ebecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 20 Jul 2018 12:37:22 +0200 Subject: [PATCH 07/10] Enable basic venv smoke test in the CI --- tests/tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/tests.yml diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..fc8183f --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,18 @@ +--- +- hosts: localhost + roles: + - role: standard-test-basic + tags: + - classic + repositories: + - repo: "https://src.fedoraproject.org/tests/python.git" + dest: "python" + tests: + - smoke: + dir: python/smoke + run: METHOD=virtualenv VERSION=2.7 ./venv.sh + required_packages: + - gcc + - /usr/bin/virtualenv + - python3-tox + - python2-devel From 0052c9fa9d76e1706d96a17460ad26f331a4e0fe Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 6 Sep 2018 13:27:15 +0200 Subject: [PATCH 08/10] Deprecate python2 and all subpackages in Fedora 30+ See: - https://fedoraproject.org/wiki/Changes/Mass_Python_2_Package_Removal - https://fedoraproject.org/wiki/Packaging:Deprecating_Packages --- python2.spec | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/python2.spec b/python2.spec index 4b1eff0..4a06c22 100644 --- a/python2.spec +++ b/python2.spec @@ -105,6 +105,16 @@ # the rest of the build %global regenerate_autotooling_patch 0 +# Python 2 is deprecated in Fedora 30+, see: +# https://fedoraproject.org/wiki/Changes/Mass_Python_2_Package_Removal +# This means that new packages MUST NOT depend on python2, even transitively +# see: https://fedoraproject.org/wiki/Packaging:Deprecating_Packages +# Python 2 will not be supported after 2019. Use the python3 package instead +# if possible. +%if 0%{fedora} >= 30 +%global deprecated Provides: deprecated() +%endif + # ================== # Top-level metadata @@ -113,12 +123,14 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 8%{?dist} +Release: 9%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} Provides: python(abi) = %{pybasever} +%?deprecated + # ======================= # Build-time requirements @@ -790,6 +802,7 @@ implementation is within the "python2-libs" package. Summary: The "python" command that runs Python 2 BuildArch: noarch # https://fedoraproject.org/wiki/Changes/Move_usr_bin_python_into_separate_package +%?deprecated # In theory this could require any python2 version Requires: python2 == %{version}-%{release} @@ -804,6 +817,7 @@ This package contains /usr/bin/python - the "python" command that runs Python 2. %package libs Summary: Runtime libraries for Python 2 Group: Applications/System +%?deprecated # Needed for ctypes, to load libraries, worked around for Live CDs size # Requires: binutils @@ -839,6 +853,8 @@ This package contains files used to embed Python 2 into applications. %package devel Summary: Libraries and header files needed for Python 2 development Group: Development/Libraries +%?deprecated + Requires: %{python}%{?_isa} = %{version}-%{release} Requires: python-rpm-macros Requires: python2-rpm-macros @@ -880,6 +896,8 @@ with and native libraries for Python 2 %package tools Summary: A collection of development tools included with Python 2 Group: Development/Tools +%?deprecated + Requires: %{name} = %{version}-%{release} Requires: %{python}-tkinter = %{version}-%{release} @@ -894,6 +912,8 @@ color editor (pynche), and a python gettext program (pygettext.py). %package tkinter Summary: A graphical user interface for the Python 2 scripting language Group: Development/Languages +%?deprecated + Requires: %{name} = %{version}-%{release} Provides: tkinter = %{version}-%{release} @@ -914,6 +934,8 @@ user interface for Python 2 programming. %package test Summary: The test modules from the main python2 package Group: Development/Languages +%?deprecated + Requires: %{name} = %{version}-%{release} Provides: python-test = %{version}-%{release} @@ -932,6 +954,7 @@ code that uses more than just unittest and/or test.support. %package debug Summary: Debug version of the Python 2 runtime Group: Applications/System +%?deprecated # The debug build is an all-in-one package version of the regular build, and # shares the same .py/.pyc files and directories as the regular build. Hence @@ -1988,6 +2011,9 @@ CheckPython \ # ====================================================== %changelog +* Thu Sep 06 2018 Petr Viktorin - 2.7.15-9 +- Deprecate python2 and all subpackages in Fedora 30+ (#1625773) + * Thu Aug 30 2018 Miro Hrončok - 2.7.15-8 - Require python2-setuptools from python2-devel to prevent packaging errors (#1623922) From 0dfe62335c378c2cefe24d6c314ebbf14d2d5baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 20 Sep 2018 20:54:32 +0200 Subject: [PATCH 09/10] Security fix for CVE-2018-1000802 (#1631662) --- 00309-shutil-spawn-subprocess.patch | 61 +++++++++++++++++++++++++++++ python2.spec | 13 +++++- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 00309-shutil-spawn-subprocess.patch diff --git a/00309-shutil-spawn-subprocess.patch b/00309-shutil-spawn-subprocess.patch new file mode 100644 index 0000000..adc56c4 --- /dev/null +++ b/00309-shutil-spawn-subprocess.patch @@ -0,0 +1,61 @@ +From add531a1e55b0a739b0f42582f1c9747e5649ace Mon Sep 17 00:00:00 2001 +From: Benjamin Peterson +Date: Tue, 28 Aug 2018 22:12:56 -0700 +Subject: [PATCH] closes bpo-34540: Convert shutil._call_external_zip to use + subprocess rather than distutils.spawn. + +--- + Lib/shutil.py | 16 ++++++++++------ + .../2018-08-28-22-11-54.bpo-34540.gfQ0TM.rst | 3 +++ + 2 files changed, 13 insertions(+), 6 deletions(-) + create mode 100644 Misc/NEWS.d/next/Security/2018-08-28-22-11-54.bpo-34540.gfQ0TM.rst + +diff --git a/Lib/shutil.py b/Lib/shutil.py +index 3462f7c5e91c..0ab1a06f5260 100644 +--- a/Lib/shutil.py ++++ b/Lib/shutil.py +@@ -413,17 +413,21 @@ def _set_uid_gid(tarinfo): + + return archive_name + +-def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False): ++def _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger): + # XXX see if we want to keep an external call here + if verbose: + zipoptions = "-r" + else: + zipoptions = "-rq" +- from distutils.errors import DistutilsExecError +- from distutils.spawn import spawn ++ cmd = ["zip", zipoptions, zip_filename, base_dir] ++ if logger is not None: ++ logger.info(' '.join(cmd)) ++ if dry_run: ++ return ++ import subprocess + try: +- spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) +- except DistutilsExecError: ++ subprocess.check_call(cmd) ++ except subprocess.CalledProcessError: + # XXX really should distinguish between "couldn't find + # external 'zip' command" and "zip failed". + raise ExecError, \ +@@ -458,7 +462,7 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): + zipfile = None + + if zipfile is None: +- _call_external_zip(base_dir, zip_filename, verbose, dry_run) ++ _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger) + else: + if logger is not None: + logger.info("creating '%s' and adding '%s' to it", +diff --git a/Misc/NEWS.d/next/Security/2018-08-28-22-11-54.bpo-34540.gfQ0TM.rst b/Misc/NEWS.d/next/Security/2018-08-28-22-11-54.bpo-34540.gfQ0TM.rst +new file mode 100644 +index 000000000000..4f686962a87b +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2018-08-28-22-11-54.bpo-34540.gfQ0TM.rst +@@ -0,0 +1,3 @@ ++When ``shutil.make_archive`` falls back to the external ``zip`` problem, it ++uses :mod:`subprocess` to invoke it rather than :mod:`distutils.spawn`. This ++closes a possible shell injection vector. diff --git a/python2.spec b/python2.spec index 4a06c22..bdb9c92 100644 --- a/python2.spec +++ b/python2.spec @@ -123,7 +123,7 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 9%{?dist} +Release: 10%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} @@ -760,6 +760,13 @@ Patch193: 00193-enable-loading-sqlite-extensions.patch # (we handle it it in Setup.dist, see Patch0) Patch289: 00289-disable-nis-detection.patch +# 00309 # +# CVE-2018-1000802 +# shutil._call_external_zip to use subprocess instead of distutils.spawn +# rhbz#1631662 +# Fixed upstream https://bugs.python.org/issue34540 +Patch309: 00309-shutil-spawn-subprocess.patch + # (New patches go here ^^^) # # When adding new patches to "python2" and "python3" in Fedora, EL, etc., @@ -1112,6 +1119,7 @@ rm Lib/ensurepip/_bundled/*.whl %patch191 -p1 %patch193 -p1 %patch289 -p1 +%patch309 -p1 %if 0%{?_module_build} @@ -2011,6 +2019,9 @@ CheckPython \ # ====================================================== %changelog +* Thu Sep 20 2018 Miro Hrončok - 2.7.15-10 +- Security fix for CVE-2018-1000802 (#1631662) + * Thu Sep 06 2018 Petr Viktorin - 2.7.15-9 - Deprecate python2 and all subpackages in Fedora 30+ (#1625773) From ebba43dec3fd3b1984fb1836ba99475adead1a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Sep 2018 13:08:08 +0200 Subject: [PATCH 10/10] Security fix for CVE-2018-14647 (#1631822) --- ...0-use-xml-sethashsalt-in-elementtree.patch | 85 +++++++++++++++++++ python2.spec | 13 ++- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 00310-use-xml-sethashsalt-in-elementtree.patch diff --git a/00310-use-xml-sethashsalt-in-elementtree.patch b/00310-use-xml-sethashsalt-in-elementtree.patch new file mode 100644 index 0000000..27d8d1c --- /dev/null +++ b/00310-use-xml-sethashsalt-in-elementtree.patch @@ -0,0 +1,85 @@ +From 554c48934c599b3fb04c73d740bba1a745b89b41 Mon Sep 17 00:00:00 2001 +From: Christian Heimes +Date: Tue, 18 Sep 2018 14:38:58 +0200 +Subject: [PATCH] [2.7] bpo-34623: Use XML_SetHashSalt in _elementtree + (GH-9146) + +The C accelerated _elementtree module now initializes hash randomization +salt from _Py_HashSecret instead of libexpat's default CPRNG. + +Signed-off-by: Christian Heimes + +https://bugs.python.org/issue34623. +(cherry picked from commit cb5778f00ce48631c7140f33ba242496aaf7102b) + +Co-authored-by: Christian Heimes +--- + Include/pyexpat.h | 4 +++- + .../next/Security/2018-09-10-16-05-39.bpo-34623.Ua9jMv.rst | 2 ++ + Modules/_elementtree.c | 5 +++++ + Modules/pyexpat.c | 5 +++++ + 4 files changed, 15 insertions(+), 1 deletion(-) + create mode 100644 Misc/NEWS.d/next/Security/2018-09-10-16-05-39.bpo-34623.Ua9jMv.rst + +diff --git a/Include/pyexpat.h b/Include/pyexpat.h +index 5340ef5fa386..3fc5fa54da63 100644 +--- a/Include/pyexpat.h ++++ b/Include/pyexpat.h +@@ -3,7 +3,7 @@ + + /* note: you must import expat.h before importing this module! */ + +-#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.0" ++#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.1" + #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" + + struct PyExpat_CAPI +@@ -43,6 +43,8 @@ struct PyExpat_CAPI + XML_Parser parser, XML_UnknownEncodingHandler handler, + void *encodingHandlerData); + void (*SetUserData)(XML_Parser parser, void *userData); ++ /* might be none for expat < 2.1.0 */ ++ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* always add new stuff to the end! */ + }; + +diff --git a/Misc/NEWS.d/next/Security/2018-09-10-16-05-39.bpo-34623.Ua9jMv.rst b/Misc/NEWS.d/next/Security/2018-09-10-16-05-39.bpo-34623.Ua9jMv.rst +new file mode 100644 +index 000000000000..31ad92ef8582 +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2018-09-10-16-05-39.bpo-34623.Ua9jMv.rst +@@ -0,0 +1,2 @@ ++The C accelerated _elementtree module now initializes hash randomization ++salt from _Py_HashSecret instead of libexpat's default CSPRNG. +diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c +index f7f992dd3a95..b38e0ab329c7 100644 +--- a/Modules/_elementtree.c ++++ b/Modules/_elementtree.c +@@ -2574,6 +2574,11 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) + PyErr_NoMemory(); + return NULL; + } ++ /* expat < 2.1.0 has no XML_SetHashSalt() */ ++ if (EXPAT(SetHashSalt) != NULL) { ++ EXPAT(SetHashSalt)(self->parser, ++ (unsigned long)_Py_HashSecret.prefix); ++ } + + ALLOC(sizeof(XMLParserObject), "create expatparser"); + +diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c +index 2b4d31293c64..1f8c0d70a559 100644 +--- a/Modules/pyexpat.c ++++ b/Modules/pyexpat.c +@@ -2042,6 +2042,11 @@ MODULE_INITFUNC(void) + capi.SetProcessingInstructionHandler = XML_SetProcessingInstructionHandler; + capi.SetUnknownEncodingHandler = XML_SetUnknownEncodingHandler; + capi.SetUserData = XML_SetUserData; ++#if XML_COMBINED_VERSION >= 20100 ++ capi.SetHashSalt = XML_SetHashSalt; ++#else ++ capi.SetHashSalt = NULL; ++#endif + + /* export using capsule */ + capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); diff --git a/python2.spec b/python2.spec index bdb9c92..f11d4b7 100644 --- a/python2.spec +++ b/python2.spec @@ -123,7 +123,7 @@ Summary: An interpreted, interactive, object-oriented programming language Name: %{python} # Remember to also rebase python2-docs when changing this: Version: 2.7.15 -Release: 10%{?dist} +Release: 11%{?dist} License: Python Group: Development/Languages Requires: %{python}-libs%{?_isa} = %{version}-%{release} @@ -767,6 +767,13 @@ Patch289: 00289-disable-nis-detection.patch # Fixed upstream https://bugs.python.org/issue34540 Patch309: 00309-shutil-spawn-subprocess.patch +# 00310 # +# CVE-2018-14647 +# Use XML_SetHashSalt in _elementtree +# rhbz#1631822 +# Fixed upstream https://bugs.python.org/issue34623 +Patch310: 00310-use-xml-sethashsalt-in-elementtree.patch + # (New patches go here ^^^) # # When adding new patches to "python2" and "python3" in Fedora, EL, etc., @@ -1120,6 +1127,7 @@ rm Lib/ensurepip/_bundled/*.whl %patch193 -p1 %patch289 -p1 %patch309 -p1 +%patch310 -p1 %if 0%{?_module_build} @@ -2019,6 +2027,9 @@ CheckPython \ # ====================================================== %changelog +* Mon Sep 24 2018 Miro Hrončok - 2.7.15-11 +- Security fix for CVE-2018-14647 (#1631822) + * Thu Sep 20 2018 Miro Hrončok - 2.7.15-10 - Security fix for CVE-2018-1000802 (#1631662)