rpms
/
python3.11
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Drop the rewheel patch, use RPM built wheels instead

This commit is contained in:
Miro Hrončok 2018-08-15 15:36:29 +02:00
parent da8bebda6d
commit 0b241abda6
3 changed files with 80 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
00189-add-rewheel-module.patch
View File

@ -1,245 +0,0 @@
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
index 7ff938b..8c3d062 100644
--- a/Lib/ensurepip/__init__.py
+++ b/Lib/ensurepip/__init__.py
@@ -1,8 +1,10 @@
import os
import os.path
import pkgutil
+import shutil
import sys
import tempfile
+from ensurepip import rewheel
__all__ = ["version", "bootstrap"]
@@ -24,8 +26,15 @@ def _run_pip(args, additional_paths=None):
sys.path = additional_paths + sys.path
# Install the bundled software
- import pip._internal
- return pip._internal.main(args)
+ try:
+ # pip 10
+ from pip._internal import main
+ except ImportError:
+ # pip 9
+ from pip import main
+ if args[0] in ["install", "list", "wheel"]:
+ args.append('--pre')
+ return main(args)
def version():
@@ -88,20 +97,39 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
# omit pip and easy_install
os.environ["ENSUREPIP_OPTIONS"] = "install"
+ whls = []
+ rewheel_dir = None
+ # try to see if we have system-wide versions of _PROJECTS
+ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS])
+ # TODO: check if system-wide versions are the newest ones
+ # if --upgrade is used?
+ if all(dep_records):
+ # if we have all _PROJECTS installed system-wide, we'll recreate
+ # wheels from them and install those
+ rewheel_dir = tempfile.TemporaryDirectory()
+ for dr in dep_records:
+ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name)
+ whls.append(os.path.join(rewheel_dir.name, new_whl))
+ else:
+ # if we don't have all the _PROJECTS installed system-wide,
+ # let's just fall back to bundled wheels
+ for project, version in _PROJECTS:
+ whl = os.path.join(
+ os.path.dirname(__file__),
+ "_bundled",
+ "{}-{}-py2.py3-none-any.whl".format(project, version)
+ )
+ whls.append(whl)
+
with tempfile.TemporaryDirectory() as tmpdir:
# Put our bundled wheels into a temporary directory and construct the
# additional paths that need added to sys.path
additional_paths = []
- for project, version in _PROJECTS:
- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
- whl = pkgutil.get_data(
- "ensurepip",
- "_bundled/{}".format(wheel_name),
- )
- with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
- fp.write(whl)
-
- additional_paths.append(os.path.join(tmpdir, wheel_name))
+ for whl in whls:
+ shutil.copy(whl, tmpdir)
+ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl)))
+ if rewheel_dir:
+ rewheel_dir.cleanup()
# Construct the arguments to be passed to the pip command
args = ["install", "--no-index", "--find-links", tmpdir]
diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py
new file mode 100644
index 0000000..753c764
--- /dev/null
+++ b/Lib/ensurepip/rewheel/__init__.py
@@ -0,0 +1,143 @@
+import argparse
+import codecs
+import csv
+import email.parser
+import os
+import io
+import re
+import site
+import subprocess
+import sys
+import zipfile
+
+def run():
+ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.')
+ parser.add_argument('record_path',
+ help='Path to RECORD file')
+ parser.add_argument('-o', '--output-dir',
+ help='Dir where to place the wheel, defaults to current working dir.',
+ dest='outdir',
+ default=os.path.curdir)
+
+ ns = parser.parse_args()
+ retcode = 0
+ try:
+ print(rewheel_from_record(**vars(ns)))
+ except BaseException as e:
+ print('Failed: {}'.format(e))
+ retcode = 1
+ sys.exit(1)
+
+def find_system_records(projects):
+ """Return list of paths to RECORD files for system-installed projects.
+
+ If a project is not installed, the resulting list contains None instead
+ of a path to its RECORD
+ """
+ records = []
+ # get system site-packages dirs
+ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix])
+ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)]
+ # try to find all projects in all system site-packages
+ for project in projects:
+ path = None
+ for sp in sys_sitepack:
+ dist_info_re = os.path.join(sp, project) + r'-[^\{0}]+\.dist-info'.format(os.sep)
+ candidates = [os.path.join(sp, p) for p in os.listdir(sp)]
+ # filter out candidate dirs based on the above regexp
+ filtered = [c for c in candidates if re.match(dist_info_re, c)]
+ # if we have 0 or 2 or more dirs, something is wrong...
+ if len(filtered) == 1:
+ path = filtered[0]
+ if path is not None:
+ records.append(os.path.join(path, 'RECORD'))
+ else:
+ records.append(None)
+ return records
+
+def rewheel_from_record(record_path, outdir):
+ """Recreates a whee of package with given record_path and returns path
+ to the newly created wheel."""
+ site_dir = os.path.dirname(os.path.dirname(record_path))
+ record_relpath = record_path[len(site_dir):].strip(os.path.sep)
+ to_write, to_omit = get_records_to_pack(site_dir, record_relpath)
+ new_wheel_name = get_wheel_name(record_path)
+ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl')
+
+ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED)
+ # we need to write a new record with just the files that we will write,
+ # e.g. not binaries and *.pyc/*.pyo files
+ new_record = io.StringIO()
+ writer = csv.writer(new_record)
+
+ # handle files that we can write straight away
+ for f, sha_hash, size in to_write:
+ new_wheel.write(os.path.join(site_dir, f), arcname=f)
+ writer.writerow([f, sha_hash,size])
+
+ # rewrite the old wheel file with a new computed one
+ writer.writerow([record_relpath, '', ''])
+ new_wheel.writestr(record_relpath, new_record.getvalue())
+
+ new_wheel.close()
+
+ return new_wheel.filename
+
+def get_wheel_name(record_path):
+ """Return proper name of the wheel, without .whl."""
+
+ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL')
+ with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file:
+ wheel_info = email.parser.Parser().parsestr(wheel_info_file.read())
+
+ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA')
+ with codecs.open(metadata_path, encoding='utf-8') as metadata_file:
+ metadata = email.parser.Parser().parsestr(metadata_file.read())
+
+ # construct name parts according to wheel spec
+ distribution = metadata.get('Name')
+ version = metadata.get('Version')
+ build_tag = '' # nothing for now
+ lang_tag = []
+ for t in wheel_info.get_all('Tag'):
+ lang_tag.append(t.split('-')[0])
+ lang_tag = '.'.join(lang_tag)
+ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3]
+ # leave out build tag, if it is empty
+ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag])
+ return '-'.join(list(to_join))
+
+def get_records_to_pack(site_dir, record_relpath):
+ """Accepts path of sitedir and path of RECORD file relative to it.
+ Returns two lists:
+ - list of files that can be written to new RECORD straight away
+ - list of files that shouldn't be written or need some processing
+ (pyc and pyo files, scripts)
+ """
+ record_file_path = os.path.join(site_dir, record_relpath)
+ with codecs.open(record_file_path, encoding='utf-8') as record_file:
+ record_contents = record_file.read()
+ # temporary fix for https://github.com/pypa/pip/issues/1376
+ # we need to ignore files under ".data" directory
+ data_dir = os.path.dirname(record_relpath).strip(os.path.sep)
+ data_dir = data_dir[:-len('dist-info')] + 'data'
+
+ to_write = []
+ to_omit = []
+ for l in record_contents.splitlines():
+ spl = l.split(',')
+ if len(spl) == 3:
+ # new record will omit (or write differently):
+ # - abs paths, paths with ".." (entry points),
+ # - pyc+pyo files
+ # - the old RECORD file
+ # TODO: is there any better way to recognize an entry point?
+ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \
+ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \
+ spl[0] == record_relpath or spl[0].startswith(data_dir):
+ to_omit.append(spl)
+ else:
+ to_write.append(spl)
+ else:
+ pass # bad RECORD or empty line
+ return to_write, to_omit
diff --git a/Makefile.pre.in b/Makefile.pre.in
index d07b312..1c6720e 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1301,7 +1301,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \
test/test_asyncio \
collections concurrent concurrent/futures encodings \
email email/mime test/test_email test/test_email/data \
- ensurepip ensurepip/_bundled \
+ ensurepip ensurepip/_bundled ensurepip/rewheel \
html json test/test_json http dbm xmlrpc \
sqlite3 sqlite3/test \
logging csv wsgiref urllib \

51
00189-use-rpm-wheels.patch Normal file
View File

@ -0,0 +1,51 @@
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
index 4748ba4..fc02255 100644
--- a/Lib/ensurepip/__init__.py
+++ b/Lib/ensurepip/__init__.py
@@ -1,16 +1,27 @@
+import distutils.version
+import glob
import os
import os.path
-import pkgutil
import sys
import tempfile
__all__ = ["version", "bootstrap"]
+_WHEEL_DIR = "/usr/share/python-wheels/"
-_SETUPTOOLS_VERSION = "39.0.1"
-_PIP_VERSION = "10.0.1"
+def _get_most_recent_wheel_version(pkg):
+ prefix = os.path.join(_WHEEL_DIR, "{}-".format(pkg))
+ suffix = "-py2.py3-none-any.whl"
+ pattern = "{}*{}".format(prefix, suffix)
+ versions = (p[len(prefix):-len(suffix)] for p in glob.glob(pattern))
+ return str(max(versions, key=distutils.version.LooseVersion))
+
+
+_SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools")
+
+_PIP_VERSION = _get_most_recent_wheel_version("pip")
_PROJECTS = [
("setuptools", _SETUPTOOLS_VERSION),
@@ -94,12 +105,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
additional_paths = []
for project, version in _PROJECTS:
wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
- whl = pkgutil.get_data(
- "ensurepip",
- "_bundled/{}".format(wheel_name),
- )
- with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
- fp.write(whl)
+ with open(os.path.join(_WHEEL_DIR, wheel_name), "rb") as sfp:
+ with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+ fp.write(sfp.read())
additional_paths.append(os.path.join(tmpdir, wheel_name))

69
python3.spec
View File

@ -14,7 +14,7 @@ URL: https://www.python.org/
# WARNING When rebasing to a new Python version,
# remember to update the python3-docs package as well
Version: %{pybasever}.0
Release: 5%{?dist}
Release: 6%{?dist}
License: Python
@ -33,6 +33,9 @@ License: Python
# WARNING: This does not change the package name and summary above
%bcond_with flatpackage
# Whether to use RPM build wheels from the python-{pip,setuptools}-wheel package
# Uses upstream bundled prebuilt wheels otherwise
%bcond_without rpmwheels
# Expensive optimizations (mainly, profile-guided optimizations)
%ifarch %{ix86} x86_64
@ -46,13 +49,6 @@ License: Python
# Run the test suite in %%check
%bcond_without tests
# Ability to reuse RPM-installed pip using rewheel
%if %{with flatpackage}
%bcond_with rewheel
%else
%bcond_without rewheel
%endif
# Extra build for debugging the interpreter or C-API extensions
# (the -debug subpackages)
%if %{with flatpackage}
@ -190,9 +186,9 @@ BuildRequires: /usr/bin/dtrace
# workaround http://bugs.python.org/issue19804 (test_uuid requires ifconfig)
BuildRequires: /usr/sbin/ifconfig
%if %{with rewheel}
BuildRequires: python3-setuptools
BuildRequires: python3-pip
%if %{with rpmwheels}
BuildRequires: python-setuptools-wheel
BuildRequires: python-pip-wheel
%endif
@ -283,10 +279,9 @@ Patch170: 00170-gc-assertions.patch
Patch178: 00178-dont-duplicate-flags-in-sysconfig.patch
# 00189 #
# Add the rewheel module, allowing to recreate wheels from already installed
# ones
# https://github.com/bkabrda/rewheel
Patch189: 00189-add-rewheel-module.patch
# Instead of bundled wheels, use our RPM packaged wheels from
# /usr/share/python-wheels
Patch189: 00189-use-rpm-wheels.patch
# 00205 #
# LIBPL variable in makefile takes LIBPL from configure.ac
@ -363,11 +358,6 @@ Obsoletes: python%{pyshortver}
%global platpyver 3.6.2-20
Obsoletes: platform-python < %{platpyver}
%if %{with rewheel}
Requires: python3-setuptools
Requires: python3-pip
%endif
# This prevents ALL subpackages built from this spec to require
# /usr/bin/python3*. Granularity per subpackage is impossible.
# It's intended for the libs package not to drag in the interpreter, see
@ -414,6 +404,14 @@ Requires: glibc%{?_isa} >= 2.24.90-26
Requires: gdbm-libs%{?_isa} >= 1:1.13
%endif
%if %{with rpmwheels}
Requires: python-setuptools-wheel
Requires: python-pip-wheel
%else
Provides: bundled(python3-pip) = 10.0.1
Provides: bundled(python3-setuptools) = 39.0.1
%endif
# There are files in the standard library that have python shebang.
# We've filtered the automatic requirement out so libs are installable without
# the main package. This however makes it pulled in by default.
@ -443,12 +441,7 @@ Requires: %{name}-libs%{?_isa} = %{version}-%{release}
BuildRequires: python-rpm-macros
Requires: python-rpm-macros
Requires: python3-rpm-macros
%if %{with rewheel}
# without rewheel is used to bootstrap setuptools+pip
# python3-rpm-generators needs python3-setuptools, so we cannot have it yet
Requires: python3-rpm-generators
%endif
# https://bugzilla.redhat.com/show_bug.cgi?id=1217376
# https://bugzilla.redhat.com/show_bug.cgi?id=1496757
@ -579,9 +572,13 @@ Requires: redhat-rpm-config
%global __requires_exclude ^python\\(abi\\) = 3\\..$
%global __provides_exclude ^python\\(abi\\) = 3\\..$
# We keep those inside on purpose
%if %{with rpmwheels}
Requires: python-setuptools-wheel
Requires: python-pip-wheel
%else
Provides: bundled(python3-pip) = 10.0.1
Provides: bundled(python3-setuptools) = 39.0.1
%endif
# The description for the flat package
%description
@ -604,11 +601,6 @@ version once Python %{pybasever} is stable.
# Remove bundled libraries to ensure that we're using the system copy.
rm -r Modules/expat
%if %{with rewheel}
%global pip_version %(pip3 --version | cut -d' ' -f2)
sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/ensurepip/__init__.py
%endif
#
# Apply patches:
#
@ -625,8 +617,9 @@ sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/en
%patch170 -p1
%patch178 -p1
%if %{with rewheel}
%if %{with rpmwheels}
%patch189 -p1
rm Lib/ensurepip/_bundled/*.whl
%endif
%patch205 -p1
@ -1107,20 +1100,13 @@ CheckPython optimized
%{pylibdir}/ensurepip/*.py
%{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes}
%if %{without flatpackage}
%if %{with rpmwheels}
%exclude %{pylibdir}/ensurepip/_bundled
%else
%dir %{pylibdir}/ensurepip/_bundled
%{pylibdir}/ensurepip/_bundled/*.whl
%endif
%if %{with rewheel}
%dir %{pylibdir}/ensurepip/rewheel/
%dir %{pylibdir}/ensurepip/rewheel/__pycache__/
%{pylibdir}/ensurepip/rewheel/*.py
%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes}
%endif
%dir %{pylibdir}/test/
%dir %{pylibdir}/test/__pycache__/
%dir %{pylibdir}/test/support/
@ -1520,6 +1506,9 @@ CheckPython optimized
# ======================================================
%changelog
* Wed Aug 15 2018 Miro Hrončok <mhroncok@redhat.com> - 3.7.0-6
- Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch
* Fri Aug 10 2018 Igor Gnatenko <ignatenkobrain@fedoraproject.org> - 3.7.0-5
- Fix wrong requirement on gdbm