From 456f3ecffb062d1594285b6f9aec0ef2f9f22391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 10 Feb 2020 23:47:52 +0100 Subject: [PATCH] Update of bundled compileall2 module to 0.7.0 Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication This is explained in https://discuss.python.org/t/3014 and https://github.com/fedora-python/compileall2/issues/16 This option is not yet used anywhere. That allows us to backport this to all Fedoras but only use --hardlink-dupes on rawhide first. --- compileall2.py | 46 ++++++++++++++++++++++++++++++++++-------- python-rpm-macros.spec | 8 ++++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/compileall2.py b/compileall2.py index 593dda9..8976fa0 100644 --- a/compileall2.py +++ b/compileall2.py @@ -19,6 +19,7 @@ import sys import importlib.util import py_compile import struct +import filecmp from functools import partial from pathlib import Path @@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0): def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, - prependdir=None, limit_sl_dest=None): + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path + hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None if workers is not None: @@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest): + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, stripdir=None, prependdir=None, - limit_sl_dest=None): + limit_sl_dest=None, hardlink_dupes=False): """Byte-compile one file. Arguments (only fullname is required): @@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. + hardlink_dupes: hardlink duplicated pyc files """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if isinstance(optimize, int): optimize = [optimize] + if hardlink_dupes: + raise ValueError(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if rx is not None: mo = rx.search(fullname) if mo: @@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not quiet: print('Compiling {!r}...'.format(fullname)) try: - for opt_level, cfile in opt_cfiles.items(): + for index, opt_level in enumerate(sorted(optimize)): + cfile = opt_cfiles[opt_level] if PY37: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level, @@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level) + + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if previous_cfile == cfile and optimize[0] not in (1, 2): + # Python 3.4 has only one .pyo file for -O and -OO so + # we hardlink it only if there is a .pyc file + # with the same content + previous_cfile = opt_cfiles[optimize[0]] + if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) + except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -379,11 +400,14 @@ def main(): parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', - help=('Optimization levels to run compilation with.' - 'Default is -1 which uses optimization level of' + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses optimization level of ' 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') if PY37: invalidation_modes = [mode.name.lower().replace('_', '-') @@ -413,6 +437,10 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if args.ddir is not None and ( args.stripdir is not None or args.prependdir is not None ): @@ -449,7 +477,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -459,7 +488,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False return success else: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c6fe8c6..0734fe4 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 53%{?dist} +Release: 54%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Feb 10 2020 Miro Hrončok - 3-54 +- Update of bundled compileall2 module to 0.7.0 + Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication + * Thu Feb 06 2020 Miro Hrončok - 3-53 - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a