diff --git a/compileall2.py b/compileall2.py index 593dda9..8976fa0 100644 --- a/compileall2.py +++ b/compileall2.py @@ -19,6 +19,7 @@ import sys import importlib.util import py_compile import struct +import filecmp from functools import partial from pathlib import Path @@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0): def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, - prependdir=None, limit_sl_dest=None): + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path + hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None if workers is not None: @@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest): + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, stripdir=None, prependdir=None, - limit_sl_dest=None): + limit_sl_dest=None, hardlink_dupes=False): """Byte-compile one file. Arguments (only fullname is required): @@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. + hardlink_dupes: hardlink duplicated pyc files """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if isinstance(optimize, int): optimize = [optimize] + if hardlink_dupes: + raise ValueError(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if rx is not None: mo = rx.search(fullname) if mo: @@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not quiet: print('Compiling {!r}...'.format(fullname)) try: - for opt_level, cfile in opt_cfiles.items(): + for index, opt_level in enumerate(sorted(optimize)): + cfile = opt_cfiles[opt_level] if PY37: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level, @@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level) + + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if previous_cfile == cfile and optimize[0] not in (1, 2): + # Python 3.4 has only one .pyo file for -O and -OO so + # we hardlink it only if there is a .pyc file + # with the same content + previous_cfile = opt_cfiles[optimize[0]] + if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) + except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -379,11 +400,14 @@ def main(): parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', - help=('Optimization levels to run compilation with.' - 'Default is -1 which uses optimization level of' + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses optimization level of ' 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') if PY37: invalidation_modes = [mode.name.lower().replace('_', '-') @@ -413,6 +437,10 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if args.ddir is not None and ( args.stripdir is not None or args.prependdir is not None ): @@ -449,7 +477,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -459,7 +488,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False return success else: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e42fb8f..9af171c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 52%{?dist} +Release: 53%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Feb 10 2020 Miro Hrončok - 3-53 +- Update of bundled compileall2 module to 0.7.0 + Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication + * Fri Feb 07 2020 Miro Hrončok - 3-52 - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a