Update of bundled compileall2 module to 0.7.0
Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication This is explained in https://discuss.python.org/t/3014 and https://github.com/fedora-python/compileall2/issues/16 This option is not yet used anywhere. That allows us to backport this to all Fedoras but only use --hardlink-dupes on rawhide first.
This commit is contained in:
Miro Hrončok
parent
e9f07b72aa
commit
456f3ecffb
|
|
@ -19,6 +19,7 @@ import sys
|
|||
import importlib.util
|
||||
import py_compile
|
||||
import struct
|
||||
import filecmp
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
|
@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0):
|
|||
def compile_dir(dir, maxlevels=None, ddir=None, force=False,
|
||||
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
|
||||
invalidation_mode=None, stripdir=None,
|
||||
prependdir=None, limit_sl_dest=None):
|
||||
prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
|
||||
"""Byte-compile all modules in the given directory tree.
|
||||
|
||||
Arguments (only dir is required):
|
||||
|
|
@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
|
|||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
"""
|
||||
ProcessPoolExecutor = None
|
||||
if workers is not None:
|
||||
|
|
@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
|
|||
if not compile_file(file, ddir, force, rx, quiet,
|
||||
legacy, optimize, invalidation_mode,
|
||||
stripdir=stripdir, prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest):
|
||||
limit_sl_dest=limit_sl_dest,
|
||||
hardlink_dupes=hardlink_dupes):
|
||||
success = False
|
||||
return success
|
||||
|
||||
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None, stripdir=None, prependdir=None,
|
||||
limit_sl_dest=None):
|
||||
limit_sl_dest=None, hardlink_dupes=False):
|
||||
"""Byte-compile one file.
|
||||
|
||||
Arguments (only fullname is required):
|
||||
|
|
@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path.
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
"""
|
||||
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
|
|
@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
if isinstance(optimize, int):
|
||||
optimize = [optimize]
|
||||
|
||||
if hardlink_dupes:
|
||||
raise ValueError(("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level."))
|
||||
|
||||
if rx is not None:
|
||||
mo = rx.search(fullname)
|
||||
if mo:
|
||||
|
|
@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
if not quiet:
|
||||
print('Compiling {!r}...'.format(fullname))
|
||||
try:
|
||||
for opt_level, cfile in opt_cfiles.items():
|
||||
for index, opt_level in enumerate(sorted(optimize)):
|
||||
cfile = opt_cfiles[opt_level]
|
||||
if PY37:
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=opt_level,
|
||||
|
|
@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
else:
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=opt_level)
|
||||
|
||||
if index > 0 and hardlink_dupes:
|
||||
previous_cfile = opt_cfiles[optimize[index - 1]]
|
||||
if previous_cfile == cfile and optimize[0] not in (1, 2):
|
||||
# Python 3.4 has only one .pyo file for -O and -OO so
|
||||
# we hardlink it only if there is a .pyc file
|
||||
# with the same content
|
||||
previous_cfile = opt_cfiles[optimize[0]]
|
||||
if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False):
|
||||
os.unlink(cfile)
|
||||
os.link(previous_cfile, cfile)
|
||||
|
||||
except py_compile.PyCompileError as err:
|
||||
success = False
|
||||
if quiet >= 2:
|
||||
|
|
@ -379,11 +400,14 @@ def main():
|
|||
parser.add_argument('-j', '--workers', default=1,
|
||||
type=int, help='Run compileall concurrently')
|
||||
parser.add_argument('-o', action='append', type=int, dest='opt_levels',
|
||||
help=('Optimization levels to run compilation with.'
|
||||
'Default is -1 which uses optimization level of'
|
||||
help=('Optimization levels to run compilation with. '
|
||||
'Default is -1 which uses optimization level of '
|
||||
'Python interpreter itself (specified by -O).'))
|
||||
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
|
||||
help='Ignore symlinks pointing outsite of the DIR')
|
||||
parser.add_argument('--hardlink-dupes', action='store_true',
|
||||
dest='hardlink_dupes',
|
||||
help='Hardlink duplicated pyc files')
|
||||
|
||||
if PY37:
|
||||
invalidation_modes = [mode.name.lower().replace('_', '-')
|
||||
|
|
@ -413,6 +437,10 @@ def main():
|
|||
if args.opt_levels is None:
|
||||
args.opt_levels = [-1]
|
||||
|
||||
if len(args.opt_levels) == 1 and args.hardlink_dupes:
|
||||
parser.error(("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level."))
|
||||
|
||||
if args.ddir is not None and (
|
||||
args.stripdir is not None or args.prependdir is not None
|
||||
):
|
||||
|
|
@ -449,7 +477,8 @@ def main():
|
|||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest):
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
success = False
|
||||
else:
|
||||
if not compile_dir(dest, maxlevels, args.ddir,
|
||||
|
|
@ -459,7 +488,8 @@ def main():
|
|||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest):
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
success = False
|
||||
return success
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
Name: python-rpm-macros
|
||||
Version: 3
|
||||
Release: 53%{?dist}
|
||||
Release: 54%{?dist}
|
||||
Summary: The unversioned Python RPM macros
|
||||
|
||||
# macros: MIT, compileall2.py: PSFv2
|
||||
|
|
@ -10,7 +10,7 @@ Source1: macros.python-srpm
|
|||
Source2: macros.python2
|
||||
Source3: macros.python3
|
||||
Source4: macros.pybytecompile
|
||||
Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py
|
||||
Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py
|
||||
|
||||
BuildArch: noarch
|
||||
# For %%python3_pkgversion used in %%python_provide and compileall2.py
|
||||
|
|
@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \
|
|||
|
||||
|
||||
%changelog
|
||||
* Mon Feb 10 2020 Miro Hrončok <mhroncok@redhat.com> - 3-54
|
||||
- Update of bundled compileall2 module to 0.7.0
|
||||
Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication
|
||||
|
||||
* Thu Feb 06 2020 Miro Hrončok <mhroncok@redhat.com> - 3-53
|
||||
- Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue