From 3dbbc14e760256f1b8bf22b0a64e0234c4708ac5 Mon Sep 17 00:00:00 2001 From: Bohuslav Kabrda Date: Wed, 30 Oct 2013 10:50:01 +0100 Subject: [PATCH] Bytecompile all *.py files properly during build (rhbz#1023607) --- 00186-dont-raise-from-py_compile.patch | 41 ++++++++++++++++++++++ check-pyc-and-pyo-timestamps.py | 48 ++++++++++++++++++++++++++ python3.spec | 30 ++++++++++++++-- 3 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 00186-dont-raise-from-py_compile.patch create mode 100644 check-pyc-and-pyo-timestamps.py diff --git a/00186-dont-raise-from-py_compile.patch b/00186-dont-raise-from-py_compile.patch new file mode 100644 index 0000000..1e7fb06 --- /dev/null +++ b/00186-dont-raise-from-py_compile.patch @@ -0,0 +1,41 @@ +diff -r 7fa3e824a4ee Lib/py_compile.py +--- a/Lib/py_compile.py Tue Oct 29 22:25:06 2013 -0400 ++++ b/Lib/py_compile.py Wed Oct 30 11:08:31 2013 +0100 +@@ -108,15 +108,15 @@ + byte-compile all installed files (or all files in selected + directories). + """ +- with tokenize.open(file) as f: +- try: +- st = os.fstat(f.fileno()) +- except AttributeError: +- st = os.stat(file) +- timestamp = int(st.st_mtime) +- size = st.st_size & 0xFFFFFFFF +- codestring = f.read() + try: ++ with tokenize.open(file) as f: ++ try: ++ st = os.fstat(f.fileno()) ++ except AttributeError: ++ st = os.stat(file) ++ timestamp = int(st.st_mtime) ++ size = st.st_size & 0xFFFFFFFF ++ codestring = f.read() + codeobject = builtins.compile(codestring, dfile or file, 'exec', + optimize=optimize) + except Exception as err: +diff -r 7fa3e824a4ee Lib/test/test_py_compile.py +--- a/Lib/test/test_py_compile.py Tue Oct 29 22:25:06 2013 -0400 ++++ b/Lib/test/test_py_compile.py Wed Oct 30 11:08:31 2013 +0100 +@@ -54,6 +54,10 @@ + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + ++ def test_bad_coding(self): ++ bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') ++ self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) ++ + def test_main(): + support.run_unittest(PyCompileTests) + diff --git a/check-pyc-and-pyo-timestamps.py b/check-pyc-and-pyo-timestamps.py new file mode 100644 index 0000000..bcd5baf --- /dev/null +++ b/check-pyc-and-pyo-timestamps.py @@ -0,0 +1,48 @@ +"""Checks if all *.pyc and *.pyo files have later mtime than their *.py files.""" + +import imp +import os +import sys + +# list of test and other files that we expect not to have bytecode +not_compiled = [ + 'test/bad_coding.py', + 'test/bad_coding2.py', + 'test/badsyntax_3131.py', + 'test/badsyntax_future3.py', + 'test/badsyntax_future4.py', + 'test/badsyntax_future5.py', + 'test/badsyntax_future6.py', + 'test/badsyntax_future7.py', + 'test/badsyntax_future8.py', + 'test/badsyntax_future9.py', + 'test/badsyntax_pep3120.py', + 'lib2to3/tests/data/bom.py', + 'lib2to3/tests/data/crlf.py', + 'lib2to3/tests/data/different_encoding.py', + 'lib2to3/tests/data/py2_test_grammar.py', + '.debug-gdb.py', +] +failed = 0 + +def bytecode_expected(source): + for f in not_compiled: + if source.endswith(f): + return False + return True + +compiled = filter(lambda f: bytecode_expected(f), sys.argv[1:]) +for f in compiled: + # check both pyo and pyc + to_check = map(lambda b: imp.cache_from_source(f, b), (True, False)) + f_mtime = os.path.getmtime(f) + for c in to_check: + c_mtime = os.path.getmtime(c) + if c_mtime < f_mtime: + sys.stderr.write('Failed bytecompilation timestamps check: ') + sys.stderr.write('Bytecode file {} is older than source file {}.\n'.format(c, f)) + failed += 1 + +if failed: + sys.stderr.write('\n{} files failed bytecompilation timestamps check.\n'.format(failed)) + sys.exit(1) diff --git a/python3.spec b/python3.spec index 9ad3e67..f445372 100644 --- a/python3.spec +++ b/python3.spec @@ -126,7 +126,7 @@ Summary: Version 3 of the Python programming language aka Python 3000 Name: python3 Version: %{pybasever}.2 -Release: 6%{?dist} +Release: 7%{?dist} License: Python Group: Development/Languages @@ -217,6 +217,10 @@ Source6: systemtap-example.stp # Written by dmalcolm; not yet sent upstream Source7: pyfuntop.stp +# A simple script to check timestamps of bytecode files +# Run in check section with Python that is currently being built +# Written by bkabrda +Source8: check-pyc-and-pyo-timestamps.py # Fixup distutils/unixccompiler.py to remove standard library path from rpath: # Was Patch0 in ivazquez' python3000 specfile: @@ -611,6 +615,14 @@ Patch184: 00184-ctypes-should-build-with-libffi-multilib-wrapper.patch # rhbz#996399 Patch185: 00185-CVE-2013-4238-hostname-check-bypass-in-SSL-module.patch +# 00186 # +# Fix for https://bugzilla.redhat.com/show_bug.cgi?id=1023607 +# Fixes the problem of some *.py files not being bytecompiled properly +# during build. This was result of py_compile.compile raising exception +# when trying to convert test file with bad encoding, and thus not +# continuing bytecompilation for other files. +Patch186: 00186-dont-raise-from-py_compile.patch + # (New patches go here ^^^) # @@ -871,6 +883,7 @@ done %patch183 -p1 %patch184 -p1 %patch185 -p1 +%patch186 -p1 # Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there # are many differences between 2.6 and the Python 3 library. @@ -1200,12 +1213,12 @@ iconv -f iso8859-1 -t utf-8 %{buildroot}/%{pylibdir}/Demo/rpc/README > README.co # compile *.pyo find %{buildroot} -type f -a -name "*.py" -print0 | \ LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \ - PYTHONPATH="%{buildroot}%{_libdir}python%{pybasever} %{buildroot}/%{_libdir}python%{pybasever}/site-packages" \ + PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \ xargs -0 %{buildroot}%{_bindir}/python%{pybasever} -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("%{buildroot}")[2]) for f in sys.argv[1:]]' || : # compile *.pyc find %{buildroot} -type f -a -name "*.py" -print0 | \ LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \ - PYTHONPATH="%{buildroot}%{_libdir}python%{pybasever} %{buildroot}/%{_libdir}python%{pybasever}/site-packages" \ + PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \ xargs -0 %{buildroot}%{_bindir}/python%{pybasever} -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("%{buildroot}")[2], optimize=0) for f in sys.argv[1:]]' || : # Fixup permissions for shared libraries from non-standard 555 to standard 755: @@ -1287,6 +1300,14 @@ sed \ # ====================================================== %check + +# first of all, check timestamps of bytecode files +find %{buildroot} -type f -a -name "*.py" -print0 | \ + LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \ + PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \ + xargs -0 %{buildroot}%{_bindir}/python%{pybasever} %{SOURCE8} + + topdir=$(pwd) CheckPython() { ConfName=$1 @@ -1711,6 +1732,9 @@ rm -fr %{buildroot} # ====================================================== %changelog +* Wed Oct 30 2013 Bohuslav Kabrda - 3.3.2-7 +- Bytecompile all *.py files properly during build (rhbz#1023607) + * Fri Aug 23 2013 Matej Stuchlik - 3.3.2-6 - Added fix for CVE-2013-4238 (rhbz#996399)