Bytecompile all *.py files properly during build (rhbz#1023607)

00186-dont-raise-from-py_compile.patch

@@ -0,0 +1,41 @@
+diff -r 7fa3e824a4ee Lib/py_compile.py
+--- a/Lib/py_compile.py	Tue Oct 29 22:25:06 2013 -0400
++++ b/Lib/py_compile.py	Wed Oct 30 11:08:31 2013 +0100
+@@ -108,15 +108,15 @@
+     byte-compile all installed files (or all files in selected
+     directories).
+     """
+-    with tokenize.open(file) as f:
+-        try:
+-            st = os.fstat(f.fileno())
+-        except AttributeError:
+-            st = os.stat(file)
+-        timestamp = int(st.st_mtime)
+-        size = st.st_size & 0xFFFFFFFF
+-        codestring = f.read()
+     try:
++        with tokenize.open(file) as f:
++            try:
++                st = os.fstat(f.fileno())
++            except AttributeError:
++                st = os.stat(file)
++            timestamp = int(st.st_mtime)
++            size = st.st_size & 0xFFFFFFFF
++            codestring = f.read()
+         codeobject = builtins.compile(codestring, dfile or file, 'exec',
+                                       optimize=optimize)
+     except Exception as err:
+diff -r 7fa3e824a4ee Lib/test/test_py_compile.py
+--- a/Lib/test/test_py_compile.py	Tue Oct 29 22:25:06 2013 -0400
++++ b/Lib/test/test_py_compile.py	Wed Oct 30 11:08:31 2013 +0100
+@@ -54,6 +54,10 @@
+         self.assertTrue(os.path.exists(self.pyc_path))
+         self.assertFalse(os.path.exists(self.cache_path))
+ 
++    def test_bad_coding(self):
++        bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py')
++        self.assertIsNone(py_compile.compile(bad_coding, doraise=False))
++
+ def test_main():
+     support.run_unittest(PyCompileTests)
+ 

check-pyc-and-pyo-timestamps.py

@@ -0,0 +1,48 @@
+"""Checks if all *.pyc and *.pyo files have later mtime than their *.py files."""
+
+import imp
+import os
+import sys
+
+# list of test and other files that we expect not to have bytecode
+not_compiled = [
+    'test/bad_coding.py',
+    'test/bad_coding2.py',
+    'test/badsyntax_3131.py',
+    'test/badsyntax_future3.py',
+    'test/badsyntax_future4.py',
+    'test/badsyntax_future5.py',
+    'test/badsyntax_future6.py',
+    'test/badsyntax_future7.py',
+    'test/badsyntax_future8.py',
+    'test/badsyntax_future9.py',
+    'test/badsyntax_pep3120.py',
+    'lib2to3/tests/data/bom.py',
+    'lib2to3/tests/data/crlf.py',
+    'lib2to3/tests/data/different_encoding.py',
+    'lib2to3/tests/data/py2_test_grammar.py',
+    '.debug-gdb.py',
+]
+failed = 0
+
+def bytecode_expected(source):
+    for f in not_compiled:
+        if source.endswith(f):
+            return False
+    return True
+
+compiled = filter(lambda f: bytecode_expected(f), sys.argv[1:])
+for f in compiled:
+    # check both pyo and pyc
+    to_check = map(lambda b: imp.cache_from_source(f, b), (True, False))
+    f_mtime = os.path.getmtime(f)
+    for c in to_check:
+        c_mtime = os.path.getmtime(c)
+        if c_mtime < f_mtime:
+            sys.stderr.write('Failed bytecompilation timestamps check: ')
+            sys.stderr.write('Bytecode file {} is older than source file {}.\n'.format(c, f))
+            failed += 1
+
+if failed:
+    sys.stderr.write('\n{} files failed bytecompilation timestamps check.\n'.format(failed))
+    sys.exit(1)

python3.spec

@@ -126,7 +126,7 @@
 Summary: Version 3 of the Python programming language aka Python 3000
 Name: python3
 Version: %{pybasever}.2
-Release: 6%{?dist}
+Release: 7%{?dist}
 License: Python
 Group: Development/Languages
 
@@ -217,6 +217,10 @@ Source6: systemtap-example.stp
 # Written by dmalcolm; not yet sent upstream
 Source7: pyfuntop.stp
 
+# A simple script to check timestamps of bytecode files
+# Run in check section with Python that is currently being built
+# Written by bkabrda
+Source8: check-pyc-and-pyo-timestamps.py
 
 # Fixup distutils/unixccompiler.py to remove standard library path from rpath:
 # Was Patch0 in ivazquez' python3000 specfile:
@@ -611,6 +615,14 @@ Patch184: 00184-ctypes-should-build-with-libffi-multilib-wrapper.patch
 # rhbz#996399
 Patch185: 00185-CVE-2013-4238-hostname-check-bypass-in-SSL-module.patch
 
+# 00186 #
+# Fix for https://bugzilla.redhat.com/show_bug.cgi?id=1023607
+# Fixes the problem of some *.py files not being bytecompiled properly
+# during build. This was result of py_compile.compile raising exception
+# when trying to convert test file with bad encoding, and thus not
+# continuing bytecompilation for other files.
+Patch186: 00186-dont-raise-from-py_compile.patch
+
 
 # (New patches go here ^^^)
 #
@@ -871,6 +883,7 @@ done
 %patch183 -p1
 %patch184 -p1
 %patch185 -p1
+%patch186 -p1
 
 # Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there
 # are many differences between 2.6 and the Python 3 library.
@@ -1200,12 +1213,12 @@ iconv -f iso8859-1 -t utf-8 %{buildroot}/%{pylibdir}/Demo/rpc/README > README.co
 # compile *.pyo
 find %{buildroot} -type f -a -name "*.py" -print0 | \
     LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \
-    PYTHONPATH="%{buildroot}%{_libdir}python%{pybasever} %{buildroot}/%{_libdir}python%{pybasever}/site-packages" \
+    PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \
     xargs -0 %{buildroot}%{_bindir}/python%{pybasever} -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("%{buildroot}")[2]) for f in sys.argv[1:]]' || :
 # compile *.pyc
 find %{buildroot} -type f -a -name "*.py" -print0 | \
     LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \
-    PYTHONPATH="%{buildroot}%{_libdir}python%{pybasever} %{buildroot}/%{_libdir}python%{pybasever}/site-packages" \
+    PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \
     xargs -0 %{buildroot}%{_bindir}/python%{pybasever} -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("%{buildroot}")[2], optimize=0) for f in sys.argv[1:]]' || :
 
 # Fixup permissions for shared libraries from non-standard 555 to standard 755:
@@ -1287,6 +1300,14 @@ sed \
 # ======================================================
 
 %check
+
+# first of all, check timestamps of bytecode files
+find %{buildroot} -type f -a -name "*.py" -print0 | \
+    LD_LIBRARY_PATH="%{buildroot}%{dynload_dir}/:%{buildroot}%{_libdir}" \
+    PYTHONPATH="%{buildroot}%{_libdir}/python%{pybasever} %{buildroot}%{_libdir}/python%{pybasever}/site-packages" \
+    xargs -0 %{buildroot}%{_bindir}/python%{pybasever} %{SOURCE8}
+
+
 topdir=$(pwd)
 CheckPython() {
   ConfName=$1	      
@@ -1711,6 +1732,9 @@ rm -fr %{buildroot}
 # ======================================================
 
 %changelog
+* Wed Oct 30 2013 Bohuslav Kabrda <bkabrda@redhat.com> - 3.3.2-7
+- Bytecompile all *.py files properly during build (rhbz#1023607)
+
 * Fri Aug 23 2013 Matej Stuchlik <mstuchli@redhat.com> - 3.3.2-6
 - Added fix for CVE-2013-4238 (rhbz#996399)