fix the libpython.stp systemtap tapset (rhbz#697730)

detection compilation with wide unicode builds.

python-3.1.2-fix-parallel-make.patch

@@ -0,0 +1,37 @@
+diff -up Python-3.1.2/Makefile.pre.in.fix-parallel-make Python-3.1.2/Makefile.pre.in
+--- Python-3.1.2/Makefile.pre.in.fix-parallel-make	2010-10-13 13:13:52.888849568 -0400
++++ Python-3.1.2/Makefile.pre.in	2010-10-13 13:15:10.322184015 -0400
+@@ -215,6 +215,7 @@ IO_OBJS=	\
+ 
+ ##########################################################################
+ # Grammar
++GRAMMAR_STAMP=	$(srcdir)/grammar-stamp
+ GRAMMAR_H=	$(srcdir)/Include/graminit.h
+ GRAMMAR_C=	$(srcdir)/Python/graminit.c
+ GRAMMAR_INPUT=	$(srcdir)/Grammar/Grammar
+@@ -535,9 +536,24 @@ Modules/python.o: $(srcdir)/Modules/pyth
+ 
+ $(IO_OBJS): $(IO_H)
+ 
+-$(GRAMMAR_H) $(GRAMMAR_C): $(PGEN) $(GRAMMAR_INPUT)
++# GNU "make" interprets rules with two dependents as two copies of the rule.
++#
++# In a parallel build this can lead to pgen being run twice, once for each of
++# GRAMMAR_H and GRAMMAR_C, leading to race conditions in which the compiler
++# reads a partially-overwritten copy of one of these files, leading to syntax
++# errors (or linker errors if the fragment happens to be syntactically valid C)
++#
++# See http://www.gnu.org/software/hello/manual/automake/Multiple-Outputs.html
++# for more information
++#
++# Introduce ".grammar-stamp" as a contrived single output from PGEN to avoid
++# this:
++$(GRAMMAR_H) $(GRAMMAR_C): $(GRAMMAR_STAMP)
++
++$(GRAMMAR_STAMP): $(PGEN) $(GRAMMAR_INPUT)
+ 		-@$(INSTALL) -d Include
+ 		-$(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C)
++		touch $(GRAMMAR_STAMP)
+ 
+ $(PGEN):	$(PGENOBJS)
+ 		$(CC) $(OPT) $(LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN)

python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch

@@ -0,0 +1,184 @@
+Index: Python-3.1.2/Objects/unicodeobject.c
+===================================================================
+--- Python-3.1.2.orig/Objects/unicodeobject.c
++++ Python-3.1.2/Objects/unicodeobject.c
+@@ -159,6 +159,12 @@ static PyObject *unicode_encode_call_err
+        const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
+        Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos);
+ 
++static void raise_encode_exception(PyObject **exceptionObject,
++                                   const char *encoding,
++                                   const Py_UNICODE *unicode, Py_ssize_t size,
++                                   Py_ssize_t startpos, Py_ssize_t endpos,
++                                   const char *reason);
++
+ /* Same for linebreaks */
+ static unsigned char ascii_linebreak[] = {
+     0, 0, 0, 0, 0, 0, 0, 0,
+@@ -2453,67 +2459,98 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s
+     for (i = 0; i < size;) {
+         Py_UCS4 ch = s[i++];
+ 
+-        if (ch < 0x80)
++        if (ch < 0x80) {
+             /* Encode ASCII */
+             *p++ = (char) ch;
+ 
+-        else if (ch < 0x0800) {
++        } else if (ch < 0x0800) {
+             /* Encode Latin-1 */
+             *p++ = (char)(0xc0 | (ch >> 6));
+             *p++ = (char)(0x80 | (ch & 0x3f));
+-        }
+-        else {
+-            /* Encode UCS2 Unicode ordinals */
+-            if (ch < 0x10000) {
++        } else if (0xD800 <= ch && ch <= 0xDFFF) {
+ #ifndef Py_UNICODE_WIDE
+-                /* Special case: check for high surrogate */
+-                if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
+-                    Py_UCS4 ch2 = s[i];
+-                    /* Check for low surrogate and combine the two to
+-                       form a UCS4 value */
+-                    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
+-                        ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
+-                        i++;
+-                        goto encodeUCS4;
+-                    }
+-                    /* Fall through: handles isolated high surrogates */
+-                }
++            /* Special case: check for high and low surrogate */
++            if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) {
++                Py_UCS4 ch2 = s[i];
++                /* Combine the two surrogates to form a UCS4 value */
++                ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
++                i++;
++
++                /* Encode UCS4 Unicode ordinals */
++                *p++ = (char)(0xf0 | (ch >> 18));
++                *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
++                *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
++                *p++ = (char)(0x80 | (ch & 0x3f));
++
++            } else {
+ #endif
+-                if (ch >= 0xd800 && ch <= 0xdfff) {
+-                    Py_ssize_t newpos;
+-                    PyObject *rep;
+-                    char *prep;
+-                    int k;
+-                    rep = unicode_encode_call_errorhandler
+-                        (errors, &errorHandler, "utf-8", "surrogates not allowed", 
+-                         s, size, &exc, i-1, i, &newpos);
+-                    if (!rep)
+-                        goto error;
+-                    /* Implementation limitations: only support error handler that return
+-                       bytes, and only support up to four replacement bytes. */
+-                    if (!PyBytes_Check(rep)) {
+-                        PyErr_SetString(PyExc_TypeError, "error handler should have returned bytes");
+-                        Py_DECREF(rep);
++                Py_ssize_t newpos;
++                PyObject *rep;
++                Py_ssize_t repsize, k;
++                rep = unicode_encode_call_errorhandler
++                    (errors, &errorHandler, "utf-8", "surrogates not allowed",
++                     s, size, &exc, i-1, i, &newpos);
++                if (!rep)
++                    goto error;
++
++                if (PyBytes_Check(rep))
++                    repsize = PyBytes_GET_SIZE(rep);
++                else
++                    repsize = PyUnicode_GET_SIZE(rep);
++
++                if (repsize > 4) {
++                    Py_ssize_t offset;
++
++                    if (result == NULL)
++                        offset = p - stackbuf;
++                    else
++                        offset = p - PyBytes_AS_STRING(result);
++
++                    if (nallocated > PY_SSIZE_T_MAX - repsize + 4) {
++                        /* integer overflow */
++                        PyErr_NoMemory();
+                         goto error;
+                     }
+-                    if (PyBytes_Size(rep) > 4) {
+-                        PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes");
+-                        Py_DECREF(rep);
+-                        goto error;
++                    nallocated += repsize - 4;
++                    if (result != NULL) {
++                        if (_PyBytes_Resize(&result, nallocated) < 0)
++                            goto error;
++                    } else {
++                        result = PyBytes_FromStringAndSize(NULL, nallocated);
++                        if (result == NULL)
++                            goto error;
++                        Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
+                     }
+-                    prep = PyBytes_AsString(rep);
+-                    for(k = PyBytes_Size(rep); k > 0; k--)
++                    p = PyBytes_AS_STRING(result) + offset;
++                }
++
++                if (PyBytes_Check(rep)) {
++                    char *prep = PyBytes_AS_STRING(rep);
++                    for(k = repsize; k > 0; k--)
+                         *p++ = *prep++;
+-                    Py_DECREF(rep);
+-                    continue;
+-                    
++                } else /* rep is unicode */ {
++                    Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep);
++                    Py_UNICODE c;
++
++                    for(k=0; k<repsize; k++) {
++                        c = prep[k];
++                        if (0x80 <= c) {
++                            raise_encode_exception(&exc, "utf-8", s, size,
++                                                   i-1, i, "surrogates not allowed");
++                            goto error;
++                        }
++                        *p++ = (char)prep[k];
++                    }
+                 }
+-                *p++ = (char)(0xe0 | (ch >> 12));
+-                *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
+-                *p++ = (char)(0x80 | (ch & 0x3f));
+-                continue;
++                Py_DECREF(rep);
++#ifndef Py_UNICODE_WIDE
+             }
+-          encodeUCS4:
++#endif
++        } else if (ch < 0x10000) {
++            *p++ = (char)(0xe0 | (ch >> 12));
++            *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
++            *p++ = (char)(0x80 | (ch & 0x3f));
++        } else /* ch >= 0x10000 */ {
+             /* Encode UCS4 Unicode ordinals */
+             *p++ = (char)(0xf0 | (ch >> 18));
+             *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
+Index: Python-3.1.2/Lib/test/test_codecs.py
+===================================================================
+--- Python-3.1.2.orig/Lib/test/test_codecs.py
++++ Python-3.1.2/Lib/test/test_codecs.py
+@@ -571,6 +571,16 @@ class UTF8Test(ReadTest):
+     def test_lone_surrogates(self):
+         self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8")
+         self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8")
++        self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"),
++                         b'[\\udc80]')
++        self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"),
++                         b'[&#56448;]')
++        self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"),
++                         b'[\x80]')
++        self.assertEqual("[\uDC80]".encode("utf-8", "ignore"),
++                         b'[]')
++        self.assertEqual("[\uDC80]".encode("utf-8", "replace"),
++                         b'[?]')
+ 
+     def test_surrogatepass_handler(self):
+         self.assertEquals("abc\ud800def".encode("utf-8", "surrogatepass"),

python3.spec

@@ -40,7 +40,7 @@
 Summary: Version 3 of the Python programming language aka Python 3000
 Name: python3
 Version: %{pybasever}.2
-Release: 12%{?dist}
+Release: 16%{?dist}
 License: Python
 Group: Development/Languages
 Source: http://python.org/ftp/python/%{version}/Python-%{version}.tar.bz2
@@ -224,6 +224,17 @@ Patch109: python-3.1.2-CVE-2008-5983.patch
 # Sent upstream as http://bugs.python.org/issue9054
 Patch110: python-3.1.2-fix-expat-issue9054.patch
 
+# Fix encoding to utf8 when lone surrogates are present and error handler is
+# set to ignore, replace, or others that return a unicode str.
+# http://bugs.python.org/issue8092
+Patch111: python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch
+
+# Fix race condition in parallel make that could lead to graminit.c failing
+# to compile, or linker errors with "undefined reference to
+# `_PyParser_Grammar'":
+# http://bugs.python.org/issue10013 and r84068
+Patch112: python-3.1.2-fix-parallel-make.patch
+
 BuildRoot: %{_tmppath}/%{name}-%{version}-root
 BuildRequires: readline-devel, openssl-devel, gmp-devel
 BuildRequires: ncurses-devel, gdbm-devel, zlib-devel, expat-devel
@@ -384,6 +395,10 @@ rm -r Modules/zlib || exit 1
 
 %patch110 -p0 -b .fix-expat-issue9054
 
+%patch111 -p1 -b .surrogate-utf8
+
+%patch112 -p1 -b .fix-parallel-make
+
 # Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there
 # are many differences between 2.6 and the Python 3 library.
 #
@@ -678,12 +693,12 @@ mkdir -p %{buildroot}%{tapsetdir}
 
 sed \
    -e "s|LIBRARY_PATH|%{_libdir}/%{py_INSTSONAME_optimized}|" \
-   %{SOURCE6} \
+   %{_sourcedir}/libpython.stp \
    > %{buildroot}%{tapsetdir}/%{libpython_stp_optimized}
 
 sed \
    -e "s|LIBRARY_PATH|%{_libdir}/%{py_INSTSONAME_debug}|" \
-   %{SOURCE6} \
+   %{_sourcedir}/libpython.stp \
    > %{buildroot}%{tapsetdir}/%{libpython_stp_debug}
 
 %endif # with_systemtap
@@ -871,6 +886,7 @@ rm -fr %{buildroot}
 %{pylibdir}/sqlite3/*.py*
 %dir %{pylibdir}/test
 %{pylibdir}/test/__init__.py*
+%{pylibdir}/test/support.py*
 %{pylibdir}/urllib
 %{pylibdir}/wsgiref
 %{pylibdir}/xml
@@ -936,6 +952,8 @@ rm -fr %{buildroot}
 %{pylibdir}/json/tests
 %{pylibdir}/sqlite3/test
 %{pylibdir}/test
+%exclude %{pylibdir}/test/__init__.py*
+%exclude %{pylibdir}/test/support.py*
 %{dynload_dir}/_ctypes_test.so
 %{dynload_dir}/_testcapimodule.so
 %{pylibdir}/lib2to3/tests
@@ -1064,6 +1082,18 @@ rm -fr %{buildroot}
 
 
 %changelog
+* Tue Apr 19 2011 David Malcolm <dmalcolm@redhat.com> - 3.1.2-16
+- fix the libpython.stp systemtap tapset (rhbz#697730)
+
+* Wed Oct 13 2010 David Malcolm <dmalcolm@redhat.com> - 3.1.2-15
+- fix parallel make (patch 112)
+
+* Wed Sep  8 2010 David Malcolm <dmalcolm@redhat.com> - 3.1.2-14
+- add test.support to the core package (rhbz#596258)
+
+* Fri Aug 20 2010 Toshio Kuratomi <toshio@fedoraproject.org> - 3.1.2-13
+- Fix for lone surrogates, utf8 and certain encode error handlers.
+
 * Fri Jul  2 2010 David Malcolm <dmalcolm@redhat.com> - 3.1.2-12
 - rebuild