Update to 3.11.0b4
Patches 383 and 384 were merged upstream. Patch 385 was partially merged upstream, the PR with test is still open.
This commit is contained in:
parent
c7432171d0
commit
b2d80045f9
@ -33,7 +33,7 @@ Co-authored-by: Lumír Balhar <frenzy.madness@gmail.com>
|
|||||||
3 files changed, 30 insertions(+), 2 deletions(-)
|
3 files changed, 30 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
diff --git a/Lib/site.py b/Lib/site.py
|
diff --git a/Lib/site.py b/Lib/site.py
|
||||||
index b11cd48e69..63ddd5b21b 100644
|
index 69670d9d7f..104cb93899 100644
|
||||||
--- a/Lib/site.py
|
--- a/Lib/site.py
|
||||||
+++ b/Lib/site.py
|
+++ b/Lib/site.py
|
||||||
@@ -377,8 +377,15 @@ def getsitepackages(prefixes=None):
|
@@ -377,8 +377,15 @@ def getsitepackages(prefixes=None):
|
||||||
@ -54,7 +54,7 @@ index b11cd48e69..63ddd5b21b 100644
|
|||||||
if os.path.isdir(sitedir):
|
if os.path.isdir(sitedir):
|
||||||
addsitedir(sitedir, known_paths)
|
addsitedir(sitedir, known_paths)
|
||||||
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
|
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
|
||||||
index e21b7303fe..37b04452bc 100644
|
index ebe3711827..dca8aa89b6 100644
|
||||||
--- a/Lib/sysconfig.py
|
--- a/Lib/sysconfig.py
|
||||||
+++ b/Lib/sysconfig.py
|
+++ b/Lib/sysconfig.py
|
||||||
@@ -103,6 +103,25 @@
|
@@ -103,6 +103,25 @@
|
||||||
@ -84,10 +84,10 @@ index e21b7303fe..37b04452bc 100644
|
|||||||
# NOTE: site.py has copy of this function.
|
# NOTE: site.py has copy of this function.
|
||||||
# Sync it when modify this function.
|
# Sync it when modify this function.
|
||||||
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
|
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
|
||||||
index f2b93706b2..cc58f47cdb 100644
|
index 578ac1db50..e78ff15ad3 100644
|
||||||
--- a/Lib/test/test_sysconfig.py
|
--- a/Lib/test/test_sysconfig.py
|
||||||
+++ b/Lib/test/test_sysconfig.py
|
+++ b/Lib/test/test_sysconfig.py
|
||||||
@@ -333,7 +333,7 @@ def test_get_config_h_filename(self):
|
@@ -336,7 +336,7 @@ def test_get_config_h_filename(self):
|
||||||
self.assertTrue(os.path.isfile(config_h), config_h)
|
self.assertTrue(os.path.isfile(config_h), config_h)
|
||||||
|
|
||||||
def test_get_scheme_names(self):
|
def test_get_scheme_names(self):
|
||||||
@ -96,7 +96,7 @@ index f2b93706b2..cc58f47cdb 100644
|
|||||||
if HAS_USER_BASE:
|
if HAS_USER_BASE:
|
||||||
wanted.extend(['nt_user', 'osx_framework_user', 'posix_user'])
|
wanted.extend(['nt_user', 'osx_framework_user', 'posix_user'])
|
||||||
self.assertEqual(get_scheme_names(), tuple(sorted(wanted)))
|
self.assertEqual(get_scheme_names(), tuple(sorted(wanted)))
|
||||||
@@ -345,6 +345,8 @@ def test_symlink(self): # Issue 7880
|
@@ -348,6 +348,8 @@ def test_symlink(self): # Issue 7880
|
||||||
cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
|
cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
|
||||||
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
|
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
|
||||||
|
|
||||||
|
@ -33,7 +33,7 @@ index 388614e51b..db52725016 100644
|
|||||||
else:
|
else:
|
||||||
return PycInvalidationMode.TIMESTAMP
|
return PycInvalidationMode.TIMESTAMP
|
||||||
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
|
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
|
||||||
index 794d6436b6..322e072b61 100644
|
index a4a52b180d..e53f5d92aa 100644
|
||||||
--- a/Lib/test/test_py_compile.py
|
--- a/Lib/test/test_py_compile.py
|
||||||
+++ b/Lib/test/test_py_compile.py
|
+++ b/Lib/test/test_py_compile.py
|
||||||
@@ -19,6 +19,7 @@ def without_source_date_epoch(fxn):
|
@@ -19,6 +19,7 @@ def without_source_date_epoch(fxn):
|
||||||
|
@ -1,82 +0,0 @@
|
|||||||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Miss Islington (bot)"
|
|
||||||
<31488909+miss-islington@users.noreply.github.com>
|
|
||||||
Date: Sat, 4 Jun 2022 22:15:59 -0700
|
|
||||||
Subject: [PATCH] 00383: gh-93442: Make C++ version of _Py_CAST work with
|
|
||||||
0/NULL
|
|
||||||
MIME-Version: 1.0
|
|
||||||
Content-Type: text/plain; charset=UTF-8
|
|
||||||
Content-Transfer-Encoding: 8bit
|
|
||||||
|
|
||||||
Add C++ overloads for _Py_CAST_impl() to handle 0/NULL. This will allow
|
|
||||||
C++ extensions that pass 0 or NULL to macros using _Py_CAST() to
|
|
||||||
continue to compile. Without this, you get an error like:
|
|
||||||
|
|
||||||
invalid ‘static_cast’ from type ‘int’ to type ‘_object*’
|
|
||||||
|
|
||||||
The modern way to use a NULL value in C++ is to use nullptr. However,
|
|
||||||
we want to not break extensions that do things the old way.
|
|
||||||
|
|
||||||
Co-authored-by: serge-sans-paille
|
|
||||||
(cherry picked from commit 8bcc3fa3453e28511d04eaa0aa7d8e1a3495d518)
|
|
||||||
|
|
||||||
Co-authored-by: Neil Schemenauer <nas-github@arctrix.com>
|
|
||||||
---
|
|
||||||
Include/pyport.h | 14 ++++++++++++++
|
|
||||||
Lib/test/_testcppext.cpp | 4 ++++
|
|
||||||
.../2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst | 3 +++
|
|
||||||
3 files changed, 21 insertions(+)
|
|
||||||
create mode 100644 Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst
|
|
||||||
|
|
||||||
diff --git a/Include/pyport.h b/Include/pyport.h
|
|
||||||
index 5a9adf162b..a78e290931 100644
|
|
||||||
--- a/Include/pyport.h
|
|
||||||
+++ b/Include/pyport.h
|
|
||||||
@@ -24,9 +24,23 @@
|
|
||||||
//
|
|
||||||
// The type argument must not be a constant type.
|
|
||||||
#ifdef __cplusplus
|
|
||||||
+#include <cstddef>
|
|
||||||
# define _Py_STATIC_CAST(type, expr) static_cast<type>(expr)
|
|
||||||
extern "C++" {
|
|
||||||
namespace {
|
|
||||||
+ template <typename type>
|
|
||||||
+ inline type _Py_CAST_impl(long int ptr) {
|
|
||||||
+ return reinterpret_cast<type>(ptr);
|
|
||||||
+ }
|
|
||||||
+ template <typename type>
|
|
||||||
+ inline type _Py_CAST_impl(int ptr) {
|
|
||||||
+ return reinterpret_cast<type>(ptr);
|
|
||||||
+ }
|
|
||||||
+ template <typename type>
|
|
||||||
+ inline type _Py_CAST_impl(std::nullptr_t) {
|
|
||||||
+ return static_cast<type>(nullptr);
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
template <typename type, typename expr_type>
|
|
||||||
inline type _Py_CAST_impl(expr_type *expr) {
|
|
||||||
return reinterpret_cast<type>(expr);
|
|
||||||
diff --git a/Lib/test/_testcppext.cpp b/Lib/test/_testcppext.cpp
|
|
||||||
index eade7ccdaa..70f434e678 100644
|
|
||||||
--- a/Lib/test/_testcppext.cpp
|
|
||||||
+++ b/Lib/test/_testcppext.cpp
|
|
||||||
@@ -74,6 +74,10 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
|
|
||||||
Py_INCREF(strong_ref);
|
|
||||||
Py_DECREF(strong_ref);
|
|
||||||
|
|
||||||
+ // gh-93442: Pass 0 as NULL for PyObject*
|
|
||||||
+ Py_XINCREF(0);
|
|
||||||
+ Py_XDECREF(0);
|
|
||||||
+
|
|
||||||
Py_DECREF(obj);
|
|
||||||
Py_RETURN_NONE;
|
|
||||||
}
|
|
||||||
diff --git a/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst b/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000000..f48ed37c81
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst
|
|
||||||
@@ -0,0 +1,3 @@
|
|
||||||
+Add C++ overloads for _Py_CAST_impl() to handle 0/NULL. This will allow C++
|
|
||||||
+extensions that pass 0 or NULL to macros using _Py_CAST() to continue to
|
|
||||||
+compile.
|
|
@ -1,90 +0,0 @@
|
|||||||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Erlend Egeberg Aasland <erlend.aasland@protonmail.com>
|
|
||||||
Date: Tue, 21 Jun 2022 13:30:29 +0200
|
|
||||||
Subject: [PATCH] 00384: gh-94028: Clear and reset sqlite3 statements properly
|
|
||||||
in cursor iternext (GH-94042)
|
|
||||||
|
|
||||||
---
|
|
||||||
Lib/test/test_sqlite3/test_transactions.py | 39 +++++++++++++++++++
|
|
||||||
...2-06-20-23-14-43.gh-issue-94028.UofEcX.rst | 3 ++
|
|
||||||
Modules/_sqlite/cursor.c | 3 ++
|
|
||||||
3 files changed, 45 insertions(+)
|
|
||||||
create mode 100644 Misc/NEWS.d/next/Library/2022-06-20-23-14-43.gh-issue-94028.UofEcX.rst
|
|
||||||
|
|
||||||
diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py
|
|
||||||
index 040ab1ee60..226a1d03fa 100644
|
|
||||||
--- a/Lib/test/test_sqlite3/test_transactions.py
|
|
||||||
+++ b/Lib/test/test_sqlite3/test_transactions.py
|
|
||||||
@@ -141,6 +141,45 @@ def test_rollback_cursor_consistency(self):
|
|
||||||
con.rollback()
|
|
||||||
self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)])
|
|
||||||
|
|
||||||
+ def test_multiple_cursors_and_iternext(self):
|
|
||||||
+ # gh-94028: statements are cleared and reset in cursor iternext.
|
|
||||||
+
|
|
||||||
+ # Provoke the gh-94028 by using a cursor cache.
|
|
||||||
+ CURSORS = {}
|
|
||||||
+ def sql(cx, sql, *args):
|
|
||||||
+ cu = cx.cursor()
|
|
||||||
+ cu.execute(sql, args)
|
|
||||||
+ CURSORS[id(sql)] = cu
|
|
||||||
+ return cu
|
|
||||||
+
|
|
||||||
+ self.con1.execute("create table t(t)")
|
|
||||||
+ sql(self.con1, "insert into t values (?), (?), (?)", "u1", "u2", "u3")
|
|
||||||
+ self.con1.commit()
|
|
||||||
+
|
|
||||||
+ # On second connection, verify rows are visible, then delete them.
|
|
||||||
+ count = sql(self.con2, "select count(*) from t").fetchone()[0]
|
|
||||||
+ self.assertEqual(count, 3)
|
|
||||||
+ changes = sql(self.con2, "delete from t").rowcount
|
|
||||||
+ self.assertEqual(changes, 3)
|
|
||||||
+ self.con2.commit()
|
|
||||||
+
|
|
||||||
+ # Back in original connection, create 2 new users.
|
|
||||||
+ sql(self.con1, "insert into t values (?)", "u4")
|
|
||||||
+ sql(self.con1, "insert into t values (?)", "u5")
|
|
||||||
+
|
|
||||||
+ # The second connection cannot see uncommitted changes.
|
|
||||||
+ count = sql(self.con2, "select count(*) from t").fetchone()[0]
|
|
||||||
+ self.assertEqual(count, 0)
|
|
||||||
+
|
|
||||||
+ # First connection can see its own changes.
|
|
||||||
+ count = sql(self.con1, "select count(*) from t").fetchone()[0]
|
|
||||||
+ self.assertEqual(count, 2)
|
|
||||||
+
|
|
||||||
+ # The second connection can now see the changes.
|
|
||||||
+ self.con1.commit()
|
|
||||||
+ count = sql(self.con2, "select count(*) from t").fetchone()[0]
|
|
||||||
+ self.assertEqual(count, 2)
|
|
||||||
+
|
|
||||||
|
|
||||||
class RollbackTests(unittest.TestCase):
|
|
||||||
"""bpo-44092: sqlite3 now leaves it to SQLite to resolve rollback issues"""
|
|
||||||
diff --git a/Misc/NEWS.d/next/Library/2022-06-20-23-14-43.gh-issue-94028.UofEcX.rst b/Misc/NEWS.d/next/Library/2022-06-20-23-14-43.gh-issue-94028.UofEcX.rst
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000000..5775b2276d
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/Misc/NEWS.d/next/Library/2022-06-20-23-14-43.gh-issue-94028.UofEcX.rst
|
|
||||||
@@ -0,0 +1,3 @@
|
|
||||||
+Fix a regression in the :mod:`sqlite3` where statement objects were not
|
|
||||||
+properly cleared and reset after use in cursor iters. The regression was
|
|
||||||
+introduced by PR 27884 in Python 3.11a1. Patch by Erlend E. Aasland.
|
|
||||||
diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c
|
|
||||||
index c58def5f03..8048d83683 100644
|
|
||||||
--- a/Modules/_sqlite/cursor.c
|
|
||||||
+++ b/Modules/_sqlite/cursor.c
|
|
||||||
@@ -1126,10 +1126,13 @@ pysqlite_cursor_iternext(pysqlite_Cursor *self)
|
|
||||||
int rc = stmt_step(stmt);
|
|
||||||
if (rc == SQLITE_DONE) {
|
|
||||||
(void)stmt_reset(self->statement);
|
|
||||||
+ Py_CLEAR(self->statement);
|
|
||||||
}
|
|
||||||
else if (rc != SQLITE_ROW) {
|
|
||||||
(void)_pysqlite_seterror(self->connection->state,
|
|
||||||
self->connection->db);
|
|
||||||
+ (void)stmt_reset(self->statement);
|
|
||||||
+ Py_CLEAR(self->statement);
|
|
||||||
Py_DECREF(row);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
@ -1,746 +0,0 @@
|
|||||||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Miss Islington (bot)"
|
|
||||||
<31488909+miss-islington@users.noreply.github.com>
|
|
||||||
Date: Fri, 17 Jun 2022 01:43:56 -0700
|
|
||||||
Subject: [PATCH] 00385: gh-91404: Revert "bpo-23689: re module, fix memory
|
|
||||||
leak..."
|
|
||||||
MIME-Version: 1.0
|
|
||||||
Content-Type: text/plain; charset=UTF-8
|
|
||||||
Content-Transfer-Encoding: 8bit
|
|
||||||
|
|
||||||
This fixes a speed regression in the re module
|
|
||||||
which prevented chromium from building in Fedora.
|
|
||||||
|
|
||||||
Revert "bpo-23689: re module, fix memory leak when a match is terminated by a signal or memory allocation failure"
|
|
||||||
|
|
||||||
This reverts commit 6e3eee5c11b539e9aab39cff783acf57838c355a.
|
|
||||||
|
|
||||||
Manual fixups to increase the MAGIC number and to handle conflicts with
|
|
||||||
a couple of changes that landed after that.
|
|
||||||
|
|
||||||
(cherry picked from commit 4beee0c7b0c2cc78a893dde88fd8e34099dcf877)
|
|
||||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
|
||||||
|
|
||||||
gh-94675: Add a regression test for rjsmin re slowdown
|
|
||||||
|
|
||||||
Co-authored-by: Miro Hrončok <miro@hroncok.cz>
|
|
||||||
---
|
|
||||||
Lib/re/_compiler.py | 59 ++++++-----------
|
|
||||||
Lib/re/_constants.py | 2 +-
|
|
||||||
Lib/test/test_re.py | 59 +++++++++--------
|
|
||||||
...2-06-15-21-35-11.gh-issue-91404.39TZzW.rst | 3 +
|
|
||||||
...2-07-08-12-22-00.gh-issue-94675.IiTs5f.rst | 1 +
|
|
||||||
Modules/_sre/clinic/sre.c.h | 27 ++------
|
|
||||||
Modules/_sre/sre.c | 65 +++++++------------
|
|
||||||
Modules/_sre/sre.h | 4 --
|
|
||||||
Modules/_sre/sre_constants.h | 2 +-
|
|
||||||
Modules/_sre/sre_lib.h | 30 +++++----
|
|
||||||
10 files changed, 105 insertions(+), 147 deletions(-)
|
|
||||||
create mode 100644 Misc/NEWS.d/next/Library/2022-06-15-21-35-11.gh-issue-91404.39TZzW.rst
|
|
||||||
create mode 100644 Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
|
||||||
|
|
||||||
diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py
|
|
||||||
index 4b5322338c..d8e0d2fdef 100644
|
|
||||||
--- a/Lib/re/_compiler.py
|
|
||||||
+++ b/Lib/re/_compiler.py
|
|
||||||
@@ -28,21 +28,14 @@
|
|
||||||
POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE),
|
|
||||||
}
|
|
||||||
|
|
||||||
-class _CompileData:
|
|
||||||
- __slots__ = ('code', 'repeat_count')
|
|
||||||
- def __init__(self):
|
|
||||||
- self.code = []
|
|
||||||
- self.repeat_count = 0
|
|
||||||
-
|
|
||||||
def _combine_flags(flags, add_flags, del_flags,
|
|
||||||
TYPE_FLAGS=_parser.TYPE_FLAGS):
|
|
||||||
if add_flags & TYPE_FLAGS:
|
|
||||||
flags &= ~TYPE_FLAGS
|
|
||||||
return (flags | add_flags) & ~del_flags
|
|
||||||
|
|
||||||
-def _compile(data, pattern, flags):
|
|
||||||
+def _compile(code, pattern, flags):
|
|
||||||
# internal: compile a (sub)pattern
|
|
||||||
- code = data.code
|
|
||||||
emit = code.append
|
|
||||||
_len = len
|
|
||||||
LITERAL_CODES = _LITERAL_CODES
|
|
||||||
@@ -115,7 +108,7 @@ def _compile(data, pattern, flags):
|
|
||||||
skip = _len(code); emit(0)
|
|
||||||
emit(av[0])
|
|
||||||
emit(av[1])
|
|
||||||
- _compile(data, av[2], flags)
|
|
||||||
+ _compile(code, av[2], flags)
|
|
||||||
emit(SUCCESS)
|
|
||||||
code[skip] = _len(code) - skip
|
|
||||||
else:
|
|
||||||
@@ -123,11 +116,7 @@ def _compile(data, pattern, flags):
|
|
||||||
skip = _len(code); emit(0)
|
|
||||||
emit(av[0])
|
|
||||||
emit(av[1])
|
|
||||||
- # now op is in (MIN_REPEAT, MAX_REPEAT, POSSESSIVE_REPEAT)
|
|
||||||
- if op != POSSESSIVE_REPEAT:
|
|
||||||
- emit(data.repeat_count)
|
|
||||||
- data.repeat_count += 1
|
|
||||||
- _compile(data, av[2], flags)
|
|
||||||
+ _compile(code, av[2], flags)
|
|
||||||
code[skip] = _len(code) - skip
|
|
||||||
emit(REPEATING_CODES[op][1])
|
|
||||||
elif op is SUBPATTERN:
|
|
||||||
@@ -136,7 +125,7 @@ def _compile(data, pattern, flags):
|
|
||||||
emit(MARK)
|
|
||||||
emit((group-1)*2)
|
|
||||||
# _compile_info(code, p, _combine_flags(flags, add_flags, del_flags))
|
|
||||||
- _compile(data, p, _combine_flags(flags, add_flags, del_flags))
|
|
||||||
+ _compile(code, p, _combine_flags(flags, add_flags, del_flags))
|
|
||||||
if group:
|
|
||||||
emit(MARK)
|
|
||||||
emit((group-1)*2+1)
|
|
||||||
@@ -148,7 +137,7 @@ def _compile(data, pattern, flags):
|
|
||||||
# pop their stack if they reach it
|
|
||||||
emit(ATOMIC_GROUP)
|
|
||||||
skip = _len(code); emit(0)
|
|
||||||
- _compile(data, av, flags)
|
|
||||||
+ _compile(code, av, flags)
|
|
||||||
emit(SUCCESS)
|
|
||||||
code[skip] = _len(code) - skip
|
|
||||||
elif op in SUCCESS_CODES:
|
|
||||||
@@ -163,7 +152,7 @@ def _compile(data, pattern, flags):
|
|
||||||
if lo != hi:
|
|
||||||
raise error("look-behind requires fixed-width pattern")
|
|
||||||
emit(lo) # look behind
|
|
||||||
- _compile(data, av[1], flags)
|
|
||||||
+ _compile(code, av[1], flags)
|
|
||||||
emit(SUCCESS)
|
|
||||||
code[skip] = _len(code) - skip
|
|
||||||
elif op is AT:
|
|
||||||
@@ -182,7 +171,7 @@ def _compile(data, pattern, flags):
|
|
||||||
for av in av[1]:
|
|
||||||
skip = _len(code); emit(0)
|
|
||||||
# _compile_info(code, av, flags)
|
|
||||||
- _compile(data, av, flags)
|
|
||||||
+ _compile(code, av, flags)
|
|
||||||
emit(JUMP)
|
|
||||||
tailappend(_len(code)); emit(0)
|
|
||||||
code[skip] = _len(code) - skip
|
|
||||||
@@ -210,12 +199,12 @@ def _compile(data, pattern, flags):
|
|
||||||
emit(op)
|
|
||||||
emit(av[0]-1)
|
|
||||||
skipyes = _len(code); emit(0)
|
|
||||||
- _compile(data, av[1], flags)
|
|
||||||
+ _compile(code, av[1], flags)
|
|
||||||
if av[2]:
|
|
||||||
emit(JUMP)
|
|
||||||
skipno = _len(code); emit(0)
|
|
||||||
code[skipyes] = _len(code) - skipyes + 1
|
|
||||||
- _compile(data, av[2], flags)
|
|
||||||
+ _compile(code, av[2], flags)
|
|
||||||
code[skipno] = _len(code) - skipno
|
|
||||||
else:
|
|
||||||
code[skipyes] = _len(code) - skipyes + 1
|
|
||||||
@@ -582,17 +571,17 @@ def isstring(obj):
|
|
||||||
def _code(p, flags):
|
|
||||||
|
|
||||||
flags = p.state.flags | flags
|
|
||||||
- data = _CompileData()
|
|
||||||
+ code = []
|
|
||||||
|
|
||||||
# compile info block
|
|
||||||
- _compile_info(data.code, p, flags)
|
|
||||||
+ _compile_info(code, p, flags)
|
|
||||||
|
|
||||||
# compile the pattern
|
|
||||||
- _compile(data, p.data, flags)
|
|
||||||
+ _compile(code, p.data, flags)
|
|
||||||
|
|
||||||
- data.code.append(SUCCESS)
|
|
||||||
+ code.append(SUCCESS)
|
|
||||||
|
|
||||||
- return data
|
|
||||||
+ return code
|
|
||||||
|
|
||||||
def _hex_code(code):
|
|
||||||
return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code)
|
|
||||||
@@ -693,7 +682,7 @@ def print_2(*args):
|
|
||||||
else:
|
|
||||||
print_(FAILURE)
|
|
||||||
i += 1
|
|
||||||
- elif op in (REPEAT_ONE, MIN_REPEAT_ONE,
|
|
||||||
+ elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE,
|
|
||||||
POSSESSIVE_REPEAT, POSSESSIVE_REPEAT_ONE):
|
|
||||||
skip, min, max = code[i: i+3]
|
|
||||||
if max == MAXREPEAT:
|
|
||||||
@@ -701,13 +690,6 @@ def print_2(*args):
|
|
||||||
print_(op, skip, min, max, to=i+skip)
|
|
||||||
dis_(i+3, i+skip)
|
|
||||||
i += skip
|
|
||||||
- elif op is REPEAT:
|
|
||||||
- skip, min, max, repeat_index = code[i: i+4]
|
|
||||||
- if max == MAXREPEAT:
|
|
||||||
- max = 'MAXREPEAT'
|
|
||||||
- print_(op, skip, min, max, repeat_index, to=i+skip)
|
|
||||||
- dis_(i+4, i+skip)
|
|
||||||
- i += skip
|
|
||||||
elif op is GROUPREF_EXISTS:
|
|
||||||
arg, skip = code[i: i+2]
|
|
||||||
print_(op, arg, skip, to=i+skip)
|
|
||||||
@@ -762,11 +744,11 @@ def compile(p, flags=0):
|
|
||||||
else:
|
|
||||||
pattern = None
|
|
||||||
|
|
||||||
- data = _code(p, flags)
|
|
||||||
+ code = _code(p, flags)
|
|
||||||
|
|
||||||
if flags & SRE_FLAG_DEBUG:
|
|
||||||
print()
|
|
||||||
- dis(data.code)
|
|
||||||
+ dis(code)
|
|
||||||
|
|
||||||
# map in either direction
|
|
||||||
groupindex = p.state.groupdict
|
|
||||||
@@ -775,6 +757,7 @@ def compile(p, flags=0):
|
|
||||||
indexgroup[i] = k
|
|
||||||
|
|
||||||
return _sre.compile(
|
|
||||||
- pattern, flags | p.state.flags, data.code,
|
|
||||||
- p.state.groups-1, groupindex, tuple(indexgroup),
|
|
||||||
- data.repeat_count)
|
|
||||||
+ pattern, flags | p.state.flags, code,
|
|
||||||
+ p.state.groups-1,
|
|
||||||
+ groupindex, tuple(indexgroup)
|
|
||||||
+ )
|
|
||||||
diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py
|
|
||||||
index 1cc85c631f..10ee14bfab 100644
|
|
||||||
--- a/Lib/re/_constants.py
|
|
||||||
+++ b/Lib/re/_constants.py
|
|
||||||
@@ -13,7 +13,7 @@
|
|
||||||
|
|
||||||
# update when constants are added or removed
|
|
||||||
|
|
||||||
-MAGIC = 20220423
|
|
||||||
+MAGIC = 20220615
|
|
||||||
|
|
||||||
from _sre import MAXREPEAT, MAXGROUPS
|
|
||||||
|
|
||||||
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
|
|
||||||
index 6d61412f16..5d946370ee 100644
|
|
||||||
--- a/Lib/test/test_re.py
|
|
||||||
+++ b/Lib/test/test_re.py
|
|
||||||
@@ -1,6 +1,7 @@
|
|
||||||
from test.support import (gc_collect, bigmemtest, _2G,
|
|
||||||
cpython_only, captured_stdout,
|
|
||||||
- check_disallow_instantiation, is_emscripten, is_wasi)
|
|
||||||
+ check_disallow_instantiation, is_emscripten, is_wasi,
|
|
||||||
+ SHORT_TIMEOUT)
|
|
||||||
import locale
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
@@ -11,6 +12,14 @@
|
|
||||||
from re import Scanner
|
|
||||||
from weakref import proxy
|
|
||||||
|
|
||||||
+# some platforms lack working multiprocessing
|
|
||||||
+try:
|
|
||||||
+ import _multiprocessing
|
|
||||||
+except ImportError:
|
|
||||||
+ multiprocessing = None
|
|
||||||
+else:
|
|
||||||
+ import multiprocessing
|
|
||||||
+
|
|
||||||
# Misc tests from Tim Peters' re.doc
|
|
||||||
|
|
||||||
# WARNING: Don't change details in these tests if you don't know
|
|
||||||
@@ -1796,12 +1805,9 @@ def test_dealloc(self):
|
|
||||||
long_overflow = 2**128
|
|
||||||
self.assertRaises(TypeError, re.finditer, "a", {})
|
|
||||||
with self.assertRaises(OverflowError):
|
|
||||||
- _sre.compile("abc", 0, [long_overflow], 0, {}, (), 0)
|
|
||||||
+ _sre.compile("abc", 0, [long_overflow], 0, {}, ())
|
|
||||||
with self.assertRaises(TypeError):
|
|
||||||
- _sre.compile({}, 0, [], 0, [], [], 0)
|
|
||||||
- with self.assertRaises(RuntimeError):
|
|
||||||
- # invalid repeat_count -1
|
|
||||||
- _sre.compile("abc", 0, [1], 0, {}, (), -1)
|
|
||||||
+ _sre.compile({}, 0, [], 0, [], [])
|
|
||||||
|
|
||||||
def test_search_dot_unicode(self):
|
|
||||||
self.assertTrue(re.search("123.*-", '123abc-'))
|
|
||||||
@@ -2441,6 +2447,26 @@ def test_template_function_and_flag_is_deprecated(self):
|
|
||||||
self.assertTrue(template_re1.match('ahoy'))
|
|
||||||
self.assertFalse(template_re1.match('nope'))
|
|
||||||
|
|
||||||
+ @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing')
|
|
||||||
+ def test_regression_gh94675(self):
|
|
||||||
+ pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*'
|
|
||||||
+ r'((/[^/\[\n]*(([^\n]|(\[\n]*(]*)*\]))'
|
|
||||||
+ r'[^/\[]*)*/))((((//[^\n]*)?[\n])'
|
|
||||||
+ r'([\000-\040]|(/\*[^*]*\*+'
|
|
||||||
+ r'([^/*]\*+)*/))*)+(?=[^\000-\040);\]}]))')
|
|
||||||
+ input_js = '''a(function() {
|
|
||||||
+ ///////////////////////////////////////////////////////////////////
|
|
||||||
+ });'''
|
|
||||||
+ p = multiprocessing.Process(target=pattern.sub, args=('', input_js))
|
|
||||||
+ p.start()
|
|
||||||
+ p.join(SHORT_TIMEOUT)
|
|
||||||
+ try:
|
|
||||||
+ self.assertFalse(p.is_alive(), 'pattern.sub() timed out')
|
|
||||||
+ finally:
|
|
||||||
+ if p.is_alive():
|
|
||||||
+ p.terminate()
|
|
||||||
+ p.join()
|
|
||||||
+
|
|
||||||
|
|
||||||
def get_debug_out(pat):
|
|
||||||
with captured_stdout() as out:
|
|
||||||
@@ -2540,27 +2566,6 @@ def test_possesive_repeat(self):
|
|
||||||
14. SUCCESS
|
|
||||||
''')
|
|
||||||
|
|
||||||
- def test_repeat_index(self):
|
|
||||||
- self.assertEqual(get_debug_out(r'(?:ab)*?(?:cd)*'), '''\
|
|
||||||
-MIN_REPEAT 0 MAXREPEAT
|
|
||||||
- LITERAL 97
|
|
||||||
- LITERAL 98
|
|
||||||
-MAX_REPEAT 0 MAXREPEAT
|
|
||||||
- LITERAL 99
|
|
||||||
- LITERAL 100
|
|
||||||
-
|
|
||||||
- 0. INFO 4 0b0 0 MAXREPEAT (to 5)
|
|
||||||
- 5: REPEAT 8 0 MAXREPEAT 0 (to 14)
|
|
||||||
-10. LITERAL 0x61 ('a')
|
|
||||||
-12. LITERAL 0x62 ('b')
|
|
||||||
-14: MIN_UNTIL
|
|
||||||
-15. REPEAT 8 0 MAXREPEAT 1 (to 24)
|
|
||||||
-20. LITERAL 0x63 ('c')
|
|
||||||
-22. LITERAL 0x64 ('d')
|
|
||||||
-24: MAX_UNTIL
|
|
||||||
-25. SUCCESS
|
|
||||||
-''')
|
|
||||||
-
|
|
||||||
|
|
||||||
class PatternReprTests(unittest.TestCase):
|
|
||||||
def check(self, pattern, expected):
|
|
||||||
diff --git a/Misc/NEWS.d/next/Library/2022-06-15-21-35-11.gh-issue-91404.39TZzW.rst b/Misc/NEWS.d/next/Library/2022-06-15-21-35-11.gh-issue-91404.39TZzW.rst
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000000..e20b15c7b7
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/Misc/NEWS.d/next/Library/2022-06-15-21-35-11.gh-issue-91404.39TZzW.rst
|
|
||||||
@@ -0,0 +1,3 @@
|
|
||||||
+Revert the :mod:`re` memory leak when a match is terminated by a signal or
|
|
||||||
+memory allocation failure as the implemented fix caused a major performance
|
|
||||||
+regression.
|
|
||||||
diff --git a/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst b/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000000..d0005d9f60
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
|
||||||
@@ -0,0 +1 @@
|
|
||||||
+Add a regression test for :mod:`re` exponentional slowdown when using rjsmin.
|
|
||||||
diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h
|
|
||||||
index e243c756e1..048a494f1b 100644
|
|
||||||
--- a/Modules/_sre/clinic/sre.c.h
|
|
||||||
+++ b/Modules/_sre/clinic/sre.c.h
|
|
||||||
@@ -764,7 +764,7 @@ PyDoc_STRVAR(_sre_SRE_Pattern___deepcopy____doc__,
|
|
||||||
|
|
||||||
PyDoc_STRVAR(_sre_compile__doc__,
|
|
||||||
"compile($module, /, pattern, flags, code, groups, groupindex,\n"
|
|
||||||
-" indexgroup, repeat_count)\n"
|
|
||||||
+" indexgroup)\n"
|
|
||||||
"--\n"
|
|
||||||
"\n");
|
|
||||||
|
|
||||||
@@ -774,24 +774,23 @@ PyDoc_STRVAR(_sre_compile__doc__,
|
|
||||||
static PyObject *
|
|
||||||
_sre_compile_impl(PyObject *module, PyObject *pattern, int flags,
|
|
||||||
PyObject *code, Py_ssize_t groups, PyObject *groupindex,
|
|
||||||
- PyObject *indexgroup, Py_ssize_t repeat_count);
|
|
||||||
+ PyObject *indexgroup);
|
|
||||||
|
|
||||||
static PyObject *
|
|
||||||
_sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
|
|
||||||
{
|
|
||||||
PyObject *return_value = NULL;
|
|
||||||
- static const char * const _keywords[] = {"pattern", "flags", "code", "groups", "groupindex", "indexgroup", "repeat_count", NULL};
|
|
||||||
+ static const char * const _keywords[] = {"pattern", "flags", "code", "groups", "groupindex", "indexgroup", NULL};
|
|
||||||
static _PyArg_Parser _parser = {NULL, _keywords, "compile", 0};
|
|
||||||
- PyObject *argsbuf[7];
|
|
||||||
+ PyObject *argsbuf[6];
|
|
||||||
PyObject *pattern;
|
|
||||||
int flags;
|
|
||||||
PyObject *code;
|
|
||||||
Py_ssize_t groups;
|
|
||||||
PyObject *groupindex;
|
|
||||||
PyObject *indexgroup;
|
|
||||||
- Py_ssize_t repeat_count;
|
|
||||||
|
|
||||||
- args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 0, argsbuf);
|
|
||||||
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf);
|
|
||||||
if (!args) {
|
|
||||||
goto exit;
|
|
||||||
}
|
|
||||||
@@ -827,19 +826,7 @@ _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
|
|
||||||
goto exit;
|
|
||||||
}
|
|
||||||
indexgroup = args[5];
|
|
||||||
- {
|
|
||||||
- Py_ssize_t ival = -1;
|
|
||||||
- PyObject *iobj = _PyNumber_Index(args[6]);
|
|
||||||
- if (iobj != NULL) {
|
|
||||||
- ival = PyLong_AsSsize_t(iobj);
|
|
||||||
- Py_DECREF(iobj);
|
|
||||||
- }
|
|
||||||
- if (ival == -1 && PyErr_Occurred()) {
|
|
||||||
- goto exit;
|
|
||||||
- }
|
|
||||||
- repeat_count = ival;
|
|
||||||
- }
|
|
||||||
- return_value = _sre_compile_impl(module, pattern, flags, code, groups, groupindex, indexgroup, repeat_count);
|
|
||||||
+ return_value = _sre_compile_impl(module, pattern, flags, code, groups, groupindex, indexgroup);
|
|
||||||
|
|
||||||
exit:
|
|
||||||
return return_value;
|
|
||||||
@@ -1129,4 +1116,4 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const
|
|
||||||
}
|
|
||||||
return _sre_SRE_Scanner_search_impl(self, cls);
|
|
||||||
}
|
|
||||||
-/*[clinic end generated code: output=97e7ce058366760b input=a9049054013a1b77]*/
|
|
||||||
+/*[clinic end generated code: output=fd2f45c941620e6e input=a9049054013a1b77]*/
|
|
||||||
diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c
|
|
||||||
index 491734f243..0a7019a085 100644
|
|
||||||
--- a/Modules/_sre/sre.c
|
|
||||||
+++ b/Modules/_sre/sre.c
|
|
||||||
@@ -427,12 +427,6 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
|
|
||||||
state->lastmark = -1;
|
|
||||||
state->lastindex = -1;
|
|
||||||
|
|
||||||
- state->repeats_array = PyMem_New(SRE_REPEAT, pattern->repeat_count);
|
|
||||||
- if (!state->repeats_array) {
|
|
||||||
- PyErr_NoMemory();
|
|
||||||
- goto err;
|
|
||||||
- }
|
|
||||||
-
|
|
||||||
state->buffer.buf = NULL;
|
|
||||||
ptr = getstring(string, &length, &isbytes, &charsize, &state->buffer);
|
|
||||||
if (!ptr)
|
|
||||||
@@ -482,9 +476,6 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
|
|
||||||
safely casted to `void*`, see bpo-39943 for details. */
|
|
||||||
PyMem_Free((void*) state->mark);
|
|
||||||
state->mark = NULL;
|
|
||||||
- PyMem_Free(state->repeats_array);
|
|
||||||
- state->repeats_array = NULL;
|
|
||||||
-
|
|
||||||
if (state->buffer.buf)
|
|
||||||
PyBuffer_Release(&state->buffer);
|
|
||||||
return NULL;
|
|
||||||
@@ -500,8 +491,6 @@ state_fini(SRE_STATE* state)
|
|
||||||
/* See above PyMem_Del for why we explicitly cast here. */
|
|
||||||
PyMem_Free((void*) state->mark);
|
|
||||||
state->mark = NULL;
|
|
||||||
- PyMem_Free(state->repeats_array);
|
|
||||||
- state->repeats_array = NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* calculate offset from start of string */
|
|
||||||
@@ -1418,15 +1407,14 @@ _sre.compile
|
|
||||||
groups: Py_ssize_t
|
|
||||||
groupindex: object(subclass_of='&PyDict_Type')
|
|
||||||
indexgroup: object(subclass_of='&PyTuple_Type')
|
|
||||||
- repeat_count: Py_ssize_t
|
|
||||||
|
|
||||||
[clinic start generated code]*/
|
|
||||||
|
|
||||||
static PyObject *
|
|
||||||
_sre_compile_impl(PyObject *module, PyObject *pattern, int flags,
|
|
||||||
PyObject *code, Py_ssize_t groups, PyObject *groupindex,
|
|
||||||
- PyObject *indexgroup, Py_ssize_t repeat_count)
|
|
||||||
-/*[clinic end generated code: output=922af562d51b1657 input=77e39c322501ec2a]*/
|
|
||||||
+ PyObject *indexgroup)
|
|
||||||
+/*[clinic end generated code: output=ef9c2b3693776404 input=0a68476dbbe5db30]*/
|
|
||||||
{
|
|
||||||
/* "compile" pattern descriptor to pattern object */
|
|
||||||
|
|
||||||
@@ -1484,8 +1472,8 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags,
|
|
||||||
self->pattern = pattern;
|
|
||||||
|
|
||||||
self->flags = flags;
|
|
||||||
+
|
|
||||||
self->groups = groups;
|
|
||||||
- self->repeat_count = repeat_count;
|
|
||||||
|
|
||||||
if (PyDict_GET_SIZE(groupindex) > 0) {
|
|
||||||
Py_INCREF(groupindex);
|
|
||||||
@@ -1657,7 +1645,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end)
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
|
||||||
-_validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
+_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups)
|
|
||||||
{
|
|
||||||
/* Some variables are manipulated by the macros above */
|
|
||||||
SRE_CODE op;
|
|
||||||
@@ -1678,8 +1666,8 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
sre_match() code is robust even if they don't, and the worst
|
|
||||||
you can get is nonsensical match results. */
|
|
||||||
GET_ARG;
|
|
||||||
- if (arg > 2 * (size_t)self->groups + 1) {
|
|
||||||
- VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)self->groups));
|
|
||||||
+ if (arg > 2 * (size_t)groups + 1) {
|
|
||||||
+ VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups));
|
|
||||||
FAIL;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
@@ -1808,7 +1796,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
if (skip == 0)
|
|
||||||
break;
|
|
||||||
/* Stop 2 before the end; we check the JUMP below */
|
|
||||||
- if (!_validate_inner(code, code+skip-3, self))
|
|
||||||
+ if (!_validate_inner(code, code+skip-3, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-3;
|
|
||||||
/* Check that it ends with a JUMP, and that each JUMP
|
|
||||||
@@ -1837,7 +1825,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
FAIL;
|
|
||||||
if (max > SRE_MAXREPEAT)
|
|
||||||
FAIL;
|
|
||||||
- if (!_validate_inner(code, code+skip-4, self))
|
|
||||||
+ if (!_validate_inner(code, code+skip-4, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-4;
|
|
||||||
GET_OP;
|
|
||||||
@@ -1849,7 +1837,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
case SRE_OP_REPEAT:
|
|
||||||
case SRE_OP_POSSESSIVE_REPEAT:
|
|
||||||
{
|
|
||||||
- SRE_CODE op1 = op, min, max, repeat_index;
|
|
||||||
+ SRE_CODE op1 = op, min, max;
|
|
||||||
GET_SKIP;
|
|
||||||
GET_ARG; min = arg;
|
|
||||||
GET_ARG; max = arg;
|
|
||||||
@@ -1857,17 +1845,9 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
FAIL;
|
|
||||||
if (max > SRE_MAXREPEAT)
|
|
||||||
FAIL;
|
|
||||||
- if (op1 == SRE_OP_REPEAT) {
|
|
||||||
- GET_ARG; repeat_index = arg;
|
|
||||||
- if (repeat_index >= (size_t)self->repeat_count)
|
|
||||||
- FAIL;
|
|
||||||
- skip -= 4;
|
|
||||||
- } else {
|
|
||||||
- skip -= 3;
|
|
||||||
- }
|
|
||||||
- if (!_validate_inner(code, code+skip, self))
|
|
||||||
+ if (!_validate_inner(code, code+skip-3, groups))
|
|
||||||
FAIL;
|
|
||||||
- code += skip;
|
|
||||||
+ code += skip-3;
|
|
||||||
GET_OP;
|
|
||||||
if (op1 == SRE_OP_POSSESSIVE_REPEAT) {
|
|
||||||
if (op != SRE_OP_SUCCESS)
|
|
||||||
@@ -1883,7 +1863,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
case SRE_OP_ATOMIC_GROUP:
|
|
||||||
{
|
|
||||||
GET_SKIP;
|
|
||||||
- if (!_validate_inner(code, code+skip-2, self))
|
|
||||||
+ if (!_validate_inner(code, code+skip-2, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-2;
|
|
||||||
GET_OP;
|
|
||||||
@@ -1897,7 +1877,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
case SRE_OP_GROUPREF_UNI_IGNORE:
|
|
||||||
case SRE_OP_GROUPREF_LOC_IGNORE:
|
|
||||||
GET_ARG;
|
|
||||||
- if (arg >= (size_t)self->groups)
|
|
||||||
+ if (arg >= (size_t)groups)
|
|
||||||
FAIL;
|
|
||||||
break;
|
|
||||||
|
|
||||||
@@ -1906,7 +1886,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
'group' is either an integer group number or a group name,
|
|
||||||
'then' and 'else' are sub-regexes, and 'else' is optional. */
|
|
||||||
GET_ARG;
|
|
||||||
- if (arg >= (size_t)self->groups)
|
|
||||||
+ if (arg >= (size_t)groups)
|
|
||||||
FAIL;
|
|
||||||
GET_SKIP_ADJ(1);
|
|
||||||
code--; /* The skip is relative to the first arg! */
|
|
||||||
@@ -1939,17 +1919,17 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
code[skip-3] == SRE_OP_JUMP)
|
|
||||||
{
|
|
||||||
VTRACE(("both then and else parts present\n"));
|
|
||||||
- if (!_validate_inner(code+1, code+skip-3, self))
|
|
||||||
+ if (!_validate_inner(code+1, code+skip-3, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-2; /* Position after JUMP, at <skipno> */
|
|
||||||
GET_SKIP;
|
|
||||||
- if (!_validate_inner(code, code+skip-1, self))
|
|
||||||
+ if (!_validate_inner(code, code+skip-1, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-1;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
VTRACE(("only a then part present\n"));
|
|
||||||
- if (!_validate_inner(code+1, code+skip-1, self))
|
|
||||||
+ if (!_validate_inner(code+1, code+skip-1, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-1;
|
|
||||||
}
|
|
||||||
@@ -1963,7 +1943,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
if (arg & 0x80000000)
|
|
||||||
FAIL; /* Width too large */
|
|
||||||
/* Stop 1 before the end; we check the SUCCESS below */
|
|
||||||
- if (!_validate_inner(code+1, code+skip-2, self))
|
|
||||||
+ if (!_validate_inner(code+1, code+skip-2, groups))
|
|
||||||
FAIL;
|
|
||||||
code += skip-2;
|
|
||||||
GET_OP;
|
|
||||||
@@ -1982,19 +1962,18 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
|
||||||
-_validate_outer(SRE_CODE *code, SRE_CODE *end, PatternObject *self)
|
|
||||||
+_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups)
|
|
||||||
{
|
|
||||||
- if (self->groups < 0 || (size_t)self->groups > SRE_MAXGROUPS ||
|
|
||||||
- self->repeat_count < 0 ||
|
|
||||||
+ if (groups < 0 || (size_t)groups > SRE_MAXGROUPS ||
|
|
||||||
code >= end || end[-1] != SRE_OP_SUCCESS)
|
|
||||||
FAIL;
|
|
||||||
- return _validate_inner(code, end-1, self);
|
|
||||||
+ return _validate_inner(code, end-1, groups);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
|
||||||
_validate(PatternObject *self)
|
|
||||||
{
|
|
||||||
- if (!_validate_outer(self->code, self->code+self->codesize, self))
|
|
||||||
+ if (!_validate_outer(self->code, self->code+self->codesize, self->groups))
|
|
||||||
{
|
|
||||||
PyErr_SetString(PyExc_RuntimeError, "invalid SRE code");
|
|
||||||
return 0;
|
|
||||||
diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h
|
|
||||||
index aff064d343..52ae3e11b5 100644
|
|
||||||
--- a/Modules/_sre/sre.h
|
|
||||||
+++ b/Modules/_sre/sre.h
|
|
||||||
@@ -29,8 +29,6 @@ typedef struct {
|
|
||||||
Py_ssize_t groups; /* must be first! */
|
|
||||||
PyObject* groupindex; /* dict */
|
|
||||||
PyObject* indexgroup; /* tuple */
|
|
||||||
- /* the number of REPEATs */
|
|
||||||
- Py_ssize_t repeat_count;
|
|
||||||
/* compatibility */
|
|
||||||
PyObject* pattern; /* pattern source (or None) */
|
|
||||||
int flags; /* flags used when compiling pattern source */
|
|
||||||
@@ -85,8 +83,6 @@ typedef struct {
|
|
||||||
size_t data_stack_base;
|
|
||||||
/* current repeat context */
|
|
||||||
SRE_REPEAT *repeat;
|
|
||||||
- /* repeat contexts array */
|
|
||||||
- SRE_REPEAT *repeats_array;
|
|
||||||
} SRE_STATE;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
diff --git a/Modules/_sre/sre_constants.h b/Modules/_sre/sre_constants.h
|
|
||||||
index 590d5be7cb..c633514736 100644
|
|
||||||
--- a/Modules/_sre/sre_constants.h
|
|
||||||
+++ b/Modules/_sre/sre_constants.h
|
|
||||||
@@ -11,7 +11,7 @@
|
|
||||||
* See the sre.c file for information on usage and redistribution.
|
|
||||||
*/
|
|
||||||
|
|
||||||
-#define SRE_MAGIC 20220423
|
|
||||||
+#define SRE_MAGIC 20220615
|
|
||||||
#define SRE_OP_FAILURE 0
|
|
||||||
#define SRE_OP_SUCCESS 1
|
|
||||||
#define SRE_OP_ANY 2
|
|
||||||
diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h
|
|
||||||
index 1e5b50170a..fb4c18b63d 100644
|
|
||||||
--- a/Modules/_sre/sre_lib.h
|
|
||||||
+++ b/Modules/_sre/sre_lib.h
|
|
||||||
@@ -1079,12 +1079,17 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
by the UNTIL operator (MAX_UNTIL, MIN_UNTIL) */
|
|
||||||
/* <REPEAT> <skip> <1=min> <2=max>
|
|
||||||
<3=repeat_index> item <UNTIL> tail */
|
|
||||||
- TRACE(("|%p|%p|REPEAT %d %d %d\n", pattern, ptr,
|
|
||||||
- pattern[1], pattern[2], pattern[3]));
|
|
||||||
-
|
|
||||||
- /* install repeat context */
|
|
||||||
- ctx->u.rep = &state->repeats_array[pattern[3]];
|
|
||||||
+ TRACE(("|%p|%p|REPEAT %d %d\n", pattern, ptr,
|
|
||||||
+ pattern[1], pattern[2]));
|
|
||||||
|
|
||||||
+ /* install new repeat context */
|
|
||||||
+ /* TODO(https://github.com/python/cpython/issues/67877): Fix this
|
|
||||||
+ * potential memory leak. */
|
|
||||||
+ ctx->u.rep = (SRE_REPEAT*) PyObject_Malloc(sizeof(*ctx->u.rep));
|
|
||||||
+ if (!ctx->u.rep) {
|
|
||||||
+ PyErr_NoMemory();
|
|
||||||
+ RETURN_FAILURE;
|
|
||||||
+ }
|
|
||||||
ctx->u.rep->count = -1;
|
|
||||||
ctx->u.rep->pattern = pattern;
|
|
||||||
ctx->u.rep->prev = state->repeat;
|
|
||||||
@@ -1094,6 +1099,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
state->ptr = ptr;
|
|
||||||
DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]);
|
|
||||||
state->repeat = ctx->u.rep->prev;
|
|
||||||
+ PyObject_Free(ctx->u.rep);
|
|
||||||
|
|
||||||
if (ret) {
|
|
||||||
RETURN_ON_ERROR(ret);
|
|
||||||
@@ -1103,8 +1109,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
|
|
||||||
TARGET(SRE_OP_MAX_UNTIL):
|
|
||||||
/* maximizing repeat */
|
|
||||||
- /* <REPEAT> <skip> <1=min> <2=max>
|
|
||||||
- <3=repeat_index> item <MAX_UNTIL> tail */
|
|
||||||
+ /* <REPEAT> <skip> <1=min> <2=max> item <MAX_UNTIL> tail */
|
|
||||||
|
|
||||||
/* FIXME: we probably need to deal with zero-width
|
|
||||||
matches in here... */
|
|
||||||
@@ -1124,7 +1129,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
/* not enough matches */
|
|
||||||
ctx->u.rep->count = ctx->count;
|
|
||||||
DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1,
|
|
||||||
- ctx->u.rep->pattern+4);
|
|
||||||
+ ctx->u.rep->pattern+3);
|
|
||||||
if (ret) {
|
|
||||||
RETURN_ON_ERROR(ret);
|
|
||||||
RETURN_SUCCESS;
|
|
||||||
@@ -1146,7 +1151,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
DATA_PUSH(&ctx->u.rep->last_ptr);
|
|
||||||
ctx->u.rep->last_ptr = state->ptr;
|
|
||||||
DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2,
|
|
||||||
- ctx->u.rep->pattern+4);
|
|
||||||
+ ctx->u.rep->pattern+3);
|
|
||||||
DATA_POP(&ctx->u.rep->last_ptr);
|
|
||||||
if (ret) {
|
|
||||||
MARK_POP_DISCARD(ctx->lastmark);
|
|
||||||
@@ -1171,8 +1176,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
|
|
||||||
TARGET(SRE_OP_MIN_UNTIL):
|
|
||||||
/* minimizing repeat */
|
|
||||||
- /* <REPEAT> <skip> <1=min> <2=max>
|
|
||||||
- <3=repeat_index> item <MIN_UNTIL> tail */
|
|
||||||
+ /* <REPEAT> <skip> <1=min> <2=max> item <MIN_UNTIL> tail */
|
|
||||||
|
|
||||||
ctx->u.rep = state->repeat;
|
|
||||||
if (!ctx->u.rep)
|
|
||||||
@@ -1189,7 +1193,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
/* not enough matches */
|
|
||||||
ctx->u.rep->count = ctx->count;
|
|
||||||
DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1,
|
|
||||||
- ctx->u.rep->pattern+4);
|
|
||||||
+ ctx->u.rep->pattern+3);
|
|
||||||
if (ret) {
|
|
||||||
RETURN_ON_ERROR(ret);
|
|
||||||
RETURN_SUCCESS;
|
|
||||||
@@ -1232,7 +1236,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
|
|
||||||
DATA_PUSH(&ctx->u.rep->last_ptr);
|
|
||||||
ctx->u.rep->last_ptr = state->ptr;
|
|
||||||
DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3,
|
|
||||||
- ctx->u.rep->pattern+4);
|
|
||||||
+ ctx->u.rep->pattern+3);
|
|
||||||
DATA_POP(&ctx->u.rep->last_ptr);
|
|
||||||
if (ret) {
|
|
||||||
RETURN_ON_ERROR(ret);
|
|
@ -0,0 +1,77 @@
|
|||||||
|
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||||
|
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
|
||||||
|
Date: Fri, 8 Jul 2022 12:19:43 +0200
|
||||||
|
Subject: [PATCH] 00385: gh-94675: Add a regression test for rjsmin re slowdown
|
||||||
|
|
||||||
|
This tests a speed regression in the re module
|
||||||
|
which prevented chromium from building in Fedora.
|
||||||
|
|
||||||
|
https://github.com/python/cpython/pull/94685
|
||||||
|
---
|
||||||
|
Lib/test/test_re.py | 31 ++++++++++++++++++-
|
||||||
|
...2-07-08-12-22-00.gh-issue-94675.IiTs5f.rst | 1 +
|
||||||
|
2 files changed, 31 insertions(+), 1 deletion(-)
|
||||||
|
create mode 100644 Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
||||||
|
|
||||||
|
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
|
||||||
|
index ab980793fa..5d946370ee 100644
|
||||||
|
--- a/Lib/test/test_re.py
|
||||||
|
+++ b/Lib/test/test_re.py
|
||||||
|
@@ -1,6 +1,7 @@
|
||||||
|
from test.support import (gc_collect, bigmemtest, _2G,
|
||||||
|
cpython_only, captured_stdout,
|
||||||
|
- check_disallow_instantiation, is_emscripten, is_wasi)
|
||||||
|
+ check_disallow_instantiation, is_emscripten, is_wasi,
|
||||||
|
+ SHORT_TIMEOUT)
|
||||||
|
import locale
|
||||||
|
import re
|
||||||
|
import string
|
||||||
|
@@ -11,6 +12,14 @@
|
||||||
|
from re import Scanner
|
||||||
|
from weakref import proxy
|
||||||
|
|
||||||
|
+# some platforms lack working multiprocessing
|
||||||
|
+try:
|
||||||
|
+ import _multiprocessing
|
||||||
|
+except ImportError:
|
||||||
|
+ multiprocessing = None
|
||||||
|
+else:
|
||||||
|
+ import multiprocessing
|
||||||
|
+
|
||||||
|
# Misc tests from Tim Peters' re.doc
|
||||||
|
|
||||||
|
# WARNING: Don't change details in these tests if you don't know
|
||||||
|
@@ -2438,6 +2447,26 @@ def test_template_function_and_flag_is_deprecated(self):
|
||||||
|
self.assertTrue(template_re1.match('ahoy'))
|
||||||
|
self.assertFalse(template_re1.match('nope'))
|
||||||
|
|
||||||
|
+ @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing')
|
||||||
|
+ def test_regression_gh94675(self):
|
||||||
|
+ pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*'
|
||||||
|
+ r'((/[^/\[\n]*(([^\n]|(\[\n]*(]*)*\]))'
|
||||||
|
+ r'[^/\[]*)*/))((((//[^\n]*)?[\n])'
|
||||||
|
+ r'([\000-\040]|(/\*[^*]*\*+'
|
||||||
|
+ r'([^/*]\*+)*/))*)+(?=[^\000-\040);\]}]))')
|
||||||
|
+ input_js = '''a(function() {
|
||||||
|
+ ///////////////////////////////////////////////////////////////////
|
||||||
|
+ });'''
|
||||||
|
+ p = multiprocessing.Process(target=pattern.sub, args=('', input_js))
|
||||||
|
+ p.start()
|
||||||
|
+ p.join(SHORT_TIMEOUT)
|
||||||
|
+ try:
|
||||||
|
+ self.assertFalse(p.is_alive(), 'pattern.sub() timed out')
|
||||||
|
+ finally:
|
||||||
|
+ if p.is_alive():
|
||||||
|
+ p.terminate()
|
||||||
|
+ p.join()
|
||||||
|
+
|
||||||
|
|
||||||
|
def get_debug_out(pat):
|
||||||
|
with captured_stdout() as out:
|
||||||
|
diff --git a/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst b/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000000..d0005d9f60
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/Misc/NEWS.d/next/Tests/2022-07-08-12-22-00.gh-issue-94675.IiTs5f.rst
|
||||||
|
@@ -0,0 +1 @@
|
||||||
|
+Add a regression test for :mod:`re` exponentional slowdown when using rjsmin.
|
@ -14,10 +14,10 @@ URL: https://www.python.org/
|
|||||||
# WARNING When rebasing to a new Python version,
|
# WARNING When rebasing to a new Python version,
|
||||||
# remember to update the python3-docs package as well
|
# remember to update the python3-docs package as well
|
||||||
%global general_version %{pybasever}.0
|
%global general_version %{pybasever}.0
|
||||||
%global prerel b3
|
%global prerel b4
|
||||||
%global upstream_version %{general_version}%{?prerel}
|
%global upstream_version %{general_version}%{?prerel}
|
||||||
Version: %{general_version}%{?prerel:~%{prerel}}
|
Version: %{general_version}%{?prerel:~%{prerel}}
|
||||||
Release: 8%{?dist}
|
Release: 1%{?dist}
|
||||||
License: Python
|
License: Python
|
||||||
|
|
||||||
|
|
||||||
@ -322,39 +322,14 @@ Patch328: 00328-pyc-timestamp-invalidation-mode.patch
|
|||||||
# https://github.com/GrahamDumpleton/mod_wsgi/issues/730
|
# https://github.com/GrahamDumpleton/mod_wsgi/issues/730
|
||||||
Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch
|
Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch
|
||||||
|
|
||||||
# 00383 # b4e1d3233b9fbcd9a60370d0f29e65012bb9532d
|
# 00385 # d7319d51c72f0a6c1af5f32a728de1e3f8fea514
|
||||||
# gh-93442: Make C++ version of _Py_CAST work with 0/NULL
|
# gh-94675: Add a regression test for rjsmin re slowdown
|
||||||
#
|
#
|
||||||
# Add C++ overloads for _Py_CAST_impl() to handle 0/NULL. This will allow
|
# This tests a speed regression in the re module
|
||||||
# C++ extensions that pass 0 or NULL to macros using _Py_CAST() to
|
|
||||||
# continue to compile. Without this, you get an error like:
|
|
||||||
#
|
|
||||||
# invalid ‘static_cast’ from type ‘int’ to type ‘_object*’
|
|
||||||
#
|
|
||||||
# The modern way to use a NULL value in C++ is to use nullptr. However,
|
|
||||||
# we want to not break extensions that do things the old way.
|
|
||||||
Patch383: 00383-gh-93442-make-c-version-of-_py_cast-work-with-0-null.patch
|
|
||||||
|
|
||||||
# 00384 # 7c809258e34925560cc13a377b1c6d9c03e83207
|
|
||||||
# gh-94028: Clear and reset sqlite3 statements properly in cursor iternext (GH-94042)
|
|
||||||
Patch384: 00384-gh-94028-clear-and-reset-sqlite3-statements-properly-in-cursor-iternext-gh-94042.patch
|
|
||||||
|
|
||||||
# 00385 # 8696ca2373ef3d7595dfb62e2b63180621f40d5d
|
|
||||||
# gh-91404: Revert "bpo-23689: re module, fix memory leak..."
|
|
||||||
#
|
|
||||||
# This fixes a speed regression in the re module
|
|
||||||
# which prevented chromium from building in Fedora.
|
# which prevented chromium from building in Fedora.
|
||||||
#
|
#
|
||||||
# Revert "bpo-23689: re module, fix memory leak when a match is terminated by a signal or memory allocation failure"
|
# https://github.com/python/cpython/pull/94685
|
||||||
#
|
Patch385: 00385-gh-94675-add-a-regression-test-for-rjsmin-re-slowdown.patch
|
||||||
# This reverts commit 6e3eee5c11b539e9aab39cff783acf57838c355a.
|
|
||||||
#
|
|
||||||
# Manual fixups to increase the MAGIC number and to handle conflicts with
|
|
||||||
# a couple of changes that landed after that.
|
|
||||||
#
|
|
||||||
#
|
|
||||||
# gh-94675: Add a regression test for rjsmin re slowdown
|
|
||||||
Patch385: 00385-gh-91404-revert-bpo-23689-re-module-fix-memory-leak.patch
|
|
||||||
|
|
||||||
# (New patches go here ^^^)
|
# (New patches go here ^^^)
|
||||||
#
|
#
|
||||||
@ -1628,6 +1603,9 @@ CheckPython optimized
|
|||||||
# ======================================================
|
# ======================================================
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Mon Jul 11 2022 Miro Hrončok <mhroncok@redhat.com> - 3.11.0~b4-1
|
||||||
|
- Update to 3.11.0b4
|
||||||
|
|
||||||
* Fri Jul 08 2022 Miro Hrončok <mhroncok@redhat.com> - 3.11.0~b3-8
|
* Fri Jul 08 2022 Miro Hrončok <mhroncok@redhat.com> - 3.11.0~b3-8
|
||||||
- Finish bootstrap of the re module speed regression fix
|
- Finish bootstrap of the re module speed regression fix
|
||||||
|
|
||||||
|
4
sources
4
sources
@ -1,2 +1,2 @@
|
|||||||
SHA512 (Python-3.11.0b3.tar.xz) = 53c379311acc9d8ec645caea44f4b83b29fcbdacb7114eb7aac5a2f0229402eddb245abc8bd6665077e9c82ba0a821cc55ae7494d0667b84675248025e872498
|
SHA512 (Python-3.11.0b4.tar.xz) = 1530f4dcba6325ac8e9bb955604fc504a4306e3291b667d1d8490b78e1c50924d81ff8ab50d08c736e70d3e42788aabaa162179a77c03860ee8064f17f93bb09
|
||||||
SHA512 (Python-3.11.0b3.tar.xz.asc) = b51a02e47de73f4998effd15eb99d356ef69cd69521f208f50de1b8acf69d5cf9c8a2f774ec8bc5e0e75534fe59efe019cffcff39bc525aa4641dc9b47efb5ef
|
SHA512 (Python-3.11.0b4.tar.xz.asc) = 8f6c633911755767d351fc86e031df6cedd6091ca5ec0ef55331ac65d2ad532c71e568c173513a2bbb2235e5e3fc9dd77e1b7ed230df1f0d8bd6e23da470b1e2
|
||||||
|
Loading…
Reference in New Issue
Block a user