From bb3bca43f1de7652b39c5cd80b77dbfaa5c137cf Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 29 Aug 2019 18:38:29 +0900 Subject: [PATCH 1/5] bpo-37587: _json: use _PyUnicodeWriter when scanning string --- Modules/_json.c | 66 ++++++++----------------------------------------- 1 file changed, 10 insertions(+), 56 deletions(-) diff --git a/Modules/_json.c b/Modules/_json.c index 112903ea577a263..cbe222fba048112 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -73,19 +73,6 @@ static PyMemberDef encoder_members[] = { {NULL} }; -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *sep = NULL; - if (sep == NULL) { - sep = PyUnicode_FromStringAndSize("", 0); - if (sep == NULL) - return NULL; - } - return PyUnicode_Join(sep, lst); -} - /* Forward decls */ static PyObject * @@ -385,21 +372,6 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { return tpl; } -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - Py_CLEAR(chunk); \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - static PyObject * scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) { @@ -417,12 +389,13 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next Py_ssize_t next /* = begin */; const void *buf; int kind; - PyObject *chunks = NULL; - PyObject *chunk = NULL; if (PyUnicode_READY(pystr) == -1) return 0; + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); @@ -455,12 +428,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next } /* Pick up this chunk if it's not zero length */ if (next != end) { - APPEND_OLD_CHUNK - chunk = PyUnicode_FromKindAndData( - kind, - (char*)buf + kind * end, - next - end); - if (chunk == NULL) { + if (_PyUnicodeWriter_WriteSubstring(&writer, pystr, end, next) < 0) { goto bail; } } @@ -473,6 +441,8 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; } + // Enable overallocating after simple case (e.g. "spam") is failed. + writer.overallocate = 1; c = PyUnicode_READ(kind, buf, next); if (c != 'u') { /* Non-unicode backslash escapes */ @@ -551,34 +521,18 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next end -= 6; } } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &c, 1); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = PyUnicode_FromStringAndSize("", 0); - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { + if (_PyUnicodeWriter_WriteChar(&writer, c) < 0) { goto bail; } - Py_CLEAR(chunks); } + rval = _PyUnicodeWriter_Finish(&writer); *next_end_ptr = end; return rval; + bail: *next_end_ptr = -1; - Py_XDECREF(chunks); - Py_XDECREF(chunk); + _PyUnicodeWriter_Dealloc(&writer); return NULL; } From b0bfd1bc3c6917cfc0bfb91eb631c8c58949fbdc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 29 Aug 2019 18:48:54 +0900 Subject: [PATCH 2/5] add NEWS entry --- .../next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst diff --git a/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst new file mode 100644 index 000000000000000..9f841a5a1795f48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst @@ -0,0 +1,2 @@ +Use ``_PyUnicodeWriter`` instead of list of unicode chunks in +``_json.scanstring``. Decoding long non-ASCII string is up to 4x faster. From f319a33475f742f7a8f0d1b03b20fb538d3528ac Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Sep 2019 19:34:45 +0900 Subject: [PATCH 3/5] Add fast path for simple case --- Modules/_json.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Modules/_json.c b/Modules/_json.c index cbe222fba048112..54ac605fd7ef422 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -395,6 +395,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next _PyUnicodeWriter writer; _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); @@ -422,10 +423,23 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next } c = d; } - if (!(c == '"' || c == '\\')) { + + if (c == '"') { + // Fast path for simple case. + if (writer.buffer == NULL) { + PyObject *ret = PyUnicode_Substring(pystr, end, next); + if (ret == NULL) { + goto bail; + } + *next_end_ptr = next + 1;; + return ret; + } + } + else if (c != '\\') { raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; } + /* Pick up this chunk if it's not zero length */ if (next != end) { if (_PyUnicodeWriter_WriteSubstring(&writer, pystr, end, next) < 0) { @@ -441,8 +455,6 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; } - // Enable overallocating after simple case (e.g. "spam") is failed. - writer.overallocate = 1; c = PyUnicode_READ(kind, buf, next); if (c != 'u') { /* Non-unicode backslash escapes */ From d3de551f515ef18fd1ae0df00f635dc2a1835053 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Sep 2019 19:54:21 +0900 Subject: [PATCH 4/5] update NEWS entry --- .../next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst index 9f841a5a1795f48..6ba03d0da1e683b 100644 --- a/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst +++ b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst @@ -1,2 +1,2 @@ -Use ``_PyUnicodeWriter`` instead of list of unicode chunks in -``_json.scanstring``. Decoding long non-ASCII string is up to 4x faster. +``_json.scanstring`` is now up to 4x faster when there are many backslash +escaped characters in the JSON string. From 09790a0cd64fb5544d39f1437cf034bdb2798aa7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Sep 2019 20:24:38 +0900 Subject: [PATCH 5/5] Update 2019-08-29-18-48-48.bpo-37587.N7TGTC.rst --- .../next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst index 6ba03d0da1e683b..92bebeea3021551 100644 --- a/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst +++ b/Misc/NEWS.d/next/Library/2019-08-29-18-48-48.bpo-37587.N7TGTC.rst @@ -1,2 +1,2 @@ -``_json.scanstring`` is now up to 4x faster when there are many backslash +``_json.scanstring`` is now up to 3x faster when there are many backslash escaped characters in the JSON string.