bpo-36229: Avoid unnecessary copies for list, set, and bytearray ops.

Lib/test/test_list.py

@@ -163,8 +163,8 @@ def test_preallocation(self):
         iterable = [0] * 10
         iter_size = sys.getsizeof(iterable)
 
-        self.assertEqual(iter_size, sys.getsizeof(list([0] * 10)))
-        self.assertEqual(iter_size, sys.getsizeof(list(range(10))))
+        self.assertGreaterEqual(iter_size, sys.getsizeof(list([0] * 10)))
+        self.assertGreaterEqual(iter_size, sys.getsizeof(list(range(10))))
 
 if __name__ == "__main__":
     unittest.main()

Misc/ACKS

@@ -222,6 +222,7 @@ Ian Bruntlett
 Floris Bruynooghe
 Matt Bryant
 Stan Bubrouski
+Brandt Bucher
 Colm Buckley
 Erik de Bueger
 Jan-Hein Bührman

Misc/NEWS.d/next/Core and Builtins/2019-03-07-22-03-19.bpo-36229.gn6GOI.rst

@@ -0,0 +1,2 @@
+Improved performance for chained operations on built-in mutable collections.
+Patch by Brandt Bucher.

Objects/bytearrayobject.c

@@ -325,29 +325,12 @@ bytearray_iconcat(PyByteArrayObject *self, PyObject *other)
 }
 
 static PyObject *
-bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count)
+bytearray_concat(PyByteArrayObject *self, PyObject *other)
 {
-    PyByteArrayObject *result;
-    Py_ssize_t mysize;
-    Py_ssize_t size;
-
-    if (count < 0)
-        count = 0;
-    mysize = Py_SIZE(self);
-    if (count > 0 && mysize > PY_SSIZE_T_MAX / count)
-        return PyErr_NoMemory();
-    size = mysize * count;
-    result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size);
-    if (result != NULL && size != 0) {
-        if (mysize == 1)
-            memset(result->ob_bytes, self->ob_bytes[0], size);
-        else {
-            Py_ssize_t i;
-            for (i = 0; i < count; i++)
-                memcpy(result->ob_bytes + i*mysize, self->ob_bytes, mysize);
-        }
+    if (Py_REFCNT(self) == 1) {
+        return bytearray_iconcat(self, other);
     }
-    return (PyObject *)result;
+    return PyByteArray_Concat((PyObject *)self, other);
 }
 
 static PyObject *
@@ -379,6 +362,35 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count)
     return (PyObject *)self;
 }
 
+static PyObject *
+bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count)
+{
+    PyByteArrayObject *result;
+    Py_ssize_t mysize;
+    Py_ssize_t size;
+
+    if (count < 0)
+        count = 0;
+    mysize = Py_SIZE(self);
+    if (count > 0 && mysize > PY_SSIZE_T_MAX / count)
+        return PyErr_NoMemory();
+    if (Py_REFCNT(self) == 1) {
+        return bytearray_irepeat(self, count);
+    }
+    size = mysize * count;
+    result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size);
+    if (result != NULL && size != 0) {
+        if (mysize == 1)
+            memset(result->ob_bytes, self->ob_bytes[0], size);
+        else {
+            Py_ssize_t i;
+            for (i = 0; i < count; i++)
+                memcpy(result->ob_bytes + i*mysize, self->ob_bytes, mysize);
+        }
+    }
+    return (PyObject *)result;
+}
+
 static PyObject *
 bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i)
 {
@@ -2113,7 +2125,7 @@ bytearray_sizeof_impl(PyByteArrayObject *self)
 
 static PySequenceMethods bytearray_as_sequence = {
     (lenfunc)bytearray_length,              /* sq_length */
-    (binaryfunc)PyByteArray_Concat,         /* sq_concat */
+    (binaryfunc)bytearray_concat,           /* sq_concat */
     (ssizeargfunc)bytearray_repeat,         /* sq_repeat */
     (ssizeargfunc)bytearray_getitem,        /* sq_item */
     0,                                      /* sq_slice */

Objects/listobject.c

@@ -515,88 +515,6 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh)
     return list_slice((PyListObject *)a, ilow, ihigh);
 }
 
-static PyObject *
-list_concat(PyListObject *a, PyObject *bb)
-{
-    Py_ssize_t size;
-    Py_ssize_t i;
-    PyObject **src, **dest;
-    PyListObject *np;
-    if (!PyList_Check(bb)) {
-        PyErr_Format(PyExc_TypeError,
-                  "can only concatenate list (not \"%.200s\") to list",
-                  bb->ob_type->tp_name);
-        return NULL;
-    }
-#define b ((PyListObject *)bb)
-    if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b))
-        return PyErr_NoMemory();
-    size = Py_SIZE(a) + Py_SIZE(b);
-    np = (PyListObject *) list_new_prealloc(size);
-    if (np == NULL) {
-        return NULL;
-    }
-    src = a->ob_item;
-    dest = np->ob_item;
-    for (i = 0; i < Py_SIZE(a); i++) {
-        PyObject *v = src[i];
-        Py_INCREF(v);
-        dest[i] = v;
-    }
-    src = b->ob_item;
-    dest = np->ob_item + Py_SIZE(a);
-    for (i = 0; i < Py_SIZE(b); i++) {
-        PyObject *v = src[i];
-        Py_INCREF(v);
-        dest[i] = v;
-    }
-    Py_SIZE(np) = size;
-    return (PyObject *)np;
-#undef b
-}
-
-static PyObject *
-list_repeat(PyListObject *a, Py_ssize_t n)
-{
-    Py_ssize_t i, j;
-    Py_ssize_t size;
-    PyListObject *np;
-    PyObject **p, **items;
-    PyObject *elem;
-    if (n < 0)
-        n = 0;
-    if (n > 0 && Py_SIZE(a) > PY_SSIZE_T_MAX / n)
-        return PyErr_NoMemory();
-    size = Py_SIZE(a) * n;
-    if (size == 0)
-        return PyList_New(0);
-    np = (PyListObject *) list_new_prealloc(size);
-    if (np == NULL)
-        return NULL;
-
-    if (Py_SIZE(a) == 1) {
-        items = np->ob_item;
-        elem = a->ob_item[0];
-        for (i = 0; i < n; i++) {
-            items[i] = elem;
-            Py_INCREF(elem);
-        }
-    }
-    else {
-        p = np->ob_item;
-        items = a->ob_item;
-        for (i = 0; i < n; i++) {
-            for (j = 0; j < Py_SIZE(a); j++) {
-                *p = items[j];
-                Py_INCREF(*p);
-                p++;
-            }
-        }
-    }
-    Py_SIZE(np) = size;
-    return (PyObject *) np;
-}
-
 static int
 _list_clear(PyListObject *a)
 {
@@ -781,6 +699,51 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n)
     return (PyObject *)self;
 }
 
+static PyObject *
+list_repeat(PyListObject *a, Py_ssize_t n)
+{
+    Py_ssize_t i, j;
+    Py_ssize_t size;
+    PyListObject *np;
+    PyObject **p, **items;
+    PyObject *elem;
+    if (n < 0)
+        n = 0;
+    if (n > 0 && Py_SIZE(a) > PY_SSIZE_T_MAX / n)
+        return PyErr_NoMemory();
+    size = Py_SIZE(a) * n;
+    if (size == 0)
+        return PyList_New(0);
+    if (Py_REFCNT(a) == 1) {
+        return list_inplace_repeat(a, n);
+    }
+    np = (PyListObject *) list_new_prealloc(size);
+    if (np == NULL)
+        return NULL;
+
+    if (Py_SIZE(a) == 1) {
+        items = np->ob_item;
+        elem = a->ob_item[0];
+        for (i = 0; i < n; i++) {
+            items[i] = elem;
+            Py_INCREF(elem);
+        }
+    }
+    else {
+        p = np->ob_item;
+        items = a->ob_item;
+        for (i = 0; i < n; i++) {
+            for (j = 0; j < Py_SIZE(a); j++) {
+                *p = items[j];
+                Py_INCREF(*p);
+                p++;
+            }
+        }
+    }
+    Py_SIZE(np) = size;
+    return (PyObject *) np;
+}
+
 static int
 list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v)
 {
@@ -1006,6 +969,49 @@ list_inplace_concat(PyListObject *self, PyObject *other)
     return (PyObject *)self;
 }
 
+static PyObject *
+list_concat(PyListObject *a, PyObject *bb)
+{
+    Py_ssize_t size;
+    Py_ssize_t i;
+    PyObject **src, **dest;
+    PyListObject *np;
+    if (!PyList_Check(bb)) {
+        PyErr_Format(PyExc_TypeError,
+                  "can only concatenate list (not \"%.200s\") to list",
+                  bb->ob_type->tp_name);
+        return NULL;
+    }
+#define b ((PyListObject *)bb)
+    if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b))
+        return PyErr_NoMemory();
+    if (Py_REFCNT(a) == 1) {
+        return list_inplace_concat(a, bb);
+    }
+    size = Py_SIZE(a) + Py_SIZE(b);
+    np = (PyListObject *) list_new_prealloc(size);
+    if (np == NULL) {
+        return NULL;
+    }
+    src = a->ob_item;
+    dest = np->ob_item;
+    for (i = 0; i < Py_SIZE(a); i++) {
+        PyObject *v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    src = b->ob_item;
+    dest = np->ob_item + Py_SIZE(a);
+    for (i = 0; i < Py_SIZE(b); i++) {
+        PyObject *v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    Py_SIZE(np) = size;
+    return (PyObject *)np;
+#undef b
+}
+
 /*[clinic input]
 list.pop