bpo-43574: Dont overallocate list literals

Lib/test/test_list.py

@@ -2,6 +2,7 @@
 from test import list_tests
 from test.support import cpython_only
 import pickle
+import struct
 import unittest
 
 class ListTest(list_tests.CommonTest):
@@ -196,6 +197,63 @@ def test_preallocation(self):
  self.assertEqual(iter_size, sys.getsizeof(list([0] * 10)))
  self.assertEqual(iter_size, sys.getsizeof(list(range(10))))
 
+ @cpython_only
+ def test_overallocation(self):
+ # bpo-33234: Don't overallocate when initialized from known lengths
+ # bpo-38373: Allows list over-allocation to be zero for some lengths
+ # bpo-43574: Don't overallocate for list-literals
+ sizeof = sys.getsizeof
+
+ # First handle empty list and empty list-literal cases. Should have no
+ # overallocation, including init from iterable of unknown length.
+ self.assertEqual(sizeof([]), sizeof(list()))
+ self.assertEqual(sizeof([]), sizeof(list(tuple())))
+ self.assertEqual(sizeof([]), sizeof(list(x for x in [])))
+
+ # Must use actual list-literals to test the overallocation behavior of
+ # compiled list-literals as well as those initialized from them.
+ test_literals = [
+ [1],
+ [1,2],
+ [1,2,3], # Literals of length > 2 are special-cased in compile
+ [1,2,3,4],
+ [1,2,3,4,5,6,7],
+ [1,2,3,4,5,6,7,8], # bpo-38373: Length 8 init won't over-alloc
+ [1,2,3,4,5,6,7,8,9],
+ ]
+
+ overalloc_amts = []
+ for literal in test_literals:
+ # Direct check that list-literals do not over-allocate, by
+ # calculating the total size of used pointers.
+ total_ptr_size = len(literal) * struct.calcsize('P')
+ self.assertEqual(sizeof(literal), sizeof([]) + total_ptr_size)
+
+ # Ensure that both list literals, and lists made from an iterable
+ # of known size, use the same amount of allocation.
+ self.assertEqual(sizeof(literal), sizeof(list(literal)))
+ self.assertEqual(sizeof(literal), sizeof(list(tuple(literal))))
+
+ # By contrast, confirm that non-empty lists initialized from an
+ # iterable where the length is unknown at the time of
+ # initialization, can be overallocated.
+ iterated_list = list(x for x in literal)
+ overalloc_amts.append(sizeof(iterated_list) - sizeof(literal))
+ self.assertGreaterEqual(sizeof(iterated_list), sizeof(literal))
+
+ # bpo-38373: initialized or grown lists are not always over-allocated.
+ # Confirm that over-allocation occurs at least some of the time.
+ self.assertEqual(True, any(x>0 for x in overalloc_amts))
+
+ # Empty lists should overallocate on initial append/insert (unlike
+ # list-literals)
+ l1 = []
+ l1.append(1)
+ self.assertGreater(sizeof(l1), sizeof([1]))
+ l2 = []
+ l2.insert(0, 1)
+ self.assertGreater(sizeof(l2), sizeof([1]))
+
  def test_count_index_remove_crashes(self):
  # bpo-38610: The count(), index(), and remove() methods were not
  # holding strong references to list elements while calling

Misc/NEWS.d/next/Core and Builtins/2021-03-21-18-51-30.bpo-43574.mteI-I.rst

@@ -0,0 +1,3 @@
+``list`` objects don't overallocate when starting empty and then extended, or
+when set to be empty. This effectively restores previous ``list`` memory
+behavior for lists initialized from literals.

Objects/listobject.c

@@ -58,25 +58,33 @@ list_resize(PyListObject *self, Py_ssize_t newsize)
  return 0;
  }
 
- /* This over-allocates proportional to the list size, making room
- * for additional growth. The over-allocation is mild, but is
- * enough to give linear-time amortized behavior over a long
- * sequence of appends() in the presence of a poorly-performing
- * system realloc().
- * Add padding to make the allocated size multiple of 4.
- * The growth pattern is: 0, 4, 8, 16, 24, 32, 40, 52, 64, 76, ...
- * Note: new_allocated won't overflow because the largest possible value
- * is PY_SSIZE_T_MAX * (9 / 8) + 6 which always fits in a size_t.
- */
- new_allocated = ((size_t)newsize + (newsize >> 3) + 6) & ~(size_t)3;
- /* Do not overallocate if the new size is closer to overallocated size
- * than to the old size.
- */
- if (newsize - Py_SIZE(self) > (Py_ssize_t)(new_allocated - newsize))
- new_allocated = ((size_t)newsize + 3) & ~(size_t)3;
+ if (newsize == 0 || (Py_SIZE(self) == 0 && newsize > 1)){
+ /* Don't overallocate empty lists that are extended by more than 1
+ * element. This helps ensure that list-literals aren't
+ * over-allocated, but still allows it for empty-list append/insert.
+ */
+ new_allocated = newsize;
+ }
+ else{
+ /* This over-allocates proportional to the list size, making room
+ * for additional growth. The over-allocation is mild, but is
+ * enough to give linear-time amortized behavior over a long
+ * sequence of appends() in the presence of a poorly-performing
+ * system realloc().
+ * Add padding to make the allocated size multiple of 4.
+ * The growth pattern is: 0, 4, 8, 16, 24, 32, 40, 52, 64, 76, ...
+ * Note: new_allocated won't overflow because the largest possible value
+ * is PY_SSIZE_T_MAX * (9 / 8) + 6 which always fits in a size_t.
+ */
+ new_allocated = ((size_t)newsize + (newsize >> 3) + 6) & ~(size_t)3;
+ /* Do not overallocate if the new size is closer to overallocated size
+ * than to the old size.
+ */
+ if (newsize - Py_SIZE(self) > (Py_ssize_t)(new_allocated - newsize)){
+ new_allocated = ((size_t)newsize + 3) & ~(size_t)3;
+ }
+ }
 
- if (newsize == 0)
- new_allocated = 0;
  num_allocated_bytes = new_allocated * sizeof(PyObject *);
  items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes);
  if (items == NULL){