Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions Doc/library/stdtypes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4660,6 +4660,27 @@ copying.
>>> m[::2].tolist()
[-11111111, -33333333]

Slicing a multi-dimensional memoryview,
or indexing it with fewer integers than it has dimensions, also results in a sub-view.
A key may mix slices, integers and up to one ellipsis;
each integer drops the corresponding dimension,
and the ellipsis expands to full slices over the remaining dimensions.
An empty tuple or a bare ellipsis therefore selects the whole view::

>>> m = memoryview(bytearray(range(12))).cast('B', shape=[3, 4])
>>> m.tolist()
[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]
>>> m[1].tolist() # partial index: a row sub-view
[4, 5, 6, 7]
>>> m[1:3, 1:3].tolist() # rectangular sub-view
[[5, 6], [9, 10]]
>>> m[1, 1:3].tolist() # mix an index and a slice
[5, 6]
>>> m[..., 1].tolist() # ellipsis fills the leading dimensions
[1, 5, 9]
>>> m[...].tolist() # the whole view
[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]

If the underlying object is writable, the memoryview supports
one-dimensional slice assignment. Resizing is not allowed::

Expand All @@ -4681,6 +4702,20 @@ copying.
>>> data
bytearray(b'z1spam')

Multi-dimensional views and sub-views support slice assignment as well,
provided the right-hand side is a :term:`bytes-like object`
whose shape and format match the selected sub-view.
A one-dimensional sub-view accepts any matching bytes-like object,
while a multi-dimensional one needs an object of the same shape::

>>> m = memoryview(bytearray(range(12))).cast('B', shape=[3, 4])
>>> m[0] = b'ABCD' # a one-dimensional row sub-view
>>> m.tolist()
[[65, 66, 67, 68], [4, 5, 6, 7], [8, 9, 10, 11]]
>>> m[1:3, 1:3] = memoryview(bytes([10, 11, 12, 13])).cast('B', shape=[2, 2])
>>> m.tolist()
[[65, 66, 67, 68], [4, 10, 11, 7], [8, 12, 13, 11]]

One-dimensional memoryviews of :term:`hashable` (read-only) types with formats
'B', 'b' or 'c' are also hashable. The hash is defined as
``hash(m) == hash(m.tobytes())``::
Expand All @@ -4707,6 +4742,12 @@ copying.
.. versionchanged:: 3.14
memoryview is now a :term:`generic type`.

.. versionchanged:: next
Slicing a multi-dimensional memoryview,
or indexing it with a combination of integers, slices and an ellipsis,
now returns a sub-view, and such sub-views support slice assignment.
Previously these operations raised :exc:`NotImplementedError`.

:class:`memoryview` has several methods:

.. method:: __eq__(exporter)
Expand Down
6 changes: 6 additions & 0 deletions Doc/whatsnew/3.16.rst
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ New features
Other language changes
======================

* Slicing a multi-dimensional :class:`memoryview`, or indexing it with a
combination of integers, slices and an ellipsis, now returns a sub-view,
and such sub-views support slice assignment.
Previously these operations raised :exc:`NotImplementedError`.
(Contributed by Serhiy Storchaka in :gh:`58338`.)



New modules
Expand Down
52 changes: 40 additions & 12 deletions Lib/test/test_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1680,8 +1680,8 @@ def test_ndarray_slice_invalid(self):
self.assertRaises(TypeError, nd.__getitem__, ("@%$", slice(0,1,1)))
self.assertRaises(TypeError, nd.__getitem__, (slice(0,1,1), {}))

# memoryview: not implemented
self.assertRaises(NotImplementedError, mv.__getitem__,
# memoryview: multi-dimensional slicing validates each slice
self.assertRaises(ValueError, mv.__getitem__,
(slice(0,1,1), slice(0,1,0)))
self.assertRaises(TypeError, mv.__getitem__, "@%$")

Expand Down Expand Up @@ -1712,7 +1712,7 @@ def test_ndarray_slice_invalid(self):
mr = memoryview(xr)
self.assertRaises(ValueError, xl.__setitem__, slice(0,1,1), xr[7:8])
self.assertEqual(xl.tolist(), [[1,2,3,4], [5,6,7,8]])
self.assertRaises(NotImplementedError, ml.__setitem__, slice(0,1,1),
self.assertRaises(ValueError, ml.__setitem__, slice(0,1,1),
mr[7:8])

# differing shape
Expand Down Expand Up @@ -1789,6 +1789,22 @@ def test_ndarray_slice_multidim(self):
else:
self.assertEqual(ndsliced.tolist(), sliced)

def test_memoryview_suboffsets_subview(self):
# Sub-views of a buffer with suboffsets (PIL-style indirect buffer)
# are not implemented (gh-58338). A full index and a leading-dimension
# slice follow the suboffsets and still work.
nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_PIL|ND_WRITABLE)
m = memoryview(nd)
self.assertIsNotNone(m.suboffsets)
self.assertEqual(m[1, 2], 6)
self.assertEqual(m[1:3].tolist(), [[4, 5, 6, 7], [8, 9, 10, 11]])
self.assertRaises(NotImplementedError, m.__getitem__, 1)
self.assertRaises(NotImplementedError, m.__getitem__,
(slice(1, 3), slice(1, 3)))
self.assertRaises(NotImplementedError, m.__getitem__, (Ellipsis, 1))
self.assertRaises(NotImplementedError, m.__setitem__, 1,
b'\x00\x00\x00\x00')

def test_ndarray_slice_redundant_suboffsets(self):
shape_t = (2, 3, 5, 2)
ndim = len(shape_t)
Expand Down Expand Up @@ -2982,9 +2998,15 @@ def test_memoryview_index(self):
self.assertRaises(TypeError, m.__getitem__, (0, 0, 0))
self.assertRaises(TypeError, m.__getitem__, (0.0, 0.0))

# Not implemented: multidimensional sub-views
self.assertRaises(NotImplementedError, m.__getitem__, ())
self.assertRaises(NotImplementedError, m.__getitem__, 0)
# Empty-tuple or bare-ellipsis indexing of an N-D view returns the
# whole view as a sub-view.
self.assertEqual(m[()].tolist(), ex.tolist())
self.assertEqual(m[...].tolist(), ex.tolist())

# Integer indexing of an N-D view returns a sub-view.
self.assertEqual(m[0].tolist(), [0, 1, 2, 3])
self.assertEqual(m[-1].tolist(), [8, 9, 10, 11])
self.assertEqual([row.tolist() for row in m], ex.tolist())

def test_memoryview_assign(self):

Expand Down Expand Up @@ -3092,8 +3114,11 @@ def test_memoryview_assign(self):
self.assertRaises(TypeError, m.__setitem__, (0, 0, 0), 0)
self.assertRaises(TypeError, m.__setitem__, (0.0, 0.0), 0)

# Not implemented: multidimensional sub-views
self.assertRaises(NotImplementedError, m.__setitem__, 0, [2, 3])
# Sub-view assignment needs a bytes-like rvalue with a matching
# shape; a list is not a buffer.
self.assertRaises(TypeError, m.__setitem__, 0, [2, 3])
m[0] = m[1]
self.assertEqual(m[0].tolist(), m[1].tolist())

def test_memoryview_slice(self):

Expand All @@ -3105,16 +3130,19 @@ def test_memoryview_slice(self):
self.assertRaises(ValueError, m.__setitem__, slice(0,2,0),
bytearray([1,2]))

# 0-dim slicing (identity function)
self.assertRaises(NotImplementedError, m.__getitem__, ())
# empty tuple / bare ellipsis select the whole view
self.assertEqual(m[()].tolist(), list(range(12)))
self.assertEqual(m[...].tolist(), list(range(12)))

# multidimensional slices
ex = ndarray(list(range(12)), shape=[12], flags=ND_WRITABLE)
m = memoryview(ex)

self.assertRaises(NotImplementedError, m.__getitem__,
# too many indices for a 1-D view
self.assertRaises(TypeError, m.__getitem__,
(slice(0,2,1), slice(0,2,1)))
self.assertRaises(NotImplementedError, m.__setitem__,
# too many indices for a 1-D view (assignment)
self.assertRaises(TypeError, m.__setitem__,
(slice(0,2,1), slice(0,2,1)), bytearray([1,2]))

# invalid slice tuple
Expand Down
187 changes: 184 additions & 3 deletions Lib/test/test_memoryview.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,9 @@ def setitem(key, value):
self.assertRaises(TypeError, setitem, (0, slice(0,1,1)), b"a")
self.assertRaises(TypeError, setitem, (0,), b"a")
self.assertRaises(TypeError, setitem, "a", b"a")
# Not implemented: multidimensional slices
# Too many indices for a 1-D view.
slices = (slice(0,1,1), slice(0,1,2))
self.assertRaises(NotImplementedError, setitem, slices, b"a")
self.assertRaises(TypeError, setitem, slices, b"a")
# Trying to resize the memory object
exc = ValueError if m.format == 'c' else TypeError
self.assertRaises(exc, setitem, 0, b"")
Expand Down Expand Up @@ -473,7 +473,8 @@ def __len__(self):

# Variations on source objects for the buffer: bytes-like objects, then arrays
# with itemsize > 1.
# NOTE: support for multi-dimensional objects is unimplemented.
# These sources are all one-dimensional; multi-dimensional views are
# exercised in MultidimTest.

class BaseBytesMemoryTests(AbstractMemoryTests):
ro_type = bytes
Expand Down Expand Up @@ -688,6 +689,186 @@ class ArrayMemorySliceSliceTest(unittest.TestCase,
pass


class MultidimTest(unittest.TestCase):
# Multi-dimensional slicing and partial indexing return sub-views, and
# such sub-views support assignment (gh-58338).

def grid(self, shape=(3, 4)):
n = 1
for d in shape:
n *= d
return memoryview(bytearray(range(n))).cast('B', shape=list(shape))

def test_partial_index_subview(self):
m = self.grid()
row = m[1]
self.assertIsInstance(row, memoryview)
self.assertEqual(row.shape, (4,))
self.assertEqual(row.tolist(), [4, 5, 6, 7])
self.assertEqual(m[-1].tolist(), [8, 9, 10, 11])

def test_multidim_slice_subview(self):
m = self.grid()
sub = m[1:3, 1:3]
self.assertIsInstance(sub, memoryview)
self.assertEqual(sub.shape, (2, 2))
self.assertEqual(sub.tolist(), [[5, 6], [9, 10]])

def test_mixed_index_and_slice(self):
m = self.grid()
self.assertEqual(m[1, 1:3].tolist(), [5, 6])
self.assertEqual(m[0:2, 2].tolist(), [2, 6])

def test_ellipsis(self):
# Ellipsis combined with an integer index (drops a dimension) or a
# slice (keeps it).
m = self.grid()
self.assertEqual(m[..., 1].tolist(), [1, 5, 9]) # column
self.assertEqual(m[1, ...].tolist(), [4, 5, 6, 7]) # row
self.assertEqual(m[..., 1:3].tolist(), [[1, 2], [5, 6], [9, 10]])
self.assertEqual(m[0:2, ...].tolist(), [[0, 1, 2, 3], [4, 5, 6, 7]])

m = self.grid((2, 3, 4))
self.assertEqual(m[0, ..., 1].tolist(), [1, 5, 9])
self.assertEqual(m[..., 1].shape, (2, 3))

def test_full_index_returns_element(self):
m = self.grid()
self.assertEqual(m[1, 2], 6)

def test_whole_view(self):
# The empty tuple and a bare ellipsis both select the whole view.
m = self.grid()
for key in ((), ...):
with self.subTest(key=key):
sub = m[key]
self.assertIsInstance(sub, memoryview)
self.assertIsNot(sub, m)
self.assertEqual(sub.shape, (3, 4))
self.assertEqual(sub.tolist(), m.tolist())

def test_whole_view_assign(self):
m = self.grid()
m[...] = memoryview(bytes(range(100, 112))).cast('B', shape=[3, 4])
self.assertEqual(m[0].tolist(), [100, 101, 102, 103])
m[()] = memoryview(bytes(range(12))).cast('B', shape=[3, 4])
self.assertEqual(m[0].tolist(), [0, 1, 2, 3])

def test_ellipsis_assign(self):
# Ellipsis combined with an integer index (drops a dimension) or a
# slice (keeps it) on the write path.
m = self.grid()
m[..., 1] = memoryview(bytes([90, 91, 92])) # column, strided
self.assertEqual([m[i, 1] for i in range(3)], [90, 91, 92])

m = self.grid()
m[1, ...] = memoryview(bytes([90, 91, 92, 93])) # row
self.assertEqual(m[1].tolist(), [90, 91, 92, 93])

m = self.grid()
m[..., 1:3] = memoryview(bytes(range(90, 96))).cast('B', shape=[3, 2])
self.assertEqual(m.tolist(),
[[0, 90, 91, 3], [4, 92, 93, 7], [8, 94, 95, 11]])

m = self.grid((2, 3, 4))
m[0, ..., 1] = memoryview(bytes([90, 91, 92]))
self.assertEqual([m[0, j, 1] for j in range(3)], [90, 91, 92])

def test_step_slice(self):
m = self.grid((4, 4))
self.assertEqual(m[::2, ::2].tolist(), [[0, 2], [8, 10]])
self.assertEqual(m[::-1].tolist(),
[[12, 13, 14, 15], [8, 9, 10, 11],
[4, 5, 6, 7], [0, 1, 2, 3]])

def test_step_slice_assign(self):
m = self.grid()
m[::2, ::2] = memoryview(bytes([90, 91, 92, 93])).cast('B', shape=[2, 2])
self.assertEqual(m.tolist(),
[[90, 1, 91, 3], [4, 5, 6, 7], [92, 9, 93, 11]])

def test_empty_slice(self):
m = self.grid()
self.assertEqual(m[1:1].shape, (0, 4))
self.assertEqual(m[1:1].tolist(), [])
self.assertEqual(m[0:0, :].shape, (0, 4))

def test_index_out_of_bounds(self):
m = self.grid()
for key in (3, -4, (0, 4), (0, -5), (3, 0)):
with self.subTest(key=key):
self.assertRaises(IndexError, m.__getitem__, key)

def test_too_many_indices(self):
m = self.grid()
self.assertRaises(TypeError, m.__getitem__, (0, 0, 0))

def test_nonbyte_format(self):
# itemsize > 1: strides and offsets are in bytes, so sub-views must
# scale correctly by the element size.
buf = bytearray(struct.pack('12i', *range(12)))
m = memoryview(buf).cast('i', shape=[3, 4])
self.assertEqual(m.itemsize, struct.calcsize('i'))
self.assertEqual(m[1].tolist(), [4, 5, 6, 7])
self.assertEqual(m[1, 2], 6)
self.assertEqual(m[::2, ::2].tolist(), [[0, 2], [8, 10]])
# the rvalue must share the 'i' format, not just the bytes
m[1] = memoryview(struct.pack('4i', 40, 41, 42, 43)).cast('i')
self.assertEqual(m[1].tolist(), [40, 41, 42, 43])
self.assertRaises(ValueError, m.__setitem__, 1,
struct.pack('4i', 0, 0, 0, 0))

def test_assign_block_shape_mismatch(self):
m = self.grid()
self.assertRaises(ValueError, m.__setitem__, (slice(1, 3), slice(1, 3)),
memoryview(bytes([1, 2, 3, 4])))

def test_iter_yields_subviews(self):
m = self.grid()
self.assertEqual([row.tolist() for row in m],
[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]])

def test_three_dimensions(self):
m = self.grid((2, 3, 4))
self.assertEqual(m[1].shape, (3, 4))
self.assertEqual(m[1, 2].tolist(), [20, 21, 22, 23])
self.assertEqual(m[1, 2, 3], 23) # full index: a single element
self.assertEqual(m[0, 1:3, 1:3].tolist(), [[5, 6], [9, 10]])

def test_assign_row(self):
m = self.grid()
m[0] = bytes([100, 101, 102, 103])
self.assertEqual(m[0].tolist(), [100, 101, 102, 103])

def test_assign_block(self):
m = self.grid()
m[1:3, 1:3] = memoryview(bytes([10, 11, 12, 13])).cast('B', shape=[2, 2])
self.assertEqual(m.tolist(),
[[0, 1, 2, 3], [4, 10, 11, 7], [8, 12, 13, 11]])

def test_assign_element(self):
m = self.grid()
m[1, 2] = 99
self.assertEqual(m[1, 2], 99)

def test_assign_from_subview(self):
m = self.grid()
m[0] = m[2]
self.assertEqual(m[0].tolist(), m[2].tolist())

def test_assign_shape_mismatch(self):
m = self.grid()
self.assertRaises(ValueError, m.__setitem__, 0, bytes([1, 2]))

def test_assign_non_buffer(self):
m = self.grid()
self.assertRaises(TypeError, m.__setitem__, 0, [1, 2, 3, 4])

def test_assign_readonly(self):
m = memoryview(bytes(range(12))).cast('B', shape=[3, 4])
self.assertRaises(TypeError, m.__setitem__, 0, bytes([9, 9, 9, 9]))


class OtherTest(unittest.TestCase):
def test_ctypes_cast(self):
# Issue 15944: Allow all source formats when casting to bytes.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Slicing a multi-dimensional :class:`memoryview`, or indexing it with a
combination of integers, slices and an ellipsis, now returns a sub-view,
and such sub-views support slice assignment.
Previously these operations raised :exc:`NotImplementedError`.
Loading
Loading