From 8b28c90096dd36ff30de589933c1c1f2b6341a90 Mon Sep 17 00:00:00 2001 From: Marco Ribeiro Date: Tue, 7 Jul 2026 19:12:47 -0300 Subject: [PATCH] Add support for bypassing pre-packed data --- msgpack/__init__.py | 2 +- msgpack/_packer.pyx | 4 +++- msgpack/ext.py | 13 +++++++++++ msgpack/fallback.py | 5 +++- test/test_bypass.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 test/test_bypass.py diff --git a/msgpack/__init__.py b/msgpack/__init__.py index eeb85afa..fe8c4965 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,7 +2,7 @@ import os from .exceptions import * # noqa: F403 -from .ext import ExtType, Timestamp +from .ext import Bypass, ExtType, Timestamp version = (1, 2, 1) __version__ = "1.2.1" diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 277239d8..26cbdf97 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -8,7 +8,7 @@ from cpython.datetime cimport ( cdef ExtType cdef Timestamp -from .ext import ExtType, Timestamp +from .ext import ExtType, Timestamp, Bypass cdef extern from "Python.h": @@ -222,6 +222,8 @@ cdef class Packer: llval = o.seconds ulval = o.nanoseconds msgpack_pack_timestamp(&self.pk, llval, ulval) + elif type(o) is Bypass: + msgpack_pack_raw_body(&self.pk, o.data, len(o.data)) elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)): L = Py_SIZE(o) if L > ITEM_LIMIT: diff --git a/msgpack/ext.py b/msgpack/ext.py index abaaad90..41aebb12 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -18,6 +18,19 @@ def __new__(cls, code, data): return super().__new__(cls, code, data) +class Bypass: + """Bypass is a placeholder class to skip serialization and pass the bytes value as is.""" + + __slots__ = ["data"] + + def __init__(self, data): + if isinstance(data, bytes): + self.data = data + + else: + self.data = memoryview(data).tobytes() + + class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 824f59d5..91eb2401 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -38,7 +38,7 @@ def newlist_hint(size): from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError -from .ext import ExtType, Timestamp +from .ext import Bypass, ExtType, Timestamp EX_SKIP = 0 EX_CONSTRUCT = 1 @@ -779,6 +779,9 @@ def _pack( self._buffer.write(struct.pack("b", code)) self._buffer.write(data) return + if check(obj, Bypass): + self._buffer.write(obj.data) + return if check(obj, list_types): n = len(obj) self._pack_array_header(n) diff --git a/test/test_bypass.py b/test/test_bypass.py new file mode 100644 index 00000000..d0e4d89b --- /dev/null +++ b/test/test_bypass.py @@ -0,0 +1,57 @@ +from msgpack import Bypass, packb, unpackb + + +def test_roundtrip_bypassed_bytes(): + binary = b"\x00\x01\x02\x03\x04\x05" + + binary_packed = packb(binary) + binary_bypassed_packed = packb(Bypass(binary_packed)) + + assert binary_bypassed_packed == binary_packed + assert unpackb(binary_packed) == binary + + +def test_roundtrip_bypassed_object(): + obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}} + + obj_packed = packb(obj) + obj_bypassed_packed = packb(Bypass(obj_packed)) + + assert obj_bypassed_packed == obj_packed + assert unpackb(obj_bypassed_packed) == obj + + +def test_roundtrip_deep_bypassed_object(): + obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}} + obj_bypassed = {"key0": {"key1": {"key2": {"key3": [1, 2, Bypass(packb(3))]}}}} + + assert packb(obj) == packb(obj_bypassed) + assert unpackb(packb(obj)) == obj + + +def test_roundtrip_bypassed_object_with_cache(): + class SimpleCache: + def __init__(self, obj): + self.obj = obj + self.packed = None + + def get_packed(self): + if self.packed is None: + self.packed = packb(self.obj) + + return Bypass(self.packed) + + def default(obj): + if isinstance(obj, SimpleCache): + return obj.get_packed() + return obj + + obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}} + cache = SimpleCache(obj) + + obj_packed = packb(cache.obj, default=default) + cache.packed = obj_packed + obj_bypassed_packed = packb(Bypass(cache.packed)) + + assert obj_bypassed_packed == obj_packed + assert unpackb(obj_bypassed_packed) == cache.obj