From 8156a653785efca48edca655d4efab75b49fce70 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 24 May 2026 13:37:37 +0900 Subject: [PATCH 01/50] Add msgpack-jackson3 module for Jackson 3.x support --- .github/workflows/CI.yml | 15 +- build.sbt | 40 +- .../dataformat/benchmark/BenchmarkState.java | 32 + .../benchmark/MsgpackReadBenchmark.java | 37 + .../benchmark/MsgpackWriteBenchmark.java | 43 + .../dataformat/benchmark/NopOutputStream.java | 26 + .../dataformat/benchmark/model/Image.java | 21 + .../benchmark/model/MediaContent.java | 29 + .../dataformat/benchmark/model/MediaItem.java | 28 + .../benchmark/model/MediaItems.java | 33 + .../dataformat/benchmark/model/Player.java | 6 + .../dataformat/benchmark/model/Size.java | 6 + .../ExtensionTypeCustomDeserializers.java | 56 + .../jackson/dataformat/JsonArrayFormat.java | 32 + .../dataformat/MessagePackExtensionType.java | 83 ++ .../dataformat/MessagePackFactory.java | 253 ++++ .../dataformat/MessagePackFactoryBuilder.java | 114 ++ .../dataformat/MessagePackGenerator.java | 991 +++++++++++++ .../dataformat/MessagePackKeySerializer.java | 35 + .../jackson/dataformat/MessagePackMapper.java | 120 ++ .../jackson/dataformat/MessagePackParser.java | 626 +++++++++ .../dataformat/MessagePackReadContext.java | 196 +++ .../MessagePackSerializedString.java | 153 ++ .../MessagePackSerializerFactory.java | 42 + .../jackson/dataformat/PackageVersion.java | 30 + .../dataformat/TimestampExtensionModule.java | 89 ++ .../org/msgpack/jackson/dataformat/Tuple.java | 38 + .../ExampleOfTypeInformationSerDe.java | 171 +++ .../MessagePackDataformatForFieldIdTest.java | 135 ++ .../MessagePackDataformatForPojoTest.java | 150 ++ .../MessagePackDataformatTestBase.java | 289 ++++ .../dataformat/MessagePackFactoryTest.java | 198 +++ .../dataformat/MessagePackGeneratorTest.java | 1252 +++++++++++++++++ .../dataformat/MessagePackMapperTest.java | 117 ++ .../dataformat/MessagePackParserTest.java | 1132 +++++++++++++++ .../TimestampExtensionModuleTest.java | 217 +++ plans/preexisting-issues.md | 307 ++++ project/plugins.sbt | 1 + 38 files changed, 7138 insertions(+), 5 deletions(-) create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/ExtensionTypeCustomDeserializers.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactoryBuilder.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackKeySerializer.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackMapper.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/Tuple.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/ExampleOfTypeInformationSerDe.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForFieldIdTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatTestBase.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackFactoryTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackMapperTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/TimestampExtensionModuleTest.java create mode 100644 plans/preexisting-issues.md diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 26c11bfb..941e53ec 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,6 +28,7 @@ jobs: - 'project/build.properties' - 'msgpack-core/**' - 'msgpack-jackson/**' + - 'msgpack-jackson3/**' docs: - '**.md' - '**.txt' @@ -65,6 +66,16 @@ jobs: key: ${{ runner.os }}-jdk${{ matrix.java }}-${{ hashFiles('**/*.sbt') }} restore-keys: ${{ runner.os }}-jdk${{ matrix.java }}- - name: Test - run: ./sbt test + run: | + if [[ ${{ matrix.java }} -lt 17 ]]; then + ./sbt msgpack-core/test msgpack-jackson/test + else + ./sbt test + fi - name: Universal Buffer Test - run: ./sbt test -J-Dmsgpack.universal-buffer=true \ No newline at end of file + run: | + if [[ ${{ matrix.java }} -lt 17 ]]; then + ./sbt msgpack-core/test msgpack-jackson/test -J-Dmsgpack.universal-buffer=true + else + ./sbt test -J-Dmsgpack.universal-buffer=true + fi diff --git a/build.sbt b/build.sbt index 36d6f05e..f39cba7e 100644 --- a/build.sbt +++ b/build.sbt @@ -96,10 +96,16 @@ val buildSettings = Seq[Setting[?]]( Test / compile := ((Test / compile) dependsOn (Test / jcheckStyle)).value ) -val junitJupiter = "org.junit.jupiter" % "junit-jupiter" % "5.14.4" % "test" -val junitVintage = "org.junit.vintage" % "junit-vintage-engine" % "5.14.4" % "test" +val junitJupiter = "org.junit.jupiter" % "junit-jupiter" % "5.14.4" % "test" +val junitVintage = "org.junit.vintage" % "junit-vintage-engine" % "5.14.4" % "test" +val junitInterface = "com.github.sbt" % "junit-interface" % "0.13.3" % "test" // Project settings +val isJava17Plus: Boolean = { + val v = sys.props.getOrElse("java.specification.version", "1.8") + if (v.startsWith("1.")) false else scala.util.Try(v.toInt >= 17).getOrElse(false) +} + lazy val root = Project(id = "msgpack-java", base = file(".")) .settings( buildSettings, @@ -108,7 +114,10 @@ lazy val root = Project(id = "msgpack-java", base = file(".")) publish := {}, publishLocal := {} ) - .aggregate(msgpackCore, msgpackJackson) + .aggregate( + Seq[ProjectReference](msgpackCore, msgpackJackson) ++ + (if (isJava17Plus) Seq[ProjectReference](msgpackJackson3) else Nil): _* + ) lazy val msgpackCore = Project(id = "msgpack-core", base = file("msgpack-core")) .enablePlugins(SbtOsgi) @@ -170,3 +179,28 @@ lazy val msgpackJackson = Project(id = "msgpack-jackson", base = file("msgpack-j testOptions += Tests.Argument(TestFrameworks.JUnit, "-v") ) .dependsOn(msgpackCore) + +lazy val msgpackJackson3 = Project(id = "msgpack-jackson3", base = file("msgpack-jackson3")) + .enablePlugins(SbtOsgi, JmhPlugin) + .settings( + buildSettings, + name := "jackson-dataformat-msgpack3", + description := "Jackson 3.x extension that adds support for MessagePack", + OsgiKeys.bundleSymbolicName := "org.msgpack.msgpack-jackson3", + OsgiKeys.exportPackage := Seq("org.msgpack.jackson", "org.msgpack.jackson.dataformat"), + OsgiKeys.importPackage := Seq("!android.os", "!sun.*"), + Test / fork := true, + javacOptions := Seq("--release", "17"), + doc / javacOptions := Seq("--release", "17", "-Xdoclint:none"), + libraryDependencies ++= + Seq( + "tools.jackson.core" % "jackson-databind" % "3.1.2", + junitInterface + ), + testOptions += Tests.Argument(TestFrameworks.JUnit, "-v"), + Jmh / javaOptions ++= Seq( + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" + ) + ) + .dependsOn(msgpackCore) diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java new file mode 100644 index 00000000..63d43064 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java @@ -0,0 +1,32 @@ +package org.msgpack.jackson.dataformat.benchmark; + +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.msgpack.jackson.dataformat.MessagePackMapper; +import org.msgpack.jackson.dataformat.benchmark.model.MediaItem; +import org.msgpack.jackson.dataformat.benchmark.model.MediaItems; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +@State(Scope.Thread) +public class BenchmarkState +{ + public final ObjectMapper msgpackMapper = MessagePackMapper.builder(new MessagePackFactory()).build(); + public final ObjectMapper jsonMapper = JsonMapper.builder().build(); + + public final byte[] msgpackBytes; + public final byte[] jsonBytes; + + public BenchmarkState() + { + try { + MediaItem item = MediaItems.stdMediaItem(); + msgpackBytes = msgpackMapper.writeValueAsBytes(item); + jsonBytes = jsonMapper.writeValueAsBytes(item); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java new file mode 100644 index 00000000..8bb1ec9d --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java @@ -0,0 +1,37 @@ +package org.msgpack.jackson.dataformat.benchmark; + +import org.msgpack.jackson.dataformat.benchmark.model.MediaItem; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Fork(2) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class MsgpackReadBenchmark +{ + private final BenchmarkState state = new BenchmarkState(); + + @Benchmark + public Object readPojoMsgpack() throws Exception + { + return state.msgpackMapper.readValue(state.msgpackBytes, MediaItem.class); + } + + @Benchmark + public Object readPojoJson() throws Exception + { + return state.jsonMapper.readValue(state.jsonBytes, MediaItem.class); + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java new file mode 100644 index 00000000..b534b833 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java @@ -0,0 +1,43 @@ +package org.msgpack.jackson.dataformat.benchmark; + +import org.msgpack.jackson.dataformat.benchmark.model.MediaItem; +import org.msgpack.jackson.dataformat.benchmark.model.MediaItems; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Fork(2) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class MsgpackWriteBenchmark +{ + private final BenchmarkState state = new BenchmarkState(); + private final MediaItem item = MediaItems.stdMediaItem(); + + @Benchmark + public int writePojoMsgpack() throws Exception + { + NopOutputStream out = new NopOutputStream(); + state.msgpackMapper.writeValue(out, item); + return out.size(); + } + + @Benchmark + public int writePojoJson() throws Exception + { + NopOutputStream out = new NopOutputStream(); + state.jsonMapper.writeValue(out, item); + return out.size(); + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java new file mode 100644 index 00000000..e3b932c5 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java @@ -0,0 +1,26 @@ +package org.msgpack.jackson.dataformat.benchmark; + +import java.io.OutputStream; + +public class NopOutputStream + extends OutputStream +{ + private int size; + + @Override + public void write(int b) + { + size++; + } + + @Override + public void write(byte[] b, int off, int len) + { + size += len; + } + + public int size() + { + return size; + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java new file mode 100644 index 00000000..1d78c1e5 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java @@ -0,0 +1,21 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +public class Image +{ + public String uri; + public String title; + public int width; + public int height; + public Size size; + + public Image() {} + + public Image(String uri, String title, int width, int height, Size size) + { + this.uri = uri; + this.title = title; + this.width = width; + this.height = height; + this.size = size; + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java new file mode 100644 index 00000000..91e30846 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java @@ -0,0 +1,29 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +import java.util.ArrayList; +import java.util.List; + +public class MediaContent +{ + public String uri; + public String title; + public int width; + public int height; + public String format; + public long duration; + public long size; + public int bitrate; + public List persons; + public Player player; + public String copyright; + + public MediaContent() {} + + public void addPerson(String person) + { + if (persons == null) { + persons = new ArrayList<>(); + } + persons.add(person); + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java new file mode 100644 index 00000000..b3c3b5a2 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java @@ -0,0 +1,28 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.util.ArrayList; +import java.util.List; + +@JsonPropertyOrder({"content", "images"}) +public class MediaItem +{ + public MediaContent content; + public List images; + + public MediaItem() {} + + public MediaItem(MediaContent content) + { + this.content = content; + } + + public void addImage(Image image) + { + if (images == null) { + images = new ArrayList<>(); + } + images.add(image); + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java new file mode 100644 index 00000000..fa9fd41f --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java @@ -0,0 +1,33 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +public class MediaItems +{ + private static final MediaItem STD_MEDIA_ITEM; + + static { + MediaContent content = new MediaContent(); + content.uri = "http://javaone.com/keynote.mpg"; + content.title = "Javaone Keynote"; + content.width = 640; + content.height = 480; + content.format = "video/mpg4"; + content.duration = 18000000L; + content.size = 58982400L; + content.bitrate = 262144; + content.player = Player.JAVA; + content.copyright = "None"; + content.addPerson("Bill Gates"); + content.addPerson("Steve Jobs"); + + MediaItem item = new MediaItem(content); + item.addImage(new Image("http://javaone.com/keynote_large.jpg", "Javaone Keynote", 1024, 768, Size.LARGE)); + item.addImage(new Image("http://javaone.com/keynote_small.jpg", "Javaone Keynote", 320, 240, Size.SMALL)); + + STD_MEDIA_ITEM = item; + } + + public static MediaItem stdMediaItem() + { + return STD_MEDIA_ITEM; + } +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java new file mode 100644 index 00000000..58477386 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java @@ -0,0 +1,6 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +public enum Player +{ + JAVA, FLASH +} diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java new file mode 100644 index 00000000..43803743 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java @@ -0,0 +1,6 @@ +package org.msgpack.jackson.dataformat.benchmark.model; + +public enum Size +{ + SMALL, LARGE +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/ExtensionTypeCustomDeserializers.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/ExtensionTypeCustomDeserializers.java new file mode 100644 index 00000000..aa587975 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/ExtensionTypeCustomDeserializers.java @@ -0,0 +1,56 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ExtensionTypeCustomDeserializers +{ + private Map deserTable = new ConcurrentHashMap<>(); + + public ExtensionTypeCustomDeserializers() + { + } + + public ExtensionTypeCustomDeserializers(ExtensionTypeCustomDeserializers src) + { + this(); + this.deserTable.putAll(src.deserTable); + } + + public void addCustomDeser(byte type, final Deser deser) + { + deserTable.put(type, deser); + } + + public Deser getDeser(byte type) + { + return deserTable.get(type); + } + + public void clearEntries() + { + deserTable.clear(); + } + + public interface Deser + { + Object deserialize(byte[] data) + throws IOException; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java new file mode 100644 index 00000000..84be3f5c --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java @@ -0,0 +1,32 @@ +package org.msgpack.jackson.dataformat; + +import tools.jackson.databind.cfg.MapperConfig; +import tools.jackson.databind.introspect.Annotated; +import tools.jackson.databind.introspect.JacksonAnnotationIntrospector; + +import com.fasterxml.jackson.annotation.JsonFormat; + +import static com.fasterxml.jackson.annotation.JsonFormat.Shape.ARRAY; + +/** + * Provides the ability of serializing POJOs without their schema. + * Similar to @JsonFormat annotation with JsonFormat.Shape.ARRAY, but in a programmatic + * way. + * + * This also provides same behavior as msgpack-java 0.6.x serialization api. + */ +public class JsonArrayFormat extends JacksonAnnotationIntrospector +{ + private static final JsonFormat.Value ARRAY_FORMAT = new JsonFormat.Value().withShape(ARRAY); + + @Override + public JsonFormat.Value findFormat(MapperConfig config, Annotated ann) + { + JsonFormat.Value precedenceFormat = super.findFormat(config, ann); + if (precedenceFormat != null) { + return precedenceFormat; + } + + return ARRAY_FORMAT; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java new file mode 100644 index 00000000..92519356 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java @@ -0,0 +1,83 @@ +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.annotation.JsonSerialize; +import tools.jackson.databind.ser.std.StdSerializer; + +import java.util.Arrays; + +@JsonSerialize(using = MessagePackExtensionType.Serializer.class) +public class MessagePackExtensionType +{ + private final byte type; + private final byte[] data; + + public MessagePackExtensionType(byte type, byte[] data) + { + this.type = type; + this.data = data; + } + + public byte getType() + { + return type; + } + + public byte[] getData() + { + return data; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof MessagePackExtensionType)) { + return false; + } + + MessagePackExtensionType that = (MessagePackExtensionType) o; + + if (type != that.type) { + return false; + } + return Arrays.equals(data, that.data); + } + + @Override + public int hashCode() + { + int result = type; + result = 31 * result + Arrays.hashCode(data); + return result; + } + + @Override + public String toString() + { + return "MessagePackExtensionType(type=" + type + ", data.length=" + (data == null ? 0 : data.length) + ")"; + } + + public static class Serializer extends StdSerializer + { + public Serializer() + { + super(MessagePackExtensionType.class); + } + + @Override + public void serialize(MessagePackExtensionType value, JsonGenerator gen, SerializationContext serializers) + { + if (gen instanceof MessagePackGenerator) { + MessagePackGenerator msgpackGenerator = (MessagePackGenerator) gen; + msgpackGenerator.writeExtensionType(value); + } + else { + throw new IllegalStateException("'gen' is expected to be MessagePackGenerator but it's " + gen.getClass()); + } + } + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java new file mode 100644 index 00000000..78545a9e --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java @@ -0,0 +1,253 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.ErrorReportConfiguration; +import tools.jackson.core.FormatFeature; +import tools.jackson.core.FormatSchema; +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonParser; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.StreamReadConstraints; +import tools.jackson.core.StreamWriteConstraints; +import tools.jackson.core.TSFBuilder; +import tools.jackson.core.TokenStreamFactory; +import tools.jackson.core.Version; +import tools.jackson.core.base.BinaryTSFactory; +import tools.jackson.core.io.IOContext; +import org.msgpack.core.MessagePack; +import org.msgpack.core.annotations.VisibleForTesting; + +import org.msgpack.core.buffer.ArrayBufferInput; +import org.msgpack.core.buffer.MessageBufferInput; + +import java.io.DataInput; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class MessagePackFactory + extends BinaryTSFactory + implements java.io.Serializable +{ + private static final long serialVersionUID = 2578263992015504348L; + + private final MessagePack.PackerConfig packerConfig; + private boolean reuseResourceInGenerator = true; + private boolean reuseResourceInParser = true; + private boolean supportIntegerKeys = false; + private ExtensionTypeCustomDeserializers extTypeCustomDesers; + + public MessagePackFactory() + { + this(MessagePack.DEFAULT_PACKER_CONFIG); + } + + public MessagePackFactory(MessagePack.PackerConfig packerConfig) + { + super(StreamReadConstraints.defaults(), StreamWriteConstraints.defaults(), + ErrorReportConfiguration.defaults(), 0, 0); + this.packerConfig = packerConfig; + } + + public MessagePackFactory(MessagePackFactory src) + { + super(src); + this.packerConfig = src.packerConfig.clone(); + this.reuseResourceInGenerator = src.reuseResourceInGenerator; + this.reuseResourceInParser = src.reuseResourceInParser; + this.supportIntegerKeys = src.supportIntegerKeys; + if (src.extTypeCustomDesers != null) { + this.extTypeCustomDesers = new ExtensionTypeCustomDeserializers(src.extTypeCustomDesers); + } + } + + protected MessagePackFactory(MessagePackFactoryBuilder b) + { + super(b); + this.packerConfig = b.packerConfig().clone(); + this.reuseResourceInGenerator = b.reuseResourceInGenerator(); + this.reuseResourceInParser = b.reuseResourceInParser(); + this.supportIntegerKeys = b.supportIntegerKeys(); + this.extTypeCustomDesers = b.extTypeCustomDesers(); + } + + public MessagePackFactory setReuseResourceInGenerator(boolean reuseResourceInGenerator) + { + this.reuseResourceInGenerator = reuseResourceInGenerator; + return this; + } + + public MessagePackFactory setReuseResourceInParser(boolean reuseResourceInParser) + { + this.reuseResourceInParser = reuseResourceInParser; + return this; + } + + public MessagePackFactory setSupportIntegerKeys(boolean supportIntegerKeys) + { + this.supportIntegerKeys = supportIntegerKeys; + return this; + } + + public MessagePackFactory setExtTypeCustomDesers(ExtensionTypeCustomDeserializers extTypeCustomDesers) + { + this.extTypeCustomDesers = extTypeCustomDesers; + return this; + } + + @Override + protected JsonParser _createParser(ObjectReadContext readCtxt, IOContext ioCtxt, + InputStream in) throws JacksonException + { + try { + MessagePackParser parser = new MessagePackParser(readCtxt, ioCtxt, + readCtxt.getStreamReadFeatures(_streamReadFeatures), in, reuseResourceInParser); + if (extTypeCustomDesers != null) { + parser.setExtensionTypeCustomDeserializers(extTypeCustomDesers); + } + return parser; + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + @Override + protected JsonParser _createParser(ObjectReadContext readCtxt, IOContext ioCtxt, + byte[] data, int offset, int len) throws JacksonException + { + try { + MessageBufferInput input = new ArrayBufferInput(data, offset, len); + MessagePackParser parser = new MessagePackParser(readCtxt, ioCtxt, + readCtxt.getStreamReadFeatures(_streamReadFeatures), input, data, reuseResourceInParser); + if (extTypeCustomDesers != null) { + parser.setExtensionTypeCustomDeserializers(extTypeCustomDesers); + } + return parser; + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + @Override + protected JsonParser _createParser(ObjectReadContext readCtxt, IOContext ioCtxt, + DataInput input) throws JacksonException + { + return _unsupported(); + } + + @Override + protected JsonGenerator _createGenerator(ObjectWriteContext writeCtxt, IOContext ioCtxt, + OutputStream out) throws JacksonException + { + try { + return new MessagePackGenerator(writeCtxt, ioCtxt, + writeCtxt.getStreamWriteFeatures(_streamWriteFeatures), + out, packerConfig, reuseResourceInGenerator, supportIntegerKeys); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + @Override + public TokenStreamFactory copy() + { + return new MessagePackFactory(this); + } + + @Override + public TokenStreamFactory snapshot() + { + return copy(); + } + + @Override + public TSFBuilder rebuild() + { + return new MessagePackFactoryBuilder(this); + } + + @Override + public Version version() + { + return PackageVersion.VERSION; + } + + @VisibleForTesting + MessagePack.PackerConfig getPackerConfig() + { + return packerConfig; + } + + @VisibleForTesting + boolean isReuseResourceInGenerator() + { + return reuseResourceInGenerator; + } + + @VisibleForTesting + boolean isReuseResourceInParser() + { + return reuseResourceInParser; + } + + @VisibleForTesting + boolean isSupportIntegerKeys() + { + return supportIntegerKeys; + } + + @VisibleForTesting + ExtensionTypeCustomDeserializers getExtTypeCustomDesers() + { + return extTypeCustomDesers; + } + + @Override + public String getFormatName() + { + return "msgpack"; + } + + @Override + public boolean canParseAsync() + { + return false; + } + + @Override + public boolean canUseSchema(FormatSchema schema) + { + return false; + } + + @Override + public Class getFormatReadFeatureType() + { + return null; + } + + @Override + public Class getFormatWriteFeatureType() + { + return null; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactoryBuilder.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactoryBuilder.java new file mode 100644 index 00000000..59f80991 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactoryBuilder.java @@ -0,0 +1,114 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.ErrorReportConfiguration; +import tools.jackson.core.StreamReadConstraints; +import tools.jackson.core.StreamWriteConstraints; +import tools.jackson.core.base.DecorableTSFactory; +import org.msgpack.core.MessagePack; + +public class MessagePackFactoryBuilder + extends DecorableTSFactory.DecorableTSFBuilder +{ + private MessagePack.PackerConfig packerConfig; + private boolean reuseResourceInGenerator; + private boolean reuseResourceInParser; + private boolean supportIntegerKeys; + private ExtensionTypeCustomDeserializers extTypeCustomDesers; + + public MessagePackFactoryBuilder() + { + super(StreamReadConstraints.defaults(), StreamWriteConstraints.defaults(), + ErrorReportConfiguration.defaults(), 0, 0); + this.packerConfig = MessagePack.DEFAULT_PACKER_CONFIG; + this.reuseResourceInGenerator = true; + this.reuseResourceInParser = true; + this.supportIntegerKeys = false; + } + + public MessagePackFactoryBuilder(MessagePackFactory base) + { + super(base); + this.packerConfig = base.getPackerConfig().clone(); + this.reuseResourceInGenerator = base.isReuseResourceInGenerator(); + this.reuseResourceInParser = base.isReuseResourceInParser(); + this.supportIntegerKeys = base.isSupportIntegerKeys(); + ExtensionTypeCustomDeserializers srcDesers = base.getExtTypeCustomDesers(); + this.extTypeCustomDesers = srcDesers == null ? null : new ExtensionTypeCustomDeserializers(srcDesers); + } + + public MessagePackFactoryBuilder packerConfig(MessagePack.PackerConfig config) + { + this.packerConfig = config; + return this; + } + + public MessagePackFactoryBuilder reuseResourceInGenerator(boolean v) + { + this.reuseResourceInGenerator = v; + return this; + } + + public MessagePackFactoryBuilder reuseResourceInParser(boolean v) + { + this.reuseResourceInParser = v; + return this; + } + + public MessagePackFactoryBuilder supportIntegerKeys(boolean v) + { + this.supportIntegerKeys = v; + return this; + } + + public MessagePackFactoryBuilder extTypeCustomDesers(ExtensionTypeCustomDeserializers desers) + { + this.extTypeCustomDesers = desers; + return this; + } + + public MessagePack.PackerConfig packerConfig() + { + return packerConfig; + } + + public boolean reuseResourceInGenerator() + { + return reuseResourceInGenerator; + } + + public boolean reuseResourceInParser() + { + return reuseResourceInParser; + } + + public boolean supportIntegerKeys() + { + return supportIntegerKeys; + } + + public ExtensionTypeCustomDeserializers extTypeCustomDesers() + { + return extTypeCustomDesers; + } + + @Override + public MessagePackFactory build() + { + return new MessagePackFactory(this); + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java new file mode 100644 index 00000000..0527c0c3 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -0,0 +1,991 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.Base64Variant; +import tools.jackson.core.JacksonException; +import tools.jackson.core.util.JacksonFeatureSet; +import tools.jackson.core.util.SimpleStreamWriteContext; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.SerializableString; +import tools.jackson.core.StreamWriteCapability; +import tools.jackson.core.StreamWriteFeature; +import tools.jackson.core.TokenStreamContext; +import tools.jackson.core.base.GeneratorBase; +import tools.jackson.core.io.IOContext; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.core.buffer.MessageBufferOutput; +import org.msgpack.core.buffer.OutputStreamBufferOutput; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class MessagePackGenerator + extends GeneratorBase +{ + private static final int IN_ROOT = 0; + private static final int IN_OBJECT = 1; + private static final int IN_ARRAY = 2; + private final MessagePacker messagePacker; + private static final ThreadLocal messageBufferOutputHolder = new ThreadLocal<>(); + private final OutputStream output; + private final MessagePack.PackerConfig packerConfig; + private final boolean supportIntegerKeys; + + private int currentParentElementIndex = -1; + private int currentState = IN_ROOT; + private final List nodes; + private boolean isElementsClosed = false; + private SimpleStreamWriteContext writeContext; + + private static final class RawUtf8String + { + public final byte[] bytes; + + public RawUtf8String(byte[] bytes) + { + this.bytes = bytes; + } + } + + private abstract static class Node + { + final int parentIndex; + + public Node(int parentIndex) + { + this.parentIndex = parentIndex; + } + + abstract void incrementChildCount(); + + abstract int currentStateAsParent(); + } + + private abstract static class NodeContainer extends Node + { + int childCount; + + public NodeContainer(int parentIndex) + { + super(parentIndex); + } + + @Override + void incrementChildCount() + { + childCount++; + } + } + + private static final class NodeArray extends NodeContainer + { + public NodeArray(int parentIndex) + { + super(parentIndex); + } + + @Override + int currentStateAsParent() + { + return IN_ARRAY; + } + } + + private static final class NodeObject extends NodeContainer + { + public NodeObject(int parentIndex) + { + super(parentIndex); + } + + @Override + int currentStateAsParent() + { + return IN_OBJECT; + } + } + + private static final class NodeEntryInArray extends Node + { + final Object value; + + public NodeEntryInArray(int parentIndex, Object value) + { + super(parentIndex); + this.value = value; + } + + @Override + void incrementChildCount() + { + throw new UnsupportedOperationException(); + } + + @Override + int currentStateAsParent() + { + throw new UnsupportedOperationException(); + } + } + + private static final class NodeEntryInObject extends Node + { + final Object key; + Object value; + + public NodeEntryInObject(int parentIndex, Object key) + { + super(parentIndex); + this.key = key; + } + + @Override + void incrementChildCount() + { + assert value instanceof NodeContainer; + ((NodeContainer) value).childCount++; + } + + @Override + int currentStateAsParent() + { + if (value instanceof NodeObject) { + return IN_OBJECT; + } + else if (value instanceof NodeArray) { + return IN_ARRAY; + } + else { + throw new AssertionError(); + } + } + } + + // Internal constructor for nested serialization. + private MessagePackGenerator( + ObjectWriteContext writeCtxt, + IOContext ioCtxt, + int streamWriteFeatures, + OutputStream out, + MessagePack.PackerConfig packerConfig, + boolean supportIntegerKeys) + { + super(writeCtxt, ioCtxt, streamWriteFeatures); + this.output = out; + this.messagePacker = packerConfig.newPacker(out); + this.packerConfig = packerConfig; + this.nodes = new ArrayList<>(); + this.supportIntegerKeys = supportIntegerKeys; + this.writeContext = SimpleStreamWriteContext.createRootContext(null); + } + + public MessagePackGenerator( + ObjectWriteContext writeCtxt, + IOContext ioCtxt, + int streamWriteFeatures, + OutputStream out, + MessagePack.PackerConfig packerConfig, + boolean reuseResourceInGenerator, + boolean supportIntegerKeys) + throws IOException + { + super(writeCtxt, ioCtxt, streamWriteFeatures); + this.output = out; + this.messagePacker = packerConfig.newPacker(getMessageBufferOutputForOutputStream(out, reuseResourceInGenerator)); + this.packerConfig = packerConfig; + this.nodes = new ArrayList<>(); + this.supportIntegerKeys = supportIntegerKeys; + this.writeContext = SimpleStreamWriteContext.createRootContext(null); + } + + private MessageBufferOutput getMessageBufferOutputForOutputStream( + OutputStream out, + boolean reuseResourceInGenerator) + throws IOException + { + OutputStreamBufferOutput messageBufferOutput; + if (reuseResourceInGenerator) { + messageBufferOutput = messageBufferOutputHolder.get(); + if (messageBufferOutput == null) { + messageBufferOutput = new OutputStreamBufferOutput(out); + messageBufferOutputHolder.set(messageBufferOutput); + } + else { + messageBufferOutput.reset(out); + } + } + else { + messageBufferOutput = new OutputStreamBufferOutput(out); + } + return messageBufferOutput; + } + + private String currentStateStr() + { + switch (currentState) { + case IN_OBJECT: + return "IN_OBJECT"; + case IN_ARRAY: + return "IN_ARRAY"; + default: + return "IN_ROOT"; + } + } + + @Override + public JsonGenerator writeStartArray() throws JacksonException + { + return writeStartArray(null); + } + + @Override + public JsonGenerator writeStartArray(Object currentValue) throws JacksonException + { + return writeStartArray(currentValue, -1); + } + + @Override + public JsonGenerator writeStartArray(Object currentValue, int size) throws JacksonException + { + _verifyValueWrite("start an array"); + writeContext = writeContext.createChildArrayContext(currentValue); + if (currentState == IN_OBJECT) { + Node node = nodes.get(nodes.size() - 1); + assert node instanceof NodeEntryInObject; + NodeEntryInObject nodeEntryInObject = (NodeEntryInObject) node; + nodeEntryInObject.value = new NodeArray(currentParentElementIndex); + } + else { + nodes.add(new NodeArray(currentParentElementIndex)); + } + currentParentElementIndex = nodes.size() - 1; + currentState = IN_ARRAY; + return this; + } + + @Override + public JsonGenerator writeEndArray() throws JacksonException + { + if (currentState != IN_ARRAY) { + _reportError("Current context not an array but " + currentStateStr()); + } + endCurrentContainer(); + return this; + } + + @Override + public JsonGenerator writeStartObject() throws JacksonException + { + return writeStartObject(null); + } + + @Override + public JsonGenerator writeStartObject(Object currentValue) throws JacksonException + { + return writeStartObject(currentValue, -1); + } + + @Override + public JsonGenerator writeStartObject(Object forValue, int size) throws JacksonException + { + _verifyValueWrite("start an object"); + writeContext = writeContext.createChildObjectContext(forValue); + if (currentState == IN_OBJECT) { + Node node = nodes.get(nodes.size() - 1); + assert node instanceof NodeEntryInObject; + NodeEntryInObject nodeEntryInObject = (NodeEntryInObject) node; + nodeEntryInObject.value = new NodeObject(currentParentElementIndex); + } + else { + nodes.add(new NodeObject(currentParentElementIndex)); + } + currentParentElementIndex = nodes.size() - 1; + currentState = IN_OBJECT; + return this; + } + + @Override + public JsonGenerator writeEndObject() throws JacksonException + { + if (currentState != IN_OBJECT) { + _reportError("Current context not an object but " + currentStateStr()); + } + endCurrentContainer(); + return this; + } + + private void endCurrentContainer() + { + writeContext = writeContext.clearAndGetParent(); + Node parent = nodes.get(currentParentElementIndex); + if (currentParentElementIndex == 0) { + isElementsClosed = true; + currentParentElementIndex = parent.parentIndex; + currentState = IN_ROOT; + return; + } + + currentParentElementIndex = parent.parentIndex; + assert currentParentElementIndex >= 0; + Node currentParent = nodes.get(currentParentElementIndex); + currentParent.incrementChildCount(); + currentState = currentParent.currentStateAsParent(); + } + + private void packNonContainer(Object v) + throws IOException + { + MessagePacker messagePacker = getMessagePacker(); + if (v instanceof String) { + messagePacker.packString((String) v); + } + else if (v instanceof RawUtf8String) { + byte[] bytes = ((RawUtf8String) v).bytes; + messagePacker.packRawStringHeader(bytes.length); + messagePacker.writePayload(bytes); + } + else if (v instanceof Integer) { + messagePacker.packInt((Integer) v); + } + else if (v == null) { + messagePacker.packNil(); + } + else if (v instanceof Float) { + messagePacker.packFloat((Float) v); + } + else if (v instanceof Long) { + messagePacker.packLong((Long) v); + } + else if (v instanceof Double) { + messagePacker.packDouble((Double) v); + } + else if (v instanceof BigInteger) { + messagePacker.packBigInteger((BigInteger) v); + } + else if (v instanceof BigDecimal) { + packBigDecimal((BigDecimal) v); + } + else if (v instanceof Boolean) { + messagePacker.packBoolean((Boolean) v); + } + else if (v instanceof ByteBuffer) { + ByteBuffer bb = (ByteBuffer) v; + int len = bb.remaining(); + if (bb.hasArray()) { + messagePacker.packBinaryHeader(len); + messagePacker.writePayload(bb.array(), bb.arrayOffset() + bb.position(), len); + } + else { + byte[] data = new byte[len]; + bb.duplicate().get(data); + messagePacker.packBinaryHeader(len); + messagePacker.addPayload(data); + } + } + else if (v instanceof MessagePackExtensionType) { + MessagePackExtensionType extensionType = (MessagePackExtensionType) v; + byte[] extData = extensionType.getData(); + messagePacker.packExtensionTypeHeader(extensionType.getType(), extData.length); + messagePacker.writePayload(extData); + } + else { + messagePacker.flush(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + MessagePackGenerator messagePackGenerator = new MessagePackGenerator( + objectWriteContext(), _ioContext, _streamWriteFeatures, + outputStream, packerConfig, supportIntegerKeys); + objectWriteContext().writeValue(messagePackGenerator, v); + messagePackGenerator.flush(); + output.write(outputStream.toByteArray()); + } + } + + private void packBigDecimal(BigDecimal decimal) + throws IOException + { + MessagePacker messagePacker = getMessagePacker(); + boolean failedToPackAsBI = false; + try { + BigInteger integer = decimal.toBigIntegerExact(); + messagePacker.packBigInteger(integer); + } + catch (ArithmeticException | IllegalArgumentException e) { + failedToPackAsBI = true; + } + + if (failedToPackAsBI) { + double doubleValue = decimal.doubleValue(); + if (decimal.compareTo(BigDecimal.valueOf(doubleValue)) != 0) { + throw new IllegalArgumentException("MessagePack cannot serialize a BigDecimal that can't be represented as double. " + decimal); + } + messagePacker.packDouble(doubleValue); + } + } + + private void packObject(NodeObject container) + throws IOException + { + MessagePacker messagePacker = getMessagePacker(); + messagePacker.packMapHeader(container.childCount); + } + + private void packArray(NodeArray container) + throws IOException + { + MessagePacker messagePacker = getMessagePacker(); + messagePacker.packArrayHeader(container.childCount); + } + + private void addKeyNode(Object key) + { + if (currentState != IN_OBJECT) { + throw new IllegalStateException(); + } + Node node = new NodeEntryInObject(currentParentElementIndex, key); + nodes.add(node); + } + + private void addValueNode(Object value) throws IOException + { + if (!writeContext.writeValue()) { + _reportError("Cannot write value: expecting a property name in Object context"); + } + switch (currentState) { + case IN_OBJECT: { + Node node = nodes.get(nodes.size() - 1); + assert node instanceof NodeEntryInObject; + NodeEntryInObject nodeEntryInObject = (NodeEntryInObject) node; + nodeEntryInObject.value = value; + nodes.get(node.parentIndex).incrementChildCount(); + break; + } + case IN_ARRAY: { + Node node = new NodeEntryInArray(currentParentElementIndex, value); + nodes.add(node); + nodes.get(node.parentIndex).incrementChildCount(); + break; + } + default: + packNonContainer(value); + flushMessagePacker(); + break; + } + } + + private void writeCharArrayTextValue(char[] text, int offset, int len) throws IOException + { + addValueNode(new String(text, offset, len)); + } + + private void writeByteArrayTextValue(byte[] text, int offset, int len) throws IOException + { + byte[] slice = new byte[len]; + System.arraycopy(text, offset, slice, 0, len); + addValueNode(new RawUtf8String(slice)); + } + + @Override + public JsonGenerator writePropertyId(long id) throws JacksonException + { + if (this.supportIntegerKeys) { + if (!writeContext.writeName(String.valueOf(id))) { + _reportError("Cannot write property name, not in Object context"); + } + addKeyNode(id); + } + else { + writeName(String.valueOf(id)); + } + return this; + } + + @Override + public JacksonFeatureSet streamWriteCapabilities() + { + return DEFAULT_BINARY_WRITE_CAPABILITIES; + } + + @Override + public JsonGenerator writeName(String name) throws JacksonException + { + if (!writeContext.writeName(name)) { + _reportError("Cannot write property name, not in Object context"); + } + addKeyNode(name); + return this; + } + + @Override + public JsonGenerator writeName(SerializableString name) throws JacksonException + { + if (name instanceof MessagePackSerializedString) { + if (!writeContext.writeName(name.getValue())) { + _reportError("Cannot write property name, not in Object context"); + } + addKeyNode(((MessagePackSerializedString) name).getRawValue()); + } + else { + writeName(name.getValue()); + } + return this; + } + + @Override + public JsonGenerator writeString(String text) throws JacksonException + { + try { + addValueNode(text); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeString(char[] text, int offset, int len) throws JacksonException + { + try { + writeCharArrayTextValue(text, offset, len); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeString(Reader reader, int len) throws JacksonException + { + try { + if (len < 0) { + StringBuilder sb = new StringBuilder(); + char[] tmpBuf = new char[1024]; + int read; + while ((read = reader.read(tmpBuf)) >= 0) { + sb.append(tmpBuf, 0, read); + } + addValueNode(sb.toString()); + } + else { + int chunkSize = Math.min(len, 8192); + StringBuilder sb = new StringBuilder(chunkSize); + char[] tmpBuf = new char[chunkSize]; + int remaining = len; + while (remaining > 0) { + int read = reader.read(tmpBuf, 0, Math.min(remaining, tmpBuf.length)); + if (read < 0) { + break; + } + sb.append(tmpBuf, 0, read); + remaining -= read; + } + addValueNode(sb.toString()); + } + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeString(SerializableString text) throws JacksonException + { + return writeString(text.getValue()); + } + + @Override + public JsonGenerator writeRawUTF8String(byte[] text, int offset, int length) throws JacksonException + { + try { + writeByteArrayTextValue(text, offset, length); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeUTF8String(byte[] text, int offset, int length) throws JacksonException + { + try { + writeByteArrayTextValue(text, offset, length); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeRaw(String text) throws JacksonException + { + try { + addValueNode(text); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeRaw(String text, int offset, int len) throws JacksonException + { + try { + addValueNode(text.substring(offset, offset + len)); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeRaw(char[] text, int offset, int len) throws JacksonException + { + try { + writeCharArrayTextValue(text, offset, len); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeRaw(char c) throws JacksonException + { + try { + writeCharArrayTextValue(new char[] { c }, 0, 1); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeBinary(Base64Variant b64variant, byte[] data, int offset, int len) throws JacksonException + { + try { + addValueNode(ByteBuffer.wrap(data, offset, len)); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(short v) throws JacksonException + { + return writeNumber((int) v); + } + + @Override + public JsonGenerator writeNumber(int v) throws JacksonException + { + try { + addValueNode(v); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(long v) throws JacksonException + { + try { + addValueNode(v); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(BigInteger v) throws JacksonException + { + try { + addValueNode(v); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(double d) throws JacksonException + { + try { + addValueNode(d); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(float f) throws JacksonException + { + try { + addValueNode(f); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(BigDecimal dec) throws JacksonException + { + try { + addValueNode(dec); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNumber(String encodedValue) throws JacksonException + { + try { + try { + long l = Long.parseLong(encodedValue); + addValueNode(l); + return this; + } + catch (NumberFormatException ignored) { + } + + try { + BigInteger bi = new BigInteger(encodedValue); + addValueNode(bi); + return this; + } + catch (NumberFormatException ignored) { + } + + try { + double d = Double.parseDouble(encodedValue); + addValueNode(d); + return this; + } + catch (NumberFormatException ignored) { + } + + try { + BigDecimal bc = new BigDecimal(encodedValue); + addValueNode(bc); + return this; + } + catch (NumberFormatException ignored) { + } + + throw new NumberFormatException(encodedValue); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + @Override + public JsonGenerator writeBoolean(boolean state) throws JacksonException + { + try { + addValueNode(state); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + @Override + public JsonGenerator writeNull() throws JacksonException + { + try { + addValueNode(null); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + return this; + } + + public void writeExtensionType(MessagePackExtensionType extensionType) + { + try { + addValueNode(extensionType); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + @Override + public void close() throws JacksonException + { + if (!_closed) { + flush(); + super.close(); + } + } + + @Override + public void flush() throws JacksonException + { + if (!isElementsClosed) { + return; + } + + try { + for (int i = 0; i < nodes.size(); i++) { + Node node = nodes.get(i); + if (node instanceof NodeEntryInObject) { + NodeEntryInObject nodeEntry = (NodeEntryInObject) node; + packNonContainer(nodeEntry.key); + if (nodeEntry.value instanceof NodeObject) { + packObject((NodeObject) nodeEntry.value); + } + else if (nodeEntry.value instanceof NodeArray) { + packArray((NodeArray) nodeEntry.value); + } + else { + packNonContainer(nodeEntry.value); + } + } + else if (node instanceof NodeObject) { + packObject((NodeObject) node); + } + else if (node instanceof NodeEntryInArray) { + packNonContainer(((NodeEntryInArray) node).value); + } + else if (node instanceof NodeArray) { + packArray((NodeArray) node); + } + else { + throw new AssertionError(); + } + } + flushMessagePacker(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + nodes.clear(); + isElementsClosed = false; + } + + private void flushMessagePacker() + throws IOException + { + MessagePacker messagePacker = getMessagePacker(); + messagePacker.flush(); + } + + @Override + public tools.jackson.core.Version version() + { + return PackageVersion.VERSION; + } + + @Override + public TokenStreamContext streamWriteContext() + { + return writeContext; + } + + @Override + public Object streamWriteOutputTarget() + { + return output; + } + + @Override + public int streamWriteOutputBuffered() + { + return -1; + } + + @Override + public Object currentValue() + { + return writeContext.currentValue(); + } + + @Override + public void assignCurrentValue(Object v) + { + writeContext.assignCurrentValue(v); + } + + @Override + protected void _closeInput() throws IOException + { + if (StreamWriteFeature.AUTO_CLOSE_TARGET.enabledIn(_streamWriteFeatures)) { + messagePacker.close(); + } + } + + @Override + protected void _releaseBuffers() + { + OutputStreamBufferOutput messageBufferOutput = messageBufferOutputHolder.get(); + if (messageBufferOutput != null) { + try { + messageBufferOutput.reset(null); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + } + + @Override + protected void _verifyValueWrite(String typeMsg) throws JacksonException + { + if (!writeContext.writeValue()) { + _reportError("Cannot " + typeMsg + ", expecting a property name"); + } + } + + private MessagePacker getMessagePacker() + { + return messagePacker; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackKeySerializer.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackKeySerializer.java new file mode 100644 index 00000000..836a905d --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackKeySerializer.java @@ -0,0 +1,35 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ser.std.StdSerializer; + +public class MessagePackKeySerializer + extends StdSerializer +{ + public MessagePackKeySerializer() + { + super(Object.class); + } + + @Override + public void serialize(Object value, JsonGenerator jgen, SerializationContext provider) + { + jgen.writeName(new MessagePackSerializedString(value)); + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackMapper.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackMapper.java new file mode 100644 index 00000000..674b2733 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackMapper.java @@ -0,0 +1,120 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import com.fasterxml.jackson.annotation.JsonFormat; + +import tools.jackson.core.Version; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.cfg.MapperBuilder; +import tools.jackson.databind.cfg.MapperBuilderState; + +import java.math.BigDecimal; +import java.math.BigInteger; + +public class MessagePackMapper extends ObjectMapper +{ + private static final long serialVersionUID = 3L; + + public static class Builder extends MapperBuilder + { + public Builder(MessagePackFactory f) + { + super(f); + } + + protected Builder(StateImpl state) + { + super(state); + } + + @Override + public MessagePackMapper build() + { + return new MessagePackMapper(this); + } + + public Builder handleBigIntegerAsString() + { + return withConfigOverride(BigInteger.class, + o -> o.setFormat(JsonFormat.Value.forShape(JsonFormat.Shape.STRING))); + } + + public Builder handleBigDecimalAsString() + { + return withConfigOverride(BigDecimal.class, + o -> o.setFormat(JsonFormat.Value.forShape(JsonFormat.Shape.STRING))); + } + + public Builder handleBigIntegerAndBigDecimalAsString() + { + return handleBigIntegerAsString().handleBigDecimalAsString(); + } + + @Override + protected MapperBuilderState _saveState() + { + return new StateImpl(this); + } + + protected static class StateImpl extends MapperBuilderState + { + private static final long serialVersionUID = 3L; + + public StateImpl(Builder src) + { + super(src); + } + + @Override + protected Object readResolve() + { + return new Builder(this).build(); + } + } + } + + public MessagePackMapper() + { + this(new Builder(new MessagePackFactory())); + } + + public MessagePackMapper(MessagePackFactory f) + { + this(new Builder(f)); + } + + protected MessagePackMapper(Builder builder) + { + super(builder); + } + + public static Builder builder() + { + return new Builder(new MessagePackFactory()); + } + + public static Builder builder(MessagePackFactory f) + { + return new Builder(f); + } + + @Override + public Version version() + { + return PackageVersion.VERSION; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java new file mode 100644 index 00000000..3bd64f83 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -0,0 +1,626 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.Base64Variant; +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonToken; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.StreamReadFeature; +import tools.jackson.core.TokenStreamContext; +import tools.jackson.core.TokenStreamLocation; +import tools.jackson.core.Version; +import tools.jackson.core.base.ParserMinimalBase; +import tools.jackson.core.exc.UnexpectedEndOfInputException; +import tools.jackson.core.io.IOContext; +import tools.jackson.core.json.DupDetector; +import org.msgpack.core.ExtensionTypeHeader; +import org.msgpack.core.MessageFormat; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.core.buffer.InputStreamBufferInput; +import org.msgpack.core.buffer.MessageBufferInput; +import org.msgpack.value.ValueType; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +public class MessagePackParser + extends ParserMinimalBase +{ + private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); + private final MessageUnpacker messageUnpacker; + + private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + + private MessagePackReadContext streamReadContext; + + private boolean isClosed; + private long tokenPosition; + private long currentPosition; + private final IOContext ioContext; + private ExtensionTypeCustomDeserializers extTypeCustomDesers; + + private enum Type + { + INT, LONG, DOUBLE, STRING, BYTES, BIG_INT, EXT + } + private Type type; + private int intValue; + private long longValue; + private double doubleValue; + private byte[] bytesValue; + private String stringValue; + private BigInteger biValue; + private MessagePackExtensionType extensionTypeValue; + + public MessagePackParser( + ObjectReadContext readCtxt, + IOContext ioCtxt, + int streamReadFeatures, + InputStream in, + boolean reuseResourceInParser) + throws IOException + { + this(readCtxt, ioCtxt, streamReadFeatures, new InputStreamBufferInput(in), in, reuseResourceInParser); + } + + MessagePackParser(ObjectReadContext readCtxt, + IOContext ioCtxt, + int streamReadFeatures, + MessageBufferInput input, + Object src, + boolean reuseResourceInParser) + throws IOException + { + super(readCtxt, ioCtxt, streamReadFeatures); + + ioContext = ioCtxt; + DupDetector dups = StreamReadFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamReadFeatures) + ? DupDetector.rootDetector(this) : null; + streamReadContext = MessagePackReadContext.createRootContext(dups); + if (!reuseResourceInParser) { + messageUnpacker = MessagePack.newDefaultUnpacker(input); + return; + } + + Tuple messageUnpackerTuple = messageUnpackerHolder.get(); + if (messageUnpackerTuple == null) { + messageUnpacker = MessagePack.newDefaultUnpacker(input); + } + else { + if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || messageUnpackerTuple.first() != src || src instanceof byte[]) { + messageUnpackerTuple.second().reset(input); + } + messageUnpacker = messageUnpackerTuple.second(); + } + messageUnpackerHolder.set(new Tuple<>(src, messageUnpacker)); + } + + public void setExtensionTypeCustomDeserializers(ExtensionTypeCustomDeserializers extTypeCustomDesers) + { + this.extTypeCustomDesers = extTypeCustomDesers; + } + + @Override + public Version version() + { + return PackageVersion.VERSION; + } + + private String unpackString(MessageUnpacker messageUnpacker) throws IOException + { + return messageUnpacker.unpackString(); + } + + @Override + public JsonToken nextToken() throws JacksonException + { + try { + return _nextToken(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + + private JsonToken _nextToken() throws IOException + { + tokenPosition = messageUnpacker.getTotalReadBytes(); + + boolean isObjectValueSet = streamReadContext.inObject() && _currToken != JsonToken.PROPERTY_NAME; + if (isObjectValueSet) { + if (!streamReadContext.expectMoreValues()) { + streamReadContext = streamReadContext.getParent(); + return _updateToken(JsonToken.END_OBJECT); + } + } + else if (streamReadContext.inArray()) { + if (!streamReadContext.expectMoreValues()) { + streamReadContext = streamReadContext.getParent(); + return _updateToken(JsonToken.END_ARRAY); + } + } + + if (!messageUnpacker.hasNext()) { + if (streamReadContext.inRoot()) { + return null; + } + throw new UnexpectedEndOfInputException(this, null, null); + } + + MessageFormat format = messageUnpacker.getNextFormat(); + ValueType valueType = format.getValueType(); + + JsonToken nextToken; + switch (valueType) { + case STRING: + type = Type.STRING; + stringValue = unpackString(messageUnpacker); + if (isObjectValueSet) { + streamReadContext.setCurrentName(stringValue); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_STRING; + } + break; + case INTEGER: + Object v; + switch (format) { + case UINT64: + BigInteger bi = messageUnpacker.unpackBigInteger(); + if (0 <= bi.compareTo(LONG_MIN) && bi.compareTo(LONG_MAX) <= 0) { + type = Type.LONG; + longValue = bi.longValue(); + v = longValue; + } + else { + type = Type.BIG_INT; + biValue = bi; + v = biValue; + } + break; + default: + long l = messageUnpacker.unpackLong(); + if (Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE) { + type = Type.INT; + intValue = (int) l; + v = intValue; + } + else { + type = Type.LONG; + longValue = l; + v = longValue; + } + break; + } + + if (isObjectValueSet) { + streamReadContext.setCurrentName(String.valueOf(v)); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_NUMBER_INT; + } + break; + case NIL: + messageUnpacker.unpackNil(); + nextToken = JsonToken.VALUE_NULL; + break; + case BOOLEAN: + boolean b = messageUnpacker.unpackBoolean(); + if (isObjectValueSet) { + streamReadContext.setCurrentName(Boolean.toString(b)); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = b ? JsonToken.VALUE_TRUE : JsonToken.VALUE_FALSE; + } + break; + case FLOAT: + type = Type.DOUBLE; + doubleValue = messageUnpacker.unpackDouble(); + if (isObjectValueSet) { + streamReadContext.setCurrentName(String.valueOf(doubleValue)); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_NUMBER_FLOAT; + } + break; + case BINARY: + type = Type.BYTES; + int len = messageUnpacker.unpackBinaryHeader(); + bytesValue = messageUnpacker.readPayload(len); + if (isObjectValueSet) { + streamReadContext.setCurrentName(new String(bytesValue, MessagePack.UTF8)); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_EMBEDDED_OBJECT; + } + break; + case ARRAY: + nextToken = JsonToken.START_ARRAY; + streamReadContext = streamReadContext.createChildArrayContext(messageUnpacker.unpackArrayHeader()); + break; + case MAP: + nextToken = JsonToken.START_OBJECT; + streamReadContext = streamReadContext.createChildObjectContext(messageUnpacker.unpackMapHeader()); + break; + case EXTENSION: + type = Type.EXT; + ExtensionTypeHeader header = messageUnpacker.unpackExtensionTypeHeader(); + extensionTypeValue = new MessagePackExtensionType(header.getType(), messageUnpacker.readPayload(header.getLength())); + if (isObjectValueSet) { + streamReadContext.setCurrentName(deserializedExtensionTypeValue().toString()); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_EMBEDDED_OBJECT; + } + break; + default: + throw new IllegalStateException("Shouldn't reach here"); + } + currentPosition = messageUnpacker.getTotalReadBytes(); + + _updateToken(nextToken); + + return nextToken; + } + + @Override + protected void _handleEOF() + { + } + + @Override + public String getString() + { + switch (type) { + case STRING: + return stringValue; + case BYTES: + return new String(bytesValue, MessagePack.UTF8); + case INT: + return String.valueOf(intValue); + case LONG: + return String.valueOf(longValue); + case DOUBLE: + return String.valueOf(doubleValue); + case BIG_INT: + return String.valueOf(biValue); + case EXT: + try { + return deserializedExtensionTypeValue().toString(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public char[] getStringCharacters() + { + return getString().toCharArray(); + } + + @Override + public boolean hasStringCharacters() + { + return false; + } + + @Override + public int getStringLength() + { + return getString().length(); + } + + @Override + public int getStringOffset() + { + return 0; + } + + @Override + public byte[] getBinaryValue(Base64Variant b64variant) + { + switch (type) { + case BYTES: + return bytesValue; + case STRING: + return stringValue.getBytes(MessagePack.UTF8); + case EXT: + return extensionTypeValue.getData(); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public Number getNumberValue() + { + switch (type) { + case INT: + return intValue; + case LONG: + return longValue; + case DOUBLE: + return doubleValue; + case BIG_INT: + return biValue; + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public int getIntValue() + { + switch (type) { + case INT: + return intValue; + case LONG: + return (int) longValue; + case DOUBLE: + return (int) doubleValue; + case BIG_INT: + return biValue.intValue(); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public long getLongValue() + { + switch (type) { + case INT: + return intValue; + case LONG: + return longValue; + case DOUBLE: + return (long) doubleValue; + case BIG_INT: + return biValue.longValue(); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public BigInteger getBigIntegerValue() + { + switch (type) { + case INT: + return BigInteger.valueOf(intValue); + case LONG: + return BigInteger.valueOf(longValue); + case DOUBLE: + return BigInteger.valueOf((long) doubleValue); + case BIG_INT: + return biValue; + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public float getFloatValue() + { + switch (type) { + case INT: + return (float) intValue; + case LONG: + return (float) longValue; + case DOUBLE: + return (float) doubleValue; + case BIG_INT: + return biValue.floatValue(); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public double getDoubleValue() + { + switch (type) { + case INT: + return intValue; + case LONG: + return (double) longValue; + case DOUBLE: + return doubleValue; + case BIG_INT: + return biValue.doubleValue(); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public BigDecimal getDecimalValue() + { + switch (type) { + case INT: + return BigDecimal.valueOf(intValue); + case LONG: + return BigDecimal.valueOf(longValue); + case DOUBLE: + return BigDecimal.valueOf(doubleValue); + case BIG_INT: + return new BigDecimal(biValue); + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + private Object deserializedExtensionTypeValue() + throws IOException + { + if (extTypeCustomDesers != null) { + ExtensionTypeCustomDeserializers.Deser deser = extTypeCustomDesers.getDeser(extensionTypeValue.getType()); + if (deser != null) { + return deser.deserialize(extensionTypeValue.getData()); + } + } + return extensionTypeValue; + } + + @Override + public Object getEmbeddedObject() + { + switch (type) { + case BYTES: + return bytesValue; + case EXT: + try { + return deserializedExtensionTypeValue(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + public NumberType getNumberType() + { + switch (type) { + case INT: + return NumberType.INT; + case LONG: + return NumberType.LONG; + case DOUBLE: + return NumberType.DOUBLE; + case BIG_INT: + return NumberType.BIG_INTEGER; + default: + throw new IllegalStateException("Invalid type=" + type); + } + } + + @Override + protected void _closeInput() throws IOException + { + if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { + messageUnpacker.close(); + } + } + + @Override + protected void _releaseBuffers() + { + } + + @Override + public boolean isClosed() + { + return isClosed; + } + + @Override + public void close() + { + try { + _closeInput(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + finally { + isClosed = true; + Tuple tuple = messageUnpackerHolder.get(); + if (tuple != null && tuple.first() instanceof byte[]) { + messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); + } + } + } + + @Override + public TokenStreamContext streamReadContext() + { + return streamReadContext; + } + + @Override + public TokenStreamLocation currentTokenLocation() + { + // columnNr repurposed as byte offset; truncates for inputs > 2 GB + return new TokenStreamLocation(ioContext.contentReference(), tokenPosition, -1, (int) tokenPosition); + } + + @Override + public TokenStreamLocation currentLocation() + { + // columnNr repurposed as byte offset; truncates for inputs > 2 GB + return new TokenStreamLocation(ioContext.contentReference(), currentPosition, -1, (int) currentPosition); + } + + @Override + public String currentName() + { + if (_currToken == JsonToken.START_OBJECT || _currToken == JsonToken.START_ARRAY) { + MessagePackReadContext parent = streamReadContext.getParent(); + return parent.currentName(); + } + return streamReadContext.currentName(); + } + + @Override + public Object streamReadInputSource() + { + return ioContext.contentReference().getRawContent(); + } + + @Override + public Object currentValue() + { + return streamReadContext.currentValue(); + } + + @Override + public void assignCurrentValue(Object v) + { + streamReadContext.assignCurrentValue(v); + } + + @Override + public boolean isNaN() + { + if (type == Type.DOUBLE) { + return Double.isNaN(doubleValue) || Double.isInfinite(doubleValue); + } + return false; + } + + public boolean isCurrentFieldId() + { + return this.type == Type.INT || this.type == Type.LONG; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java new file mode 100644 index 00000000..1da2574c --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java @@ -0,0 +1,196 @@ +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.TokenStreamContext; +import tools.jackson.core.TokenStreamLocation; +import tools.jackson.core.exc.StreamReadException; +import tools.jackson.core.io.ContentReference; +import tools.jackson.core.json.DupDetector; + +/** + * Replacement of {@link tools.jackson.core.json.JsonReadContext} + * to support features needed by MessagePack format. + */ +public final class MessagePackReadContext + extends TokenStreamContext +{ + protected final MessagePackReadContext parent; + + protected final DupDetector dups; + + /** + * For fixed-size Arrays, Objects, this indicates expected number of entries. + */ + protected int expEntryCount; + + protected String currentName; + + protected Object currentValue; + + protected MessagePackReadContext child = null; + + public MessagePackReadContext(MessagePackReadContext parent, DupDetector dups, + int type, int expEntryCount) + { + super(); + this.parent = parent; + this.dups = dups; + _type = type; + this.expEntryCount = expEntryCount; + _index = -1; + _nestingDepth = parent == null ? 0 : parent._nestingDepth + 1; + } + + protected void reset(int type, int expEntryCount) + { + _type = type; + this.expEntryCount = expEntryCount; + _index = -1; + currentName = null; + currentValue = null; + if (dups != null) { + dups.reset(); + } + } + + @Override + public Object currentValue() + { + return currentValue; + } + + @Override + public void assignCurrentValue(Object v) + { + currentValue = v; + } + + public static MessagePackReadContext createRootContext(DupDetector dups) + { + return new MessagePackReadContext(null, dups, TYPE_ROOT, -1); + } + + public MessagePackReadContext createChildArrayContext(int expEntryCount) + { + MessagePackReadContext ctxt = child; + if (ctxt == null) { + ctxt = new MessagePackReadContext(this, + (dups == null) ? null : dups.child(), + TYPE_ARRAY, expEntryCount); + child = ctxt; + } + else { + ctxt.reset(TYPE_ARRAY, expEntryCount); + } + return ctxt; + } + + public MessagePackReadContext createChildObjectContext(int expEntryCount) + { + MessagePackReadContext ctxt = child; + if (ctxt == null) { + ctxt = new MessagePackReadContext(this, + (dups == null) ? null : dups.child(), + TYPE_OBJECT, expEntryCount); + child = ctxt; + return ctxt; + } + ctxt.reset(TYPE_OBJECT, expEntryCount); + return ctxt; + } + + @Override + public String currentName() + { + return currentName; + } + + @Override + public MessagePackReadContext getParent() + { + return parent; + } + + public boolean hasExpectedLength() + { + return (expEntryCount >= 0); + } + + public int getExpectedLength() + { + return expEntryCount; + } + + public boolean isEmpty() + { + return expEntryCount == 0; + } + + public int getRemainingExpectedLength() + { + int diff = expEntryCount - _index; + return Math.max(0, diff); + } + + public boolean acceptsBreakMarker() + { + return (expEntryCount < 0) && _type != TYPE_ROOT; + } + + public boolean expectMoreValues() + { + if (++_index == expEntryCount) { + return false; + } + return true; + } + + public TokenStreamLocation startLocation(ContentReference srcRef) + { + return new TokenStreamLocation(srcRef, 1L, -1, -1); + } + + public void setCurrentName(String name) + { + currentName = name; + if (dups != null) { + _checkDup(dups, name); + } + } + + private void _checkDup(DupDetector dd, String name) + { + if (dd.isDup(name)) { + throw new StreamReadException(null, + "Duplicate field '" + name + "'", dd.findLocation()); + } + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(64); + switch (_type) { + case TYPE_ROOT: + sb.append("/"); + break; + case TYPE_ARRAY: + sb.append('['); + sb.append(getCurrentIndex()); + sb.append(']'); + break; + case TYPE_OBJECT: + sb.append('{'); + if (currentName != null) { + sb.append('"'); + sb.append(currentName); + sb.append('"'); + } + else { + sb.append('?'); + } + sb.append('}'); + break; + } + return sb.toString(); + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java new file mode 100644 index 00000000..c24c1559 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java @@ -0,0 +1,153 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.SerializableString; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public class MessagePackSerializedString + implements SerializableString +{ + private static final Charset UTF8 = StandardCharsets.UTF_8; + private final Object value; + + public MessagePackSerializedString(Object value) + { + this.value = value; + } + + @Override + public String getValue() + { + return value.toString(); + } + + @Override + public int charLength() + { + return getValue().length(); + } + + @Override + public char[] asQuotedChars() + { + return getValue().toCharArray(); + } + + @Override + public byte[] asUnquotedUTF8() + { + return getValue().getBytes(UTF8); + } + + @Override + public byte[] asQuotedUTF8() + { + return asUnquotedUTF8(); + } + + @Override + public int appendQuotedUTF8(byte[] bytes, int i) + { + byte[] utf8 = asUnquotedUTF8(); + if (utf8.length > bytes.length - i) { + return -1; + } + System.arraycopy(utf8, 0, bytes, i, utf8.length); + return utf8.length; + } + + @Override + public int appendQuoted(char[] chars, int i) + { + char[] q = asQuotedChars(); + if (q.length > chars.length - i) { + return -1; + } + System.arraycopy(q, 0, chars, i, q.length); + return q.length; + } + + @Override + public int appendUnquotedUTF8(byte[] bytes, int i) + { + byte[] utf8 = asUnquotedUTF8(); + if (utf8.length > bytes.length - i) { + return -1; + } + System.arraycopy(utf8, 0, bytes, i, utf8.length); + return utf8.length; + } + + @Override + public int appendUnquoted(char[] chars, int i) + { + String v = getValue(); + if (v.length() > chars.length - i) { + return -1; + } + v.getChars(0, v.length(), chars, i); + return v.length(); + } + + @Override + public int writeQuotedUTF8(OutputStream outputStream) throws IOException + { + byte[] utf8 = asUnquotedUTF8(); + outputStream.write(utf8); + return utf8.length; + } + + @Override + public int writeUnquotedUTF8(OutputStream outputStream) throws IOException + { + byte[] utf8 = asUnquotedUTF8(); + outputStream.write(utf8); + return utf8.length; + } + + @Override + public int putQuotedUTF8(ByteBuffer byteBuffer) + { + byte[] utf8 = asUnquotedUTF8(); + if (utf8.length > byteBuffer.remaining()) { + return -1; + } + byteBuffer.put(utf8); + return utf8.length; + } + + @Override + public int putUnquotedUTF8(ByteBuffer byteBuffer) + { + byte[] utf8 = asUnquotedUTF8(); + if (utf8.length > byteBuffer.remaining()) { + return -1; + } + byteBuffer.put(utf8); + return utf8.length; + } + + public Object getRawValue() + { + return value; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java new file mode 100644 index 00000000..67ee2e2f --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java @@ -0,0 +1,42 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.databind.JavaType; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ValueSerializer; +import tools.jackson.databind.cfg.SerializerFactoryConfig; +import tools.jackson.databind.ser.BeanSerializerFactory; + +public class MessagePackSerializerFactory + extends BeanSerializerFactory +{ + public MessagePackSerializerFactory() + { + super(null); + } + + public MessagePackSerializerFactory(SerializerFactoryConfig config) + { + super(config); + } + + @Override + public ValueSerializer createKeySerializer(SerializationContext ctxt, JavaType keyType) + { + return new MessagePackKeySerializer(); + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java new file mode 100644 index 00000000..29d704cc --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java @@ -0,0 +1,30 @@ +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.Version; +import tools.jackson.core.Versioned; + +public class PackageVersion + implements Versioned +{ + public static final Version VERSION = new Version(0, 9, 12, null, "org.msgpack", "msgpack-jackson3"); + + @Override + public Version version() + { + return VERSION; + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java new file mode 100644 index 00000000..e269fa49 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java @@ -0,0 +1,89 @@ +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.deser.std.StdDeserializer; +import tools.jackson.databind.module.SimpleModule; +import tools.jackson.databind.ser.std.StdSerializer; +import org.msgpack.core.ExtensionTypeHeader; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.core.MessageUnpacker; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Instant; + +public class TimestampExtensionModule +{ + public static final byte EXT_TYPE = -1; + public static final SimpleModule INSTANCE = new SimpleModule("msgpack-ext-timestamp"); + + static { + INSTANCE.addSerializer(Instant.class, new InstantSerializer(Instant.class)); + INSTANCE.addDeserializer(Instant.class, new InstantDeserializer(Instant.class)); + } + + private static class InstantSerializer extends StdSerializer + { + protected InstantSerializer(Class t) + { + super(t); + } + + @Override + public void serialize(Instant value, JsonGenerator gen, SerializationContext provider) + { + try { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(os)) { + packer.packTimestamp(value); + } + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(os.toByteArray())) { + ExtensionTypeHeader header = unpacker.unpackExtensionTypeHeader(); + byte[] bytes = unpacker.readPayload(header.getLength()); + + MessagePackExtensionType extensionType = new MessagePackExtensionType(EXT_TYPE, bytes); + gen.writePOJO(extensionType); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + private static class InstantDeserializer extends StdDeserializer + { + protected InstantDeserializer(Class vc) + { + super(vc); + } + + @Override + public Instant deserialize(JsonParser p, DeserializationContext ctxt) + { + try { + MessagePackExtensionType ext = p.readValueAs(MessagePackExtensionType.class); + if (ext.getType() != EXT_TYPE) { + ctxt.reportInputMismatch(Instant.class, + "Unexpected extension type (0x%X) for Instant object", ext.getType() & 0xFF); + return null; // unreachable + } + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ext.getData())) { + return unpacker.unpackTimestamp(new ExtensionTypeHeader(EXT_TYPE, ext.getData().length)); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + private TimestampExtensionModule() + { + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/Tuple.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/Tuple.java new file mode 100644 index 00000000..b0f720d8 --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/Tuple.java @@ -0,0 +1,38 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +public class Tuple +{ + private final F first; + private final S second; + + public Tuple(F first, S second) + { + this.first = first; + this.second = second; + } + + public F first() + { + return first; + } + + public S second() + { + return second; + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/ExampleOfTypeInformationSerDe.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/ExampleOfTypeInformationSerDe.java new file mode 100644 index 00000000..6a194a8c --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/ExampleOfTypeInformationSerDe.java @@ -0,0 +1,171 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonParser; +import tools.jackson.core.JacksonException; +import tools.jackson.core.TreeNode; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ValueDeserializer; +import tools.jackson.databind.ValueSerializer; +import tools.jackson.databind.annotation.JsonDeserialize; +import tools.jackson.databind.annotation.JsonSerialize; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class ExampleOfTypeInformationSerDe + extends MessagePackDataformatTestBase +{ + static class A + { + private List list = new ArrayList(); + + public List getList() + { + return list; + } + + public void setList(List list) + { + this.list = list; + } + } + + static class B + { + private String str; + + public String getStr() + { + return str; + } + + public void setStr(String str) + { + this.str = str; + } + } + + @JsonSerialize(using = ObjectContainerSerializer.class) + @JsonDeserialize(using = ObjectContainerDeserializer.class) + static class ObjectContainer + { + private final Map objects; + + public ObjectContainer(Map objects) + { + this.objects = objects; + } + + public Map getObjects() + { + return objects; + } + } + + static class ObjectContainerSerializer + extends ValueSerializer + { + @Override + public void serialize(ObjectContainer value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeStartObject(); + HashMap metadata = new HashMap(); + for (Map.Entry entry : value.getObjects().entrySet()) { + metadata.put(entry.getKey(), entry.getValue().getClass().getName()); + } + gen.writePOJOProperty("__metadata", metadata); + gen.writePOJOProperty("objects", value.getObjects()); + gen.writeEndObject(); + } + } + + static class ObjectContainerDeserializer + extends ValueDeserializer + { + @Override + public ObjectContainer deserialize(JsonParser p, DeserializationContext ctxt) + throws JacksonException + { + ObjectContainer objectContainer = new ObjectContainer(new HashMap()); + TreeNode treeNode = p.readValueAsTree(); + + Map metadata = treeNode.get("__metadata") + .traverse(p.objectReadContext()) + .readValueAs(new TypeReference>() {}); + TreeNode dataMapTree = treeNode.get("objects"); + for (Map.Entry entry : metadata.entrySet()) { + try { + Object o = dataMapTree.get(entry.getKey()) + .traverse(p.objectReadContext()) + .readValueAs(Class.forName(entry.getValue())); + objectContainer.getObjects().put(entry.getKey(), o); + } + catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to deserialize: " + entry, e); + } + } + + return objectContainer; + } + } + + @Test + public void test() + throws IOException + { + ObjectContainer objectContainer = new ObjectContainer(new HashMap()); + { + A a = new A(); + a.setList(Arrays.asList("first", "second", "third")); + objectContainer.getObjects().put("a", a); + + B b = new B(); + b.setStr("hello world"); + objectContainer.getObjects().put("b", b); + + Double pi = 3.14; + objectContainer.getObjects().put("pi", pi); + } + + ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(objectContainer); + ObjectContainer restored = objectMapper.readValue(bytes, ObjectContainer.class); + + { + assertEquals(3, restored.getObjects().size()); + A a = (A) restored.getObjects().get("a"); + assertArrayEquals(new String[] {"first", "second", "third"}, a.getList().toArray()); + B b = (B) restored.getObjects().get("b"); + assertEquals("hello world", b.getStr()); + assertEquals(3.14, restored.getObjects().get("pi")); + } + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForFieldIdTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForFieldIdTest.java new file mode 100644 index 00000000..23983a39 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForFieldIdTest.java @@ -0,0 +1,135 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonParser; +import tools.jackson.core.JacksonException; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.KeyDeserializer; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.ValueDeserializer; +import tools.jackson.databind.deser.NullValueProvider; +import tools.jackson.databind.deser.jdk.JDKValueInstantiators; +import tools.jackson.databind.deser.jdk.MapDeserializer; +import tools.jackson.databind.jsontype.TypeDeserializer; +import tools.jackson.databind.module.SimpleModule; +import tools.jackson.databind.type.TypeFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.LinkedHashMap; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class MessagePackDataformatForFieldIdTest +{ + static class MessagePackMapDeserializer extends MapDeserializer + { + public static KeyDeserializer keyDeserializer = new KeyDeserializer() + { + @Override + public Object deserializeKey(String s, DeserializationContext deserializationContext) + throws JacksonException + { + JsonParser parser = deserializationContext.getParser(); + if (parser instanceof MessagePackParser) { + MessagePackParser p = (MessagePackParser) parser; + if (p.isCurrentFieldId()) { + return Integer.valueOf(s); + } + } + return s; + } + }; + + public MessagePackMapDeserializer() + { + super( + TypeFactory.createDefaultInstance().constructMapType(Map.class, Object.class, Object.class), + JDKValueInstantiators.findStdValueInstantiator(null, LinkedHashMap.class), + keyDeserializer, null, null); + } + + public MessagePackMapDeserializer(MapDeserializer src, KeyDeserializer keyDeser, + ValueDeserializer valueDeser, TypeDeserializer valueTypeDeser, NullValueProvider nuller, + Set ignorable, Set includable) + { + super(src, keyDeser, valueDeser, valueTypeDeser, nuller, ignorable, includable); + } + + @Override + protected MapDeserializer withResolved(KeyDeserializer keyDeser, TypeDeserializer valueTypeDeser, + ValueDeserializer valueDeser, NullValueProvider nuller, Set ignorable, + Set includable) + { + return new MessagePackMapDeserializer(this, keyDeser, (ValueDeserializer) valueDeser, valueTypeDeser, + nuller, ignorable, includable); + } + } + + @Test + public void testMixedKeys() + throws IOException + { + ObjectMapper mapper = MessagePackMapper.builder( + new MessagePackFactory() + .setSupportIntegerKeys(true) + ) + .addModule(new SimpleModule() + .addDeserializer(Map.class, new MessagePackMapDeserializer())) + .build(); + + Map map = new HashMap<>(); + map.put(1, "one"); + map.put("2", "two"); + + byte[] bytes = mapper.writeValueAsBytes(map); + Map deserializedInit = mapper.readValue(bytes, new TypeReference>() {}); + + Map expected = new HashMap<>(map); + Map actual = new HashMap<>(deserializedInit); + + assertEquals(expected, actual); + } + + @Test + public void testMixedKeysBackwardsCompatible() + throws IOException + { + ObjectMapper mapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule() + .addDeserializer(Map.class, new MessagePackMapDeserializer())) + .build(); + + Map map = new HashMap<>(); + map.put(1, "one"); + map.put("2", "two"); + + byte[] bytes = mapper.writeValueAsBytes(map); + Map deserializedInit = mapper.readValue(bytes, new TypeReference>() {}); + + Map expected = new HashMap<>(); + expected.put("1", "one"); + expected.put("2", "two"); + Map actual = new HashMap<>(deserializedInit); + + assertEquals(expected, actual); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java new file mode 100644 index 00000000..417c1389 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java @@ -0,0 +1,150 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.databind.ObjectMapper; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.Charset; + +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.containsString; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; +import static org.hamcrest.MatcherAssert.assertThat; + +public class MessagePackDataformatForPojoTest + extends MessagePackDataformatTestBase +{ + @Test + public void testNormal() + throws IOException + { + byte[] bytes = objectMapper.writeValueAsBytes(normalPojo); + NormalPojo value = objectMapper.readValue(bytes, NormalPojo.class); + assertEquals(normalPojo.s, value.getS()); + assertEquals(normalPojo.bool, value.bool); + assertEquals(normalPojo.i, value.i); + assertEquals(normalPojo.l, value.l); + assertEquals(normalPojo.f, value.f, 0.000001f); + assertEquals(normalPojo.d, value.d, 0.000001f); + assertArrayEquals(normalPojo.b, value.b); + assertEquals(normalPojo.bi, value.bi); + assertEquals(normalPojo.suit, Suit.HEART); + assertEquals(normalPojo.sMultibyte, value.sMultibyte); + } + + @Test + public void testNestedList() + throws IOException + { + byte[] bytes = objectMapper.writeValueAsBytes(nestedListPojo); + NestedListPojo value = objectMapper.readValue(bytes, NestedListPojo.class); + assertEquals(nestedListPojo.s, value.s); + assertArrayEquals(nestedListPojo.strs.toArray(), value.strs.toArray()); + } + + @Test + public void testNestedListComplex1() + throws IOException + { + byte[] bytes = objectMapper.writeValueAsBytes(nestedListComplexPojo1); + NestedListComplexPojo1 value = objectMapper.readValue(bytes, NestedListComplexPojo1.class); + assertEquals(nestedListComplexPojo1.s, value.s); + assertEquals(1, nestedListComplexPojo1.foos.size()); + assertEquals(nestedListComplexPojo1.foos.get(0).t, value.foos.get(0).t); + } + + @Test + public void testNestedListComplex2() + throws IOException + { + byte[] bytes = objectMapper.writeValueAsBytes(nestedListComplexPojo2); + NestedListComplexPojo2 value = objectMapper.readValue(bytes, NestedListComplexPojo2.class); + assertEquals(nestedListComplexPojo2.s, value.s); + assertEquals(2, nestedListComplexPojo2.foos.size()); + assertEquals(nestedListComplexPojo2.foos.get(0).t, value.foos.get(0).t); + assertEquals(nestedListComplexPojo2.foos.get(1).t, value.foos.get(1).t); + } + + @Test + public void testStrings() + throws IOException + { + byte[] bytes = objectMapper.writeValueAsBytes(stringPojo); + StringPojo value = objectMapper.readValue(bytes, StringPojo.class); + assertEquals(stringPojo.shortSingleByte, value.shortSingleByte); + assertEquals(stringPojo.longSingleByte, value.longSingleByte); + assertEquals(stringPojo.shortMultiByte, value.shortMultiByte); + assertEquals(stringPojo.longMultiByte, value.longMultiByte); + } + + @Test + public void testUsingCustomConstructor() + throws IOException + { + UsingCustomConstructorPojo orig = new UsingCustomConstructorPojo("komamitsu", 55); + byte[] bytes = objectMapper.writeValueAsBytes(orig); + UsingCustomConstructorPojo value = objectMapper.readValue(bytes, UsingCustomConstructorPojo.class); + assertEquals("komamitsu", value.name); + assertEquals(55, value.age); + } + + @Test + public void testIgnoringProperties() + throws IOException + { + IgnoringPropertiesPojo orig = new IgnoringPropertiesPojo(); + orig.internal = "internal"; + orig.external = "external"; + orig.setCode(1234); + byte[] bytes = objectMapper.writeValueAsBytes(orig); + IgnoringPropertiesPojo value = objectMapper.readValue(bytes, IgnoringPropertiesPojo.class); + assertEquals(0, value.getCode()); + assertEquals(null, value.internal); + assertEquals("external", value.external); + } + + @Test + public void testChangingPropertyNames() + throws IOException + { + ChangingPropertyNamesPojo orig = new ChangingPropertyNamesPojo(); + orig.setTheName("komamitsu"); + byte[] bytes = objectMapper.writeValueAsBytes(orig); + ChangingPropertyNamesPojo value = objectMapper.readValue(bytes, ChangingPropertyNamesPojo.class); + assertEquals("komamitsu", value.getTheName()); + } + + @Test + public void testSerializationWithoutSchema() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(factory) + .annotationIntrospector(new JsonArrayFormat()) + .build(); + byte[] bytes = objectMapper.writeValueAsBytes(complexPojo); + String schema = new String(bytes, Charset.forName("UTF-8")); + assertThat(schema, not(containsString("name"))); + ComplexPojo value = objectMapper.readValue(bytes, ComplexPojo.class); + assertEquals("komamitsu", value.name); + assertEquals(20, value.age); + assertArrayEquals(complexPojo.values.toArray(), value.values.toArray()); + assertEquals(complexPojo.grades.get("math"), value.grades.get("math")); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatTestBase.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatTestBase.java new file mode 100644 index 00000000..8a7f1e52 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatTestBase.java @@ -0,0 +1,289 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.databind.ObjectMapper; +import org.junit.After; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Collections; + +public class MessagePackDataformatTestBase +{ + protected MessagePackFactory factory; + protected ByteArrayOutputStream out; + protected ByteArrayInputStream in; + protected ObjectMapper objectMapper; + protected NormalPojo normalPojo; + protected NestedListPojo nestedListPojo; + protected NestedListComplexPojo1 nestedListComplexPojo1; + protected NestedListComplexPojo2 nestedListComplexPojo2; + protected StringPojo stringPojo; + protected TinyPojo tinyPojo; + protected ComplexPojo complexPojo; + + @Before + public void setup() + { + factory = new MessagePackFactory(); + objectMapper = new MessagePackMapper(factory); + out = new ByteArrayOutputStream(); + in = new ByteArrayInputStream(new byte[4096]); + + normalPojo = new NormalPojo(); + normalPojo.setS("komamitsu"); + normalPojo.bool = true; + normalPojo.i = Integer.MAX_VALUE; + normalPojo.l = Long.MIN_VALUE; + normalPojo.f = Float.MIN_VALUE; + normalPojo.d = Double.MAX_VALUE; + normalPojo.b = new byte[] {0x01, 0x02, (byte) 0xFE, (byte) 0xFF}; + normalPojo.bi = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + normalPojo.suit = Suit.HEART; + normalPojo.sMultibyte = "text文字"; + + nestedListPojo = new NestedListPojo(); + nestedListPojo.s = "a string"; + nestedListPojo.strs = Arrays.asList("string#1", "string#2", "string#3"); + + tinyPojo = new TinyPojo(); + tinyPojo.t = "t string"; + + nestedListComplexPojo1 = new NestedListComplexPojo1(); + nestedListComplexPojo1.s = "a string"; + nestedListComplexPojo1.foos = new ArrayList<>(); + nestedListComplexPojo1.foos.add(tinyPojo); + + nestedListComplexPojo2 = new NestedListComplexPojo2(); + nestedListComplexPojo2.foos = new ArrayList<>(); + nestedListComplexPojo2.foos.add(tinyPojo); + nestedListComplexPojo2.foos.add(tinyPojo); + nestedListComplexPojo2.s = "another string"; + + stringPojo = new StringPojo(); + stringPojo.shortSingleByte = "hello"; + stringPojo.longSingleByte = "helloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworldhelloworld"; + stringPojo.shortMultiByte = "こんにちは"; + stringPojo.longMultiByte = "こんにちは、世界！！こんにちは、世界！！こんにちは、世界！！こんにちは、世界！！こんにちは、世界！！"; + + complexPojo = new ComplexPojo(); + complexPojo.name = "komamitsu"; + complexPojo.age = 20; + complexPojo.grades = Collections.singletonMap("math", 97); + complexPojo.values = Arrays.asList("one", "two", "three"); + } + + @After + public void teardown() + { + if (in != null) { + try { + in.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + if (out != null) { + try { + out.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } + + public enum Suit + { + SPADE, HEART, DIAMOND, CLUB; + } + + public static class NestedListPojo + { + public String s; + public List strs; + } + + public static class ComplexPojo + { + public String name; + public int age; + public List values; + public Map grades; + } + + public static class TinyPojo + { + public String t; + } + + public static class NestedListComplexPojo1 + { + public String s; + public List foos; + } + + public static class NestedListComplexPojo2 + { + public List foos; + public String s; + } + + public static class StringPojo + { + public String shortSingleByte; + public String longSingleByte; + public String shortMultiByte; + public String longMultiByte; + } + + public static class NormalPojo + { + String s; + public boolean bool; + public int i; + public long l; + public Float f; + public Double d; + public byte[] b; + public BigInteger bi; + public Suit suit; + public String sMultibyte; + + public String getS() + { + return s; + } + + public void setS(String s) + { + this.s = s; + } + } + + public static class BinKeyPojo + { + public byte[] b; + public String s; + } + + public static class UsingCustomConstructorPojo + { + final String name; + final int age; + + public UsingCustomConstructorPojo(@JsonProperty("name") String name, @JsonProperty("age") int age) + { + this.name = name; + this.age = age; + } + + public String getName() + { + return name; + } + + public int getAge() + { + return age; + } + } + + @JsonIgnoreProperties({"foo", "bar"}) + public static class IgnoringPropertiesPojo + { + int code; + + @JsonIgnore + public String internal; + + public String external; + + @JsonIgnore + public void setCode(int c) + { + code = c; + } + + public int getCode() + { + return code; + } + } + + public static class ChangingPropertyNamesPojo + { + String name; + + @JsonProperty("name") + public String getTheName() + { + return name; + } + + public void setTheName(String n) + { + name = n; + } + } + + protected interface FileSetup + { + void setup(File f) + throws Exception; + } + + protected File createTempFile() + throws Exception + { + return createTempFile(null); + } + + protected File createTempFile(FileSetup fileSetup) + throws Exception + { + File tempFile = File.createTempFile("test", "msgpack"); + tempFile.deleteOnExit(); + if (fileSetup != null) { + fileSetup.setup(tempFile); + } + return tempFile; + } + + protected OutputStream createTempFileOutputStream() + throws IOException + { + File tempFile = File.createTempFile("test", "msgpack"); + tempFile.deleteOnExit(); + return new FileOutputStream(tempFile); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackFactoryTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackFactoryTest.java new file mode 100644 index 00000000..25ef0fe2 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackFactoryTest.java @@ -0,0 +1,198 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonEncoding; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonParser; +import tools.jackson.core.TSFBuilder; +import tools.jackson.core.TokenStreamFactory; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.ObjectMapper; +import org.junit.Test; +import org.msgpack.core.MessagePack; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertEquals; +import static org.hamcrest.MatcherAssert.assertThat; + +public class MessagePackFactoryTest + extends MessagePackDataformatTestBase +{ + @Test + public void testCreateGenerator() + throws IOException + { + JsonEncoding enc = JsonEncoding.UTF8; + JsonGenerator generator = factory.createGenerator(out, enc); + assertEquals(MessagePackGenerator.class, generator.getClass()); + } + + @Test + public void testCreateParser() + throws IOException + { + JsonParser parser = factory.createParser(in); + assertEquals(MessagePackParser.class, parser.getClass()); + } + + @Test + public void testCopyWithDefaultConfig() + throws IOException + { + MessagePackFactory messagePackFactory = new MessagePackFactory(); + ObjectMapper objectMapper = new MessagePackMapper(messagePackFactory); + + // Use the original ObjectMapper in advance + { + byte[] bytes = objectMapper.writeValueAsBytes(1234); + assertThat(objectMapper.readValue(bytes, Integer.class), is(1234)); + } + + // Copy the factory + TokenStreamFactory copiedFactory = messagePackFactory.copy(); + assertThat(copiedFactory, is(instanceOf(MessagePackFactory.class))); + MessagePackFactory copiedMessagePackFactory = (MessagePackFactory) copiedFactory; + + assertThat(copiedMessagePackFactory.getPackerConfig().isStr8FormatSupport(), is(true)); + assertThat(copiedMessagePackFactory.getExtTypeCustomDesers(), is(nullValue())); + + // Check the copied factory works fine + ObjectMapper copiedObjectMapper = new MessagePackMapper(copiedMessagePackFactory); + Map map = new HashMap<>(); + map.put("one", 1); + Map deserialized = copiedObjectMapper + .readValue(objectMapper.writeValueAsBytes(map), new TypeReference>() {}); + assertThat(deserialized.size(), is(1)); + assertThat(deserialized.get("one"), is(1)); + } + + @Test + public void testRebuildWithDefaultConfig() + throws IOException + { + MessagePackFactory messagePackFactory = new MessagePackFactory(); + TSFBuilder builder = messagePackFactory.rebuild(); + assertThat(builder, is(instanceOf(MessagePackFactoryBuilder.class))); + + MessagePackFactory rebuilt = (MessagePackFactory) builder.build(); + assertThat(rebuilt, is(not(sameInstance(messagePackFactory)))); + assertThat(rebuilt.getPackerConfig().isStr8FormatSupport(), is(true)); + assertThat(rebuilt.getExtTypeCustomDesers(), is(nullValue())); + + ObjectMapper rebuiltObjectMapper = new MessagePackMapper(rebuilt); + byte[] bytes = rebuiltObjectMapper.writeValueAsBytes(42); + assertThat(rebuiltObjectMapper.readValue(bytes, Integer.class), is(42)); + } + + @Test + public void testRebuildWithAdvancedConfig() + throws IOException + { + ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); + extTypeCustomDesers.addCustomDeser((byte) 42, + new ExtensionTypeCustomDeserializers.Deser() + { + @Override + public Object deserialize(byte[] data) + throws IOException + { + TinyPojo pojo = new TinyPojo(); + pojo.t = new String(data); + return pojo; + } + } + ); + MessagePack.PackerConfig packerConfig = new MessagePack.PackerConfig().withStr8FormatSupport(false); + MessagePackFactory messagePackFactory = new MessagePackFactory(packerConfig); + messagePackFactory.setExtTypeCustomDesers(extTypeCustomDesers); + + MessagePackFactory rebuilt = (MessagePackFactory) messagePackFactory.rebuild().build(); + assertThat(rebuilt, is(not(sameInstance(messagePackFactory)))); + assertThat(rebuilt.getPackerConfig().isStr8FormatSupport(), is(false)); + assertThat(rebuilt.getExtTypeCustomDesers().getDeser((byte) 42), is(notNullValue())); + assertThat(rebuilt.getExtTypeCustomDesers().getDeser((byte) 43), is(nullValue())); + } + + @Test + public void testSnapshotReturnsNewInstance() + { + MessagePackFactory messagePackFactory = new MessagePackFactory(); + TokenStreamFactory snapshot = messagePackFactory.snapshot(); + assertThat(snapshot, is(not(sameInstance(messagePackFactory)))); + assertThat(snapshot, is(instanceOf(MessagePackFactory.class))); + } + + @Test + public void testCopyWithAdvancedConfig() + throws IOException + { + ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); + extTypeCustomDesers.addCustomDeser((byte) 42, + new ExtensionTypeCustomDeserializers.Deser() + { + @Override + public Object deserialize(byte[] data) + throws IOException + { + TinyPojo pojo = new TinyPojo(); + pojo.t = new String(data); + return pojo; + } + } + ); + + MessagePack.PackerConfig msgpackPackerConfig = new MessagePack.PackerConfig().withStr8FormatSupport(false); + + MessagePackFactory messagePackFactory = new MessagePackFactory(msgpackPackerConfig); + messagePackFactory.setExtTypeCustomDesers(extTypeCustomDesers); + + ObjectMapper objectMapper = new MessagePackMapper(messagePackFactory); + + // Use the original ObjectMapper in advance + { + byte[] bytes = objectMapper.writeValueAsBytes(1234); + assertThat(objectMapper.readValue(bytes, Integer.class), is(1234)); + } + + // Copy the factory + TokenStreamFactory copiedFactory = messagePackFactory.copy(); + assertThat(copiedFactory, is(instanceOf(MessagePackFactory.class))); + MessagePackFactory copiedMessagePackFactory = (MessagePackFactory) copiedFactory; + + assertThat(copiedMessagePackFactory.getPackerConfig().isStr8FormatSupport(), is(false)); + assertThat(copiedMessagePackFactory.getExtTypeCustomDesers().getDeser((byte) 42), is(notNullValue())); + assertThat(copiedMessagePackFactory.getExtTypeCustomDesers().getDeser((byte) 43), is(nullValue())); + + // Check the copied factory works fine + ObjectMapper copiedObjectMapper = new MessagePackMapper(copiedMessagePackFactory); + Map map = new HashMap<>(); + map.put("one", 1); + Map deserialized = copiedObjectMapper + .readValue(objectMapper.writeValueAsBytes(map), new TypeReference>() {}); + assertThat(deserialized.size(), is(1)); + assertThat(deserialized.get("one"), is(1)); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java new file mode 100644 index 00000000..2b79bbf8 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java @@ -0,0 +1,1252 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonEncoding; +import tools.jackson.core.JacksonException; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.StreamWriteFeature; +import tools.jackson.core.TokenStreamContext; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ValueSerializer; +import tools.jackson.databind.annotation.JsonSerialize; +import tools.jackson.databind.module.SimpleModule; +import org.junit.Test; +import org.msgpack.core.ExtensionTypeHeader; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.core.buffer.ArrayBufferInput; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assert.assertThrows; +import static org.hamcrest.MatcherAssert.assertThat; + +public class MessagePackGeneratorTest + extends MessagePackDataformatTestBase +{ + @Test + public void testGeneratorShouldWriteObject() + throws IOException + { + Map hashMap = new HashMap(); + // #1 + hashMap.put("str", "komamitsu"); + // #2 + hashMap.put("boolean", true); + // #3 + hashMap.put("int", Integer.MAX_VALUE); + // #4 + hashMap.put("long", Long.MIN_VALUE); + // #5 + hashMap.put("float", 3.14159f); + // #6 + hashMap.put("double", 3.14159d); + // #7 + hashMap.put("bin", new byte[] {0x00, 0x01, (byte) 0xFE, (byte) 0xFF}); + // #8 + Map childObj = new HashMap(); + childObj.put("co_str", "child#0"); + childObj.put("co_int", 12345); + hashMap.put("childObj", childObj); + // #9 + List childArray = new ArrayList(); + childArray.add("child#1"); + childArray.add(1.23f); + hashMap.put("childArray", childArray); + // #10 + byte[] hello = "hello".getBytes("UTF-8"); + hashMap.put("ext", new MessagePackExtensionType((byte) 17, hello)); + + long bitmap = 0; + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(bytes)); + assertEquals(hashMap.size(), messageUnpacker.unpackMapHeader()); + for (int i = 0; i < hashMap.size(); i++) { + String key = messageUnpacker.unpackString(); + if (key.equals("str")) { + // #1 + assertEquals("komamitsu", messageUnpacker.unpackString()); + bitmap |= 0x1 << 0; + } + else if (key.equals("boolean")) { + // #2 + assertTrue(messageUnpacker.unpackBoolean()); + bitmap |= 0x1 << 1; + } + else if (key.equals("int")) { + // #3 + assertEquals(Integer.MAX_VALUE, messageUnpacker.unpackInt()); + bitmap |= 0x1 << 2; + } + else if (key.equals("long")) { + // #4 + assertEquals(Long.MIN_VALUE, messageUnpacker.unpackLong()); + bitmap |= 0x1 << 3; + } + else if (key.equals("float")) { + // #5 + assertEquals(3.14159f, messageUnpacker.unpackFloat(), 0.01f); + bitmap |= 0x1 << 4; + } + else if (key.equals("double")) { + // #6 + assertEquals(3.14159d, messageUnpacker.unpackDouble(), 0.01f); + bitmap |= 0x1 << 5; + } + else if (key.equals("bin")) { + // #7 + assertEquals(4, messageUnpacker.unpackBinaryHeader()); + assertEquals((byte) 0x00, messageUnpacker.unpackByte()); + assertEquals((byte) 0x01, messageUnpacker.unpackByte()); + assertEquals((byte) 0xFE, messageUnpacker.unpackByte()); + assertEquals((byte) 0xFF, messageUnpacker.unpackByte()); + bitmap |= 0x1 << 6; + } + else if (key.equals("childObj")) { + // #8 + assertEquals(2, messageUnpacker.unpackMapHeader()); + for (int j = 0; j < 2; j++) { + String childKey = messageUnpacker.unpackString(); + if (childKey.equals("co_str")) { + assertEquals("child#0", messageUnpacker.unpackString()); + bitmap |= 0x1 << 7; + } + else if (childKey.equals("co_int")) { + assertEquals(12345, messageUnpacker.unpackInt()); + bitmap |= 0x1 << 8; + } + else { + assertTrue(false); + } + } + } + else if (key.equals("childArray")) { + // #9 + assertEquals(2, messageUnpacker.unpackArrayHeader()); + assertEquals("child#1", messageUnpacker.unpackString()); + assertEquals(1.23f, messageUnpacker.unpackFloat(), 0.01f); + bitmap |= 0x1 << 9; + } + else if (key.equals("ext")) { + // #10 + ExtensionTypeHeader header = messageUnpacker.unpackExtensionTypeHeader(); + assertEquals(17, header.getType()); + assertEquals(5, header.getLength()); + ByteBuffer payload = ByteBuffer.allocate(header.getLength()); + payload.flip(); + payload.limit(payload.capacity()); + messageUnpacker.readPayload(payload); + payload.flip(); + assertArrayEquals("hello".getBytes(), payload.array()); + bitmap |= 0x1 << 10; + } + else { + assertTrue(false); + } + } + assertEquals(0x07FF, bitmap); + } + + @Test + public void testGeneratorShouldWriteArray() + throws IOException + { + List array = new ArrayList(); + // #1 + array.add("komamitsu"); + // #2 + array.add(Integer.MAX_VALUE); + // #3 + array.add(Long.MIN_VALUE); + // #4 + array.add(3.14159f); + // #5 + array.add(3.14159d); + // #6 + Map childObject = new HashMap(); + childObject.put("str", "foobar"); + childObject.put("num", 123456); + array.add(childObject); + // #7 + array.add(false); + + long bitmap = 0; + byte[] bytes = objectMapper.writeValueAsBytes(array); + MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(bytes)); + assertEquals(array.size(), messageUnpacker.unpackArrayHeader()); + // #1 + assertEquals("komamitsu", messageUnpacker.unpackString()); + // #2 + assertEquals(Integer.MAX_VALUE, messageUnpacker.unpackInt()); + // #3 + assertEquals(Long.MIN_VALUE, messageUnpacker.unpackLong()); + // #4 + assertEquals(3.14159f, messageUnpacker.unpackFloat(), 0.01f); + // #5 + assertEquals(3.14159d, messageUnpacker.unpackDouble(), 0.01f); + // #6 + assertEquals(2, messageUnpacker.unpackMapHeader()); + for (int i = 0; i < childObject.size(); i++) { + String key = messageUnpacker.unpackString(); + if (key.equals("str")) { + assertEquals("foobar", messageUnpacker.unpackString()); + bitmap |= 0x1 << 0; + } + else if (key.equals("num")) { + assertEquals(123456, messageUnpacker.unpackInt()); + bitmap |= 0x1 << 1; + } + else { + assertTrue(false); + } + } + assertEquals(0x3, bitmap); + // #7 + assertEquals(false, messageUnpacker.unpackBoolean()); + } + + @Test + public void testMessagePackGeneratorDirectly() + throws Exception + { + MessagePackFactory messagePackFactory = new MessagePackFactory(); + File tempFile = createTempFile(); + + JsonGenerator generator = messagePackFactory.createGenerator(ObjectWriteContext.empty(), tempFile, JsonEncoding.UTF8); + assertTrue(generator instanceof MessagePackGenerator); + generator.writeStartArray(); + generator.writeNumber(0); + generator.writeString("one"); + generator.writeNumber(2.0f); + generator.writeEndArray(); + generator.flush(); + generator.flush(); // intentional + generator.close(); + + FileInputStream fileInputStream = new FileInputStream(tempFile); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(fileInputStream); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals(0, unpacker.unpackInt()); + assertEquals("one", unpacker.unpackString()); + assertEquals(2.0f, unpacker.unpackFloat(), 0.001f); + assertFalse(unpacker.hasNext()); + } + + @Test + public void testWritePrimitives() + throws Exception + { + MessagePackFactory messagePackFactory = new MessagePackFactory(); + File tempFile = createTempFile(); + + JsonGenerator generator = messagePackFactory.createGenerator(ObjectWriteContext.empty(), tempFile, JsonEncoding.UTF8); + assertTrue(generator instanceof MessagePackGenerator); + generator.writeNumber(0); + generator.writeString("one"); + generator.writeNumber(2.0f); + generator.writeString("三"); + generator.writeString("444④"); + generator.flush(); + generator.close(); + + FileInputStream fileInputStream = new FileInputStream(tempFile); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(fileInputStream); + assertEquals(0, unpacker.unpackInt()); + assertEquals("one", unpacker.unpackString()); + assertEquals(2.0f, unpacker.unpackFloat(), 0.001f); + assertEquals("三", unpacker.unpackString()); + assertEquals("444④", unpacker.unpackString()); + assertFalse(unpacker.hasNext()); + } + + @Test + public void testBigDecimal() + throws IOException + { + ObjectMapper mapper = new MessagePackMapper(new MessagePackFactory()); + + { + double d0 = 1.23456789; + double d1 = 1.23450000000000000000006789; + String d2 = "12.30"; + String d3 = "0.00001"; + List bigDecimals = Arrays.asList( + BigDecimal.valueOf(d0), + BigDecimal.valueOf(d1), + new BigDecimal(d2), + new BigDecimal(d3), + BigDecimal.valueOf(Double.MIN_VALUE), + BigDecimal.valueOf(Double.MAX_VALUE), + BigDecimal.valueOf(Double.MIN_NORMAL) + ); + + byte[] bytes = mapper.writeValueAsBytes(bigDecimals); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes); + + assertEquals(bigDecimals.size(), unpacker.unpackArrayHeader()); + assertEquals(d0, unpacker.unpackDouble(), 0.000000000000001); + assertEquals(d1, unpacker.unpackDouble(), 0.000000000000001); + assertEquals(Double.valueOf(d2), unpacker.unpackDouble(), 0.000000000000001); + assertEquals(Double.valueOf(d3), unpacker.unpackDouble(), 0.000000000000001); + assertEquals(Double.MIN_VALUE, unpacker.unpackDouble(), 0.000000000000001); + assertEquals(Double.MAX_VALUE, unpacker.unpackDouble(), 0.000000000000001); + assertEquals(Double.MIN_NORMAL, unpacker.unpackDouble(), 0.000000000000001); + } + + { + BigDecimal decimal = new BigDecimal("1234.567890123456789012345678901234567890"); + List bigDecimals = Arrays.asList( + decimal + ); + + try { + mapper.writeValueAsBytes(bigDecimals); + assertTrue(false); + } + catch (IllegalArgumentException e) { + assertTrue(true); + } + } + } + + @Test + public void testBigDecimalCompareTo() + throws IOException + { + ObjectMapper mapper = new MessagePackMapper(new MessagePackFactory()); + + // BigDecimal with trailing zeros is representable as double — must not throw + BigDecimal trailingZeros = new BigDecimal("1.50"); + byte[] bytes = mapper.writeValueAsBytes(trailingZeros); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes); + assertEquals(1.5, unpacker.unpackDouble(), 0.0); + + // BigDecimal with precision beyond double range must throw + BigDecimal tooHighPrecision = new BigDecimal("1.00000000000000000000000000000000000001"); + try { + mapper.writeValueAsBytes(tooHighPrecision); + assertTrue(false); + } + catch (IllegalArgumentException e) { + assertTrue(true); + } + } + + @Test + public void testEnableFeatureAutoCloseTarget() + throws IOException + { + OutputStream out = createTempFileOutputStream(); + ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory()); + List integers = Arrays.asList(1); + objectMapper.writeValue(out, integers); + assertThrows(JacksonException.class, () -> { + objectMapper.writeValue(out, integers); + }); + } + + @Test + public void testDisableFeatureAutoCloseTarget() + throws Exception + { + File tempFile = createTempFile(); + OutputStream out = new FileOutputStream(tempFile); + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(StreamWriteFeature.AUTO_CLOSE_TARGET) + .build(); + List integers = Arrays.asList(1); + objectMapper.writeValue(out, integers); + objectMapper.writeValue(out, integers); + out.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new FileInputStream(tempFile)); + assertEquals(1, unpacker.unpackArrayHeader()); + assertEquals(1, unpacker.unpackInt()); + assertEquals(1, unpacker.unpackArrayHeader()); + assertEquals(1, unpacker.unpackInt()); + } + + @Test + public void testWritePrimitiveObjectViaObjectMapper() + throws Exception + { + File tempFile = createTempFile(); + try (OutputStream out = Files.newOutputStream(tempFile.toPath())) { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(StreamWriteFeature.AUTO_CLOSE_TARGET) + .build(); + objectMapper.writeValue(out, 1); + objectMapper.writeValue(out, "two"); + objectMapper.writeValue(out, 3.14); + objectMapper.writeValue(out, Arrays.asList(4)); + objectMapper.writeValue(out, 5L); + } + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new FileInputStream(tempFile))) { + assertEquals(1, unpacker.unpackInt()); + assertEquals("two", unpacker.unpackString()); + assertEquals(3.14, unpacker.unpackFloat(), 0.0001); + assertEquals(1, unpacker.unpackArrayHeader()); + assertEquals(4, unpacker.unpackInt()); + assertEquals(5, unpacker.unpackLong()); + } + } + + @Test + public void testInMultiThreads() + throws Exception + { + int threadCount = 8; + final int loopCount = 4000; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + final ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(StreamWriteFeature.AUTO_CLOSE_TARGET) + .build(); + final List buffers = new ArrayList(threadCount); + List> results = new ArrayList>(); + + for (int ti = 0; ti < threadCount; ti++) { + buffers.add(new ByteArrayOutputStream()); + final int threadIndex = ti; + results.add(executorService.submit(new Callable() + { + @Override + public Exception call() + throws Exception + { + try { + for (int i = 0; i < loopCount; i++) { + objectMapper.writeValue(buffers.get(threadIndex), threadIndex); + } + return null; + } + catch (Exception e) { + return e; + } + } + })); + } + + for (int ti = 0; ti < threadCount; ti++) { + Future exceptionFuture = results.get(ti); + Exception exception = exceptionFuture.get(20, TimeUnit.SECONDS); + if (exception != null) { + throw exception; + } + else { + try (ByteArrayOutputStream outputStream = buffers.get(ti); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(outputStream.toByteArray())) { + for (int i = 0; i < loopCount; i++) { + assertEquals(ti, unpacker.unpackInt()); + } + } + } + } + } + + @Test + public void testDisableStr8Support() + throws Exception + { + String str8LengthString = new String(new char[32]).replace("\0", "a"); + + ObjectMapper defaultMapper = new MessagePackMapper(new MessagePackFactory()); + byte[] resultWithStr8Format = defaultMapper.writeValueAsBytes(str8LengthString); + assertEquals(resultWithStr8Format[0], MessagePack.Code.STR8); + + MessagePack.PackerConfig config = new MessagePack.PackerConfig().withStr8FormatSupport(false); + ObjectMapper mapperWithConfig = new MessagePackMapper(new MessagePackFactory(config)); + byte[] resultWithoutStr8Format = mapperWithConfig.writeValueAsBytes(str8LengthString); + assertNotEquals(resultWithoutStr8Format[0], MessagePack.Code.STR8); + } + + interface NonStringKeyMapHolder + { + Map getIntMap(); + + void setIntMap(Map intMap); + + Map getLongMap(); + + void setLongMap(Map longMap); + + Map getFloatMap(); + + void setFloatMap(Map floatMap); + + Map getDoubleMap(); + + void setDoubleMap(Map doubleMap); + + Map getBigIntMap(); + + void setBigIntMap(Map doubleMap); + } + + public static class NonStringKeyMapHolderWithAnnotation + implements NonStringKeyMapHolder + { + @JsonSerialize(keyUsing = MessagePackKeySerializer.class) + private Map intMap = new HashMap(); + + @JsonSerialize(keyUsing = MessagePackKeySerializer.class) + private Map longMap = new HashMap(); + + @JsonSerialize(keyUsing = MessagePackKeySerializer.class) + private Map floatMap = new HashMap(); + + @JsonSerialize(keyUsing = MessagePackKeySerializer.class) + private Map doubleMap = new HashMap(); + + @JsonSerialize(keyUsing = MessagePackKeySerializer.class) + private Map bigIntMap = new HashMap(); + + @Override + public Map getIntMap() + { + return intMap; + } + + @Override + public void setIntMap(Map intMap) + { + this.intMap = intMap; + } + + @Override + public Map getLongMap() + { + return longMap; + } + + @Override + public void setLongMap(Map longMap) + { + this.longMap = longMap; + } + + @Override + public Map getFloatMap() + { + return floatMap; + } + + @Override + public void setFloatMap(Map floatMap) + { + this.floatMap = floatMap; + } + + @Override + public Map getDoubleMap() + { + return doubleMap; + } + + @Override + public void setDoubleMap(Map doubleMap) + { + this.doubleMap = doubleMap; + } + + @Override + public Map getBigIntMap() + { + return bigIntMap; + } + + @Override + public void setBigIntMap(Map bigIntMap) + { + this.bigIntMap = bigIntMap; + } + } + + public static class NonStringKeyMapHolderWithoutAnnotation + implements NonStringKeyMapHolder + { + private Map intMap = new HashMap(); + + private Map longMap = new HashMap(); + + private Map floatMap = new HashMap(); + + private Map doubleMap = new HashMap(); + + private Map bigIntMap = new HashMap(); + + @Override + public Map getIntMap() + { + return intMap; + } + + @Override + public void setIntMap(Map intMap) + { + this.intMap = intMap; + } + + @Override + public Map getLongMap() + { + return longMap; + } + + @Override + public void setLongMap(Map longMap) + { + this.longMap = longMap; + } + + @Override + public Map getFloatMap() + { + return floatMap; + } + + @Override + public void setFloatMap(Map floatMap) + { + this.floatMap = floatMap; + } + + @Override + public Map getDoubleMap() + { + return doubleMap; + } + + @Override + public void setDoubleMap(Map doubleMap) + { + this.doubleMap = doubleMap; + } + + @Override + public Map getBigIntMap() + { + return bigIntMap; + } + + @Override + public void setBigIntMap(Map bigIntMap) + { + this.bigIntMap = bigIntMap; + } + } + + @Test + @SuppressWarnings("unchecked") + public void testNonStringKey() + throws Exception + { + for (Class clazz : + Arrays.asList( + NonStringKeyMapHolderWithAnnotation.class, + NonStringKeyMapHolderWithoutAnnotation.class)) { + NonStringKeyMapHolder mapHolder = clazz.getConstructor().newInstance(); + mapHolder.getIntMap().put(Integer.MAX_VALUE, "i"); + mapHolder.getLongMap().put(Long.MIN_VALUE, "l"); + mapHolder.getFloatMap().put(Float.MAX_VALUE, "f"); + mapHolder.getDoubleMap().put(Double.MIN_VALUE, "d"); + mapHolder.getBigIntMap().put(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), "bi"); + + ObjectMapper objectMapper; + if (mapHolder instanceof NonStringKeyMapHolderWithoutAnnotation) { + SimpleModule mod = new SimpleModule("test"); + mod.addKeySerializer(Object.class, new MessagePackKeySerializer()); + objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(mod) + .build(); + } + else { + objectMapper = new MessagePackMapper(new MessagePackFactory()); + } + + byte[] bytes = objectMapper.writeValueAsBytes(mapHolder); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes); + assertEquals(5, unpacker.unpackMapHeader()); + for (int i = 0; i < 5; i++) { + String keyName = unpacker.unpackString(); + assertThat(unpacker.unpackMapHeader(), is(1)); + if (keyName.equals("intMap")) { + assertThat(unpacker.unpackInt(), is(Integer.MAX_VALUE)); + assertThat(unpacker.unpackString(), is("i")); + } + else if (keyName.equals("longMap")) { + assertThat(unpacker.unpackLong(), is(Long.MIN_VALUE)); + assertThat(unpacker.unpackString(), is("l")); + } + else if (keyName.equals("floatMap")) { + assertThat(unpacker.unpackFloat(), is(Float.MAX_VALUE)); + assertThat(unpacker.unpackString(), is("f")); + } + else if (keyName.equals("doubleMap")) { + assertThat(unpacker.unpackDouble(), is(Double.MIN_VALUE)); + assertThat(unpacker.unpackString(), is("d")); + } + else if (keyName.equals("bigIntMap")) { + assertThat(unpacker.unpackBigInteger(), is(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE))); + assertThat(unpacker.unpackString(), is("bi")); + } + else { + fail("Unexpected key name: " + keyName); + } + } + } + } + + @Test + public void testComplexTypeKey() + throws IOException + { + HashMap map = new HashMap(); + TinyPojo pojo = new TinyPojo(); + pojo.t = "foo"; + map.put(pojo, 42); + + SimpleModule mod = new SimpleModule("test"); + mod.addKeySerializer(TinyPojo.class, new MessagePackKeySerializer()); + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(mod) + .build(); + byte[] bytes = objectMapper.writeValueAsBytes(map); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes); + assertThat(unpacker.unpackMapHeader(), is(1)); + assertThat(unpacker.unpackMapHeader(), is(1)); + assertThat(unpacker.unpackString(), is("t")); + assertThat(unpacker.unpackString(), is("foo")); + assertThat(unpacker.unpackInt(), is(42)); + } + + @Test + public void testComplexTypeKeyWithV06Format() + throws IOException + { + HashMap map = new HashMap(); + TinyPojo pojo = new TinyPojo(); + pojo.t = "foo"; + map.put(pojo, 42); + + SimpleModule mod = new SimpleModule("test"); + mod.addKeySerializer(TinyPojo.class, new MessagePackKeySerializer()); + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .annotationIntrospector(new JsonArrayFormat()) + .addModule(mod) + .build(); + byte[] bytes = objectMapper.writeValueAsBytes(map); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes); + assertThat(unpacker.unpackMapHeader(), is(1)); + assertThat(unpacker.unpackArrayHeader(), is(1)); + assertThat(unpacker.unpackString(), is("foo")); + assertThat(unpacker.unpackInt(), is(42)); + } + + public static class IntegerSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(Integer value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsInteger() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(Integer.class, new IntegerSerializerStoringAsString())) + .build(); + + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(Integer.MAX_VALUE)).unpackInt(), + is(Integer.MAX_VALUE)); + } + + public static class LongSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(Long value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsLong() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(Long.class, new LongSerializerStoringAsString())) + .build(); + + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(Long.MIN_VALUE)).unpackLong(), + is(Long.MIN_VALUE)); + } + + public static class FloatSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(Float value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsFloat() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(Float.class, new FloatSerializerStoringAsString())) + .build(); + + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(Float.MAX_VALUE)).unpackFloat(), + is(Float.MAX_VALUE)); + } + + public static class DoubleSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(Double value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsDouble() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(Double.class, new DoubleSerializerStoringAsString())) + .build(); + + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(Double.MIN_VALUE)).unpackDouble(), + is(Double.MIN_VALUE)); + } + + public static class BigDecimalSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(BigDecimal value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsBigDecimal() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(BigDecimal.class, new BigDecimalSerializerStoringAsString())) + .build(); + + BigDecimal bd = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(bd)).unpackBigInteger(), + is(bd.toBigIntegerExact())); + } + + public static class BigIntegerSerializerStoringAsString + extends ValueSerializer + { + @Override + public void serialize(BigInteger value, JsonGenerator gen, SerializationContext serializers) + throws JacksonException + { + gen.writeNumber(String.valueOf(value)); + } + } + + @Test + public void serializeStringAsBigInteger() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule().addSerializer(BigInteger.class, new BigIntegerSerializerStoringAsString())) + .build(); + + BigInteger bi = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + assertThat( + MessagePack.newDefaultUnpacker(objectMapper.writeValueAsBytes(bi)).unpackBigInteger(), + is(bi)); + } + + @Test + public void testNestedSerialization() throws Exception + { + ObjectMapper objectMapper = new MessagePackMapper( + new MessagePackFactory().setReuseResourceInGenerator(false)); + OuterClass outerClass = objectMapper.readValue( + objectMapper.writeValueAsBytes(new OuterClass("Foo")), + OuterClass.class); + assertEquals("Foo", outerClass.getName()); + } + + static class OuterClass + { + private final String name; + + public OuterClass(@JsonProperty("name") String name) + { + this.name = name; + } + + public String getName() + { + ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory()); + InnerClass innerClass = objectMapper.readValue( + objectMapper.writeValueAsBytes(new InnerClass("Bar")), + InnerClass.class); + assertEquals("Bar", innerClass.getName()); + + return name; + } + } + + static class InnerClass + { + private final String name; + + public InnerClass(@JsonProperty("name") String name) + { + this.name = name; + } + + public String getName() + { + return name; + } + } + + @Test + public void testIsClosedAfterClose() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + assertFalse(generator.isClosed()); + generator.writeStartArray(); + generator.writeEndArray(); + generator.close(); + assertTrue(generator.isClosed()); + } + + @Test + public void testGeneratorReusableAfterRootContainerClose() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeNumber(1); + generator.writeEndArray(); + generator.flush(); + + // Write a second root value; currentState must have reset to IN_ROOT + generator.writeStartObject(); + generator.writeName("k"); + generator.writeNumber(2); + generator.writeEndObject(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + assertEquals(1, unpacker.unpackArrayHeader()); + assertEquals(1, unpacker.unpackInt()); + assertEquals(1, unpacker.unpackMapHeader()); + assertEquals("k", unpacker.unpackString()); + assertEquals(2, unpacker.unpackInt()); + } + + @Test + public void testWriteRawStringWithOffset() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeRaw("XXhelloXX", 2, 5); // "hello" + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + assertEquals("hello", unpacker.unpackString()); + } + + @Test + public void testWriteStringCharArrayWithOffset() + throws IOException + { + // Padding chars before/after the actual content to test non-zero offset + char[] buf = new char[] {'X', 'X', 'h', 'e', 'l', 'l', 'o', 'X'}; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeString(buf, 2, 5); // "hello" + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + assertEquals("hello", unpacker.unpackString()); + } + + @Test + public void testWriteStringCharArrayWithOffsetNonAscii() + throws IOException + { + // Non-ASCII to exercise the non-fast-path in getBytesIfAscii + char[] buf = new char[] {'X', '三', '四', '五', 'X'}; // 三四五 + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeString(buf, 1, 3); + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + assertEquals("三四五", unpacker.unpackString()); + } + + @Test + public void testWriteUTF8StringWithOffset() + throws IOException + { + // Padding bytes before/after to test non-zero offset in writeUTF8String + byte[] buf = new byte[] {'X', 'X', 'h', 'i', 'X'}; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeUTF8String(buf, 2, 2); // "hi" + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + assertEquals("hi", unpacker.unpackString()); + } + + @Test + public void testWriteBinaryWithOffset() + throws IOException + { + byte[] data = new byte[] {0x00, 0x01, 0x02, 0x03, 0x04}; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + generator.writeBinary(data, 1, 3); // bytes 0x01, 0x02, 0x03 + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + byte[] result = unpacker.readPayload(unpacker.unpackBinaryHeader()); + assertArrayEquals(new byte[] {0x01, 0x02, 0x03}, result); + } + + @Test + public void testWriteBinaryByteBufferWithOffset() + throws IOException + { + byte[] data = new byte[] {0x00, 0x01, 0x02, 0x03, 0x04}; + ByteBuffer bb = ByteBuffer.wrap(data, 1, 3); // position=1, limit=4, remaining=3 + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartArray(); + ObjectMapper mapper = new MessagePackMapper(factory); + mapper.writeValue(generator, bb); + generator.writeEndArray(); + generator.close(); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(baos.toByteArray()); + unpacker.unpackArrayHeader(); + byte[] result = unpacker.readPayload(unpacker.unpackBinaryHeader()); + assertArrayEquals(new byte[] {0x01, 0x02, 0x03}, result); + } + + @Test + public void testStreamWriteContext() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + + TokenStreamContext ctx = generator.streamWriteContext(); + assertNotEquals(null, ctx); + assertTrue(ctx.inRoot()); + + generator.writeStartArray(); + ctx = generator.streamWriteContext(); + assertTrue(ctx.inArray()); + + generator.writeStartObject(); + ctx = generator.streamWriteContext(); + assertTrue(ctx.inObject()); + + generator.writeName("k"); + assertEquals("k", ctx.currentName()); + + generator.writeNumber(1); + generator.writeEndObject(); + ctx = generator.streamWriteContext(); + assertTrue(ctx.inArray()); + + generator.writeEndArray(); + ctx = generator.streamWriteContext(); + assertTrue(ctx.inRoot()); + + generator.close(); + } + + @Test + public void testCurrentValue() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + + Object pojo = new Object(); + generator.writeStartObject(pojo); + assertEquals(pojo, generator.currentValue()); + generator.writeName("k"); + generator.writeNumber(1); + generator.writeEndObject(); + generator.close(); + } + + @Test + public void testVersion() + { + assertNotEquals(null, factory.version()); + assertEquals("org.msgpack", factory.version().getGroupId()); + assertEquals("msgpack-jackson3", factory.version().getArtifactId()); + } + + @Test + public void testSerializedStringMethods() throws IOException + { + MessagePackSerializedString s = new MessagePackSerializedString("hello"); + + byte[] utf8Target = new byte[10]; + int written = s.appendUnquotedUTF8(utf8Target, 2); + assertEquals(5, written); + assertArrayEquals(new byte[] {'h', 'e', 'l', 'l', 'o'}, Arrays.copyOfRange(utf8Target, 2, 7)); + + char[] charTarget = new char[10]; + written = s.appendUnquoted(charTarget, 3); + assertEquals(5, written); + assertEquals("hello", new String(charTarget, 3, 5)); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + written = s.writeUnquotedUTF8(baos); + assertEquals(5, written); + assertArrayEquals("hello".getBytes(java.nio.charset.StandardCharsets.UTF_8), baos.toByteArray()); + } + + // Regression: addValueNode must call writeContext.writeValue() so that the Jackson write + // context resets _gotPropertyId after each value. Without it, the second writeName() in the + // same object finds _gotPropertyId still set from the first writeName() and returns false + // without updating currentName(), leaving streamWriteContext().currentName() stale. + @Test + public void testWriteContextCurrentNameIsUpdatedForEveryProperty() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = factory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartObject(); + + generator.writeName("alpha"); + generator.writeNumber(1); + + generator.writeName("beta"); + assertEquals("beta", generator.streamWriteContext().currentName()); + generator.writeNumber(2); + + generator.writeEndObject(); + generator.close(); + } + + // Regression: writePropertyId() must call writeContext.writeName() when supportIntegerKeys + // is true. Without it, streamWriteContext() never learns a name was written, so currentName() + // returns null and any downstream code relying on context state (e.g. duplicate-name + // detection, error messages) sees wrong state. + @Test + public void testWritePropertyIdUpdatesWriteContext() + throws IOException + { + MessagePackFactory intKeyFactory = new MessagePackFactory().setSupportIntegerKeys(true); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = intKeyFactory.createGenerator(ObjectWriteContext.empty(), baos); + generator.writeStartObject(); + generator.writePropertyId(42L); + assertEquals("42", generator.streamWriteContext().currentName()); + generator.writeString("value"); + generator.writeEndObject(); + generator.close(); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackMapperTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackMapperTest.java new file mode 100644 index 00000000..776a1109 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackMapperTest.java @@ -0,0 +1,117 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JacksonException; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class MessagePackMapperTest +{ + static class PojoWithBigInteger + { + public BigInteger value; + } + + static class PojoWithBigDecimal + { + public BigDecimal value; + } + + private void shouldFailToHandleBigInteger(MessagePackMapper messagePackMapper) throws JacksonException + { + PojoWithBigInteger obj = new PojoWithBigInteger(); + obj.value = BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.valueOf(10)); + + try { + messagePackMapper.writeValueAsBytes(obj); + fail(); + } + catch (IllegalArgumentException e) { + // Expected + } + } + + private void shouldSuccessToHandleBigInteger(MessagePackMapper messagePackMapper) throws IOException + { + PojoWithBigInteger obj = new PojoWithBigInteger(); + obj.value = BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.valueOf(10)); + + byte[] converted = messagePackMapper.writeValueAsBytes(obj); + + PojoWithBigInteger deserialized = messagePackMapper.readValue(converted, PojoWithBigInteger.class); + assertEquals(obj.value, deserialized.value); + } + + private void shouldFailToHandleBigDecimal(MessagePackMapper messagePackMapper) throws JacksonException + { + PojoWithBigDecimal obj = new PojoWithBigDecimal(); + obj.value = new BigDecimal("1234567890.98765432100"); + + try { + messagePackMapper.writeValueAsBytes(obj); + fail(); + } + catch (IllegalArgumentException e) { + // Expected + } + } + + private void shouldSuccessToHandleBigDecimal(MessagePackMapper messagePackMapper) throws IOException + { + PojoWithBigDecimal obj = new PojoWithBigDecimal(); + obj.value = new BigDecimal("1234567890.98765432100"); + + byte[] converted = messagePackMapper.writeValueAsBytes(obj); + + PojoWithBigDecimal deserialized = messagePackMapper.readValue(converted, PojoWithBigDecimal.class); + assertEquals(obj.value, deserialized.value); + } + + @Test + public void handleBigIntegerAsString() throws IOException + { + shouldFailToHandleBigInteger(new MessagePackMapper()); + shouldSuccessToHandleBigInteger(MessagePackMapper.builder() + .handleBigIntegerAsString() + .build()); + } + + @Test + public void handleBigDecimalAsString() throws IOException + { + shouldFailToHandleBigDecimal(new MessagePackMapper()); + shouldSuccessToHandleBigDecimal(MessagePackMapper.builder() + .handleBigDecimalAsString() + .build()); + } + + @Test + public void handleBigIntegerAndBigDecimalAsString() throws IOException + { + MessagePackMapper messagePackMapper = MessagePackMapper.builder() + .handleBigIntegerAndBigDecimalAsString() + .build(); + shouldSuccessToHandleBigInteger(messagePackMapper); + shouldSuccessToHandleBigDecimal(messagePackMapper); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java new file mode 100644 index 00000000..c2fd3ae0 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java @@ -0,0 +1,1132 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonParser; +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonToken; +import tools.jackson.core.exc.UnexpectedEndOfInputException; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.KeyDeserializer; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.deser.std.StdDeserializer; +import tools.jackson.databind.module.SimpleModule; +import org.junit.Test; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.value.ExtensionValue; +import org.msgpack.value.MapValue; +import org.msgpack.value.ValueFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; + +public class MessagePackParserTest + extends MessagePackDataformatTestBase +{ + @Test + public void testParserShouldReadObject() + throws IOException + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packMapHeader(9); + // #1 + packer.packString("str"); + packer.packString("foobar"); + // #2 + packer.packString("int"); + packer.packInt(Integer.MIN_VALUE); + // #3 + packer.packString("map"); + { + packer.packMapHeader(2); + packer.packString("child_str"); + packer.packString("bla bla bla"); + packer.packString("child_int"); + packer.packInt(Integer.MAX_VALUE); + } + // #4 + packer.packString("double"); + packer.packDouble(Double.MAX_VALUE); + // #5 + packer.packString("long"); + packer.packLong(Long.MIN_VALUE); + // #6 + packer.packString("bi"); + BigInteger bigInteger = new BigInteger(Long.toString(Long.MAX_VALUE)); + packer.packBigInteger(bigInteger.add(BigInteger.ONE)); + // #7 + packer.packString("array"); + { + packer.packArrayHeader(3); + packer.packFloat(Float.MIN_VALUE); + packer.packNil(); + packer.packString("array_child_str"); + } + // #8 + packer.packString("bool"); + packer.packBoolean(false); + // #9 + byte[] extPayload = {-80, -50, -25, -114, -25, 16, 60, 68}; + packer.packString("ext"); + packer.packExtensionTypeHeader((byte) 0, extPayload.length); + packer.writePayload(extPayload); + + packer.flush(); + + byte[] bytes = out.toByteArray(); + + TypeReference> typeReference = new TypeReference>() {}; + Map object = objectMapper.readValue(bytes, typeReference); + assertEquals(9, object.keySet().size()); + + int bitmap = 0; + for (Map.Entry entry : object.entrySet()) { + String k = entry.getKey(); + Object v = entry.getValue(); + if (k.equals("str")) { + // #1 + bitmap |= 1 << 0; + assertEquals("foobar", v); + } + else if (k.equals("int")) { + // #2 + bitmap |= 1 << 1; + assertEquals(Integer.MIN_VALUE, v); + } + else if (k.equals("map")) { + // #3 + bitmap |= 1 << 2; + @SuppressWarnings("unchecked") + Map child = (Map) v; + assertEquals(2, child.keySet().size()); + for (Map.Entry childEntry : child.entrySet()) { + String ck = childEntry.getKey(); + Object cv = childEntry.getValue(); + if (ck.equals("child_str")) { + bitmap |= 1 << 3; + assertEquals("bla bla bla", cv); + } + else if (ck.equals("child_int")) { + bitmap |= 1 << 4; + assertEquals(Integer.MAX_VALUE, cv); + } + } + } + else if (k.equals("double")) { + // #4 + bitmap |= 1 << 5; + assertEquals(Double.MAX_VALUE, (Double) v, 0.0001f); + } + else if (k.equals("long")) { + // #5 + bitmap |= 1 << 6; + assertEquals(Long.MIN_VALUE, v); + } + else if (k.equals("bi")) { + // #6 + bitmap |= 1 << 7; + BigInteger bi = new BigInteger(Long.toString(Long.MAX_VALUE)); + assertEquals(bi.add(BigInteger.ONE), v); + } + else if (k.equals("array")) { + // #7 + bitmap |= 1 << 8; + @SuppressWarnings("unchecked") + List expected = Arrays.asList((double) Float.MIN_VALUE, null, "array_child_str"); + assertEquals(expected, v); + } + else if (k.equals("bool")) { + // #8 + bitmap |= 1 << 9; + assertEquals(false, v); + } + else if (k.equals("ext")) { + // #9 + bitmap |= 1 << 10; + MessagePackExtensionType extensionType = (MessagePackExtensionType) v; + assertEquals(0, extensionType.getType()); + assertArrayEquals(extPayload, extensionType.getData()); + } + } + assertEquals(0x7FF, bitmap); + } + + @Test + public void testParserShouldReadArray() + throws IOException + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packArrayHeader(11); + // #1 + packer.packArrayHeader(3); + { + packer.packLong(Long.MAX_VALUE); + packer.packNil(); + packer.packString("FOO BAR"); + } + // #2 + packer.packString("str"); + // #3 + packer.packInt(Integer.MAX_VALUE); + // #4 + packer.packLong(Long.MIN_VALUE); + // #5 + packer.packFloat(Float.MAX_VALUE); + // #6 + packer.packDouble(Double.MIN_VALUE); + // #7 + BigInteger bi = new BigInteger(Long.toString(Long.MAX_VALUE)); + bi = bi.add(BigInteger.ONE); + packer.packBigInteger(bi); + // #8 + byte[] bytes = new byte[] {(byte) 0xFF, (byte) 0xFE, 0x01, 0x00}; + packer.packBinaryHeader(bytes.length); + packer.writePayload(bytes); + // #9 + packer.packMapHeader(2); + { + packer.packString("child_map_name"); + packer.packString("komamitsu"); + packer.packString("child_map_age"); + packer.packInt(42); + } + // #10 + packer.packBoolean(true); + // #11 + byte[] extPayload = {-80, -50, -25, -114, -25, 16, 60, 68}; + packer.packExtensionTypeHeader((byte) -1, extPayload.length); + packer.writePayload(extPayload); + + packer.flush(); + + bytes = out.toByteArray(); + + TypeReference> typeReference = new TypeReference>() {}; + List array = objectMapper.readValue(bytes, typeReference); + assertEquals(11, array.size()); + int i = 0; + // #1 + @SuppressWarnings("unchecked") + List childArray = (List) array.get(i++); + { + int j = 0; + assertEquals(Long.MAX_VALUE, childArray.get(j++)); + assertEquals(null, childArray.get(j++)); + assertEquals("FOO BAR", childArray.get(j++)); + } + // #2 + assertEquals("str", array.get(i++)); + // #3 + assertEquals(Integer.MAX_VALUE, array.get(i++)); + // #4 + assertEquals(Long.MIN_VALUE, array.get(i++)); + // #5 + assertEquals(Float.MAX_VALUE, (Double) array.get(i++), 0.001f); + // #6 + assertEquals(Double.MIN_VALUE, (Double) array.get(i++), 0.001f); + // #7 + assertEquals(bi, array.get(i++)); + // #8 + byte[] bs = (byte[]) array.get(i++); + assertEquals(4, bs.length); + assertEquals((byte) 0xFF, bs[0]); + assertEquals((byte) 0xFE, bs[1]); + assertEquals((byte) 0x01, bs[2]); + assertEquals((byte) 0x00, bs[3]); + // #9 + @SuppressWarnings("unchecked") + Map childMap = (Map) array.get(i++); + { + assertEquals(2, childMap.keySet().size()); + for (Map.Entry entry : childMap.entrySet()) { + String k = entry.getKey(); + Object v = entry.getValue(); + if (k.equals("child_map_name")) { + assertEquals("komamitsu", v); + } + else if (k.equals("child_map_age")) { + assertEquals(42, v); + } + } + } + // #10 + assertEquals(true, array.get(i++)); + // #11 + MessagePackExtensionType extensionType = (MessagePackExtensionType) array.get(i++); + assertEquals(-1, extensionType.getType()); + assertArrayEquals(extPayload, extensionType.getData()); + } + + @Test + public void testMessagePackParserDirectly() + throws IOException + { + MessagePackFactory factory = new MessagePackFactory(); + File tempFile = File.createTempFile("msgpackTest", "msgpack"); + tempFile.deleteOnExit(); + + FileOutputStream fileOutputStream = new FileOutputStream(tempFile); + MessagePacker packer = MessagePack.newDefaultPacker(fileOutputStream); + packer.packMapHeader(2); + packer.packString("zero"); + packer.packInt(0); + packer.packString("one"); + packer.packFloat(1.0f); + packer.close(); + + JsonParser parser = factory.createParser(tempFile); + assertTrue(parser instanceof MessagePackParser); + + JsonToken jsonToken = parser.nextToken(); + assertEquals(JsonToken.START_OBJECT, jsonToken); + assertEquals(-1, parser.currentTokenLocation().getLineNr()); + assertEquals(0, parser.currentTokenLocation().getColumnNr()); + assertEquals(-1, parser.currentLocation().getLineNr()); + assertEquals(1, parser.currentLocation().getColumnNr()); + + jsonToken = parser.nextToken(); + assertEquals(JsonToken.PROPERTY_NAME, jsonToken); + assertEquals("zero", parser.currentName()); + assertEquals(1, parser.currentTokenLocation().getColumnNr()); + assertEquals(6, parser.currentLocation().getColumnNr()); + + jsonToken = parser.nextToken(); + assertEquals(JsonToken.VALUE_NUMBER_INT, jsonToken); + assertEquals(0, parser.getIntValue()); + assertEquals(6, parser.currentTokenLocation().getColumnNr()); + assertEquals(7, parser.currentLocation().getColumnNr()); + + jsonToken = parser.nextToken(); + assertEquals(JsonToken.PROPERTY_NAME, jsonToken); + assertEquals("one", parser.currentName()); + assertEquals(7, parser.currentTokenLocation().getColumnNr()); + assertEquals(11, parser.currentLocation().getColumnNr()); + + jsonToken = parser.nextToken(); + assertEquals(JsonToken.VALUE_NUMBER_FLOAT, jsonToken); + assertEquals(1.0f, parser.getIntValue(), 0.001f); + assertEquals(11, parser.currentTokenLocation().getColumnNr()); + assertEquals(16, parser.currentLocation().getColumnNr()); + + jsonToken = parser.nextToken(); + assertEquals(JsonToken.END_OBJECT, jsonToken); + assertEquals(-1, parser.currentTokenLocation().getLineNr()); + assertEquals(16, parser.currentTokenLocation().getColumnNr()); + assertEquals(-1, parser.currentLocation().getLineNr()); + assertEquals(16, parser.currentLocation().getColumnNr()); + + parser.close(); + parser.close(); // Intentional + } + + @Test + public void testReadPrimitives() + throws Exception + { + MessagePackFactory factory = new MessagePackFactory(); + File tempFile = createTempFile(); + + FileOutputStream out = new FileOutputStream(tempFile); + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packString("foo"); + packer.packDouble(3.14); + packer.packInt(Integer.MIN_VALUE); + packer.packLong(Long.MAX_VALUE); + packer.packBigInteger(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE)); + byte[] bytes = {0x00, 0x11, 0x22}; + packer.packBinaryHeader(bytes.length); + packer.writePayload(bytes); + packer.close(); + + JsonParser parser = factory.createParser(new FileInputStream(tempFile)); + assertEquals(JsonToken.VALUE_STRING, parser.nextToken()); + assertEquals("foo", parser.getString()); + + assertEquals(JsonToken.VALUE_NUMBER_FLOAT, parser.nextToken()); + assertEquals(3.14, parser.getDoubleValue(), 0.0001); + assertEquals("3.14", parser.getString()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, parser.nextToken()); + assertEquals(Integer.MIN_VALUE, parser.getIntValue()); + assertEquals(Integer.MIN_VALUE, parser.getLongValue()); + assertEquals("-2147483648", parser.getString()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, parser.nextToken()); + assertEquals(Long.MAX_VALUE, parser.getLongValue()); + assertEquals("9223372036854775807", parser.getString()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, parser.nextToken()); + assertEquals(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), parser.getBigIntegerValue()); + assertEquals("9223372036854775808", parser.getString()); + + assertEquals(JsonToken.VALUE_EMBEDDED_OBJECT, parser.nextToken()); + assertEquals(bytes.length, parser.getBinaryValue().length); + assertEquals(bytes[0], parser.getBinaryValue()[0]); + assertEquals(bytes[1], parser.getBinaryValue()[1]); + assertEquals(bytes[2], parser.getBinaryValue()[2]); + } + + @Test + public void testBigDecimal() + throws IOException + { + double d0 = 1.23456789; + double d1 = 1.23450000000000000000006789; + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packArrayHeader(5); + packer.packDouble(d0); + packer.packDouble(d1); + packer.packDouble(Double.MIN_VALUE); + packer.packDouble(Double.MAX_VALUE); + packer.packDouble(Double.MIN_NORMAL); + packer.flush(); + + ObjectMapper mapper = MessagePackMapper.builder(new MessagePackFactory()) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .build(); + List objects = mapper.readValue(out.toByteArray(), new TypeReference>() {}); + assertEquals(5, objects.size()); + int idx = 0; + assertEquals(BigDecimal.valueOf(d0), objects.get(idx++)); + assertEquals(BigDecimal.valueOf(d1), objects.get(idx++)); + assertEquals(BigDecimal.valueOf(Double.MIN_VALUE), objects.get(idx++)); + assertEquals(BigDecimal.valueOf(Double.MAX_VALUE), objects.get(idx++)); + assertEquals(BigDecimal.valueOf(Double.MIN_NORMAL), objects.get(idx++)); + } + + private File createTestFile() + throws Exception + { + File tempFile = createTempFile(new FileSetup() + { + @Override + public void setup(File f) + throws IOException + { + MessagePack.newDefaultPacker(new FileOutputStream(f)) + .packArrayHeader(1).packInt(1) + .packArrayHeader(1).packInt(1) + .close(); + } + }); + return tempFile; + } + + @Test + public void testEnableFeatureAutoCloseSource() + throws Exception + { + File tempFile = createTestFile(); + MessagePackFactory factory = new MessagePackFactory(); + FileInputStream in = new FileInputStream(tempFile); + ObjectMapper objectMapper = MessagePackMapper.builder(factory) + .disable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .build(); + objectMapper.readValue(in, new TypeReference>() {}); + assertThrows(JacksonException.class, () -> { + objectMapper.readValue(in, new TypeReference>() {}); + }); + } + + @Test + public void testDisableFeatureAutoCloseSource() + throws Exception + { + File tempFile = createTestFile(); + FileInputStream in = new FileInputStream(tempFile); + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(tools.jackson.core.StreamReadFeature.AUTO_CLOSE_SOURCE) + .disable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .build(); + objectMapper.readValue(in, new TypeReference>() {}); + objectMapper.readValue(in, new TypeReference>() {}); + } + + @Test + public void testParseBigDecimal() + throws IOException + { + ArrayList list = new ArrayList(); + list.add(new BigDecimal(Long.MAX_VALUE)); + ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(list); + + ArrayList result = objectMapper.readValue( + bytes, new TypeReference>() {}); + assertEquals(list, result); + } + + @Test + public void testReadPrimitiveObjectViaObjectMapper() + throws Exception + { + File tempFile = createTempFile(); + FileOutputStream out = new FileOutputStream(tempFile); + + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packString("foo"); + packer.packLong(Long.MAX_VALUE); + packer.packDouble(3.14); + byte[] bytes = {0x00, 0x11, 0x22}; + packer.packBinaryHeader(bytes.length); + packer.writePayload(bytes); + packer.close(); + + FileInputStream in = new FileInputStream(tempFile); + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(tools.jackson.core.StreamReadFeature.AUTO_CLOSE_SOURCE) + .disable(tools.jackson.databind.DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .build(); + assertEquals("foo", objectMapper.readValue(in, new TypeReference() {})); + long l = objectMapper.readValue(in, new TypeReference() {}); + assertEquals(Long.MAX_VALUE, l); + double d = objectMapper.readValue(in, new TypeReference() {}); + assertEquals(3.14, d, 0.001); + byte[] bs = objectMapper.readValue(in, new TypeReference() {}); + assertEquals(bytes.length, bs.length); + assertEquals(bytes[0], bs[0]); + assertEquals(bytes[1], bs[1]); + assertEquals(bytes[2], bs[2]); + } + + @Test + public void testBinaryKey() + throws Exception + { + File tempFile = createTempFile(); + FileOutputStream out = new FileOutputStream(tempFile); + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packMapHeader(2); + packer.packString("foo"); + packer.packDouble(3.14); + byte[] bytes = "bar".getBytes(); + packer.packBinaryHeader(bytes.length); + packer.writePayload(bytes); + packer.packLong(42); + packer.close(); + + ObjectMapper mapper = new MessagePackMapper(new MessagePackFactory()); + Map object = mapper.readValue(new FileInputStream(tempFile), new TypeReference>() {}); + assertEquals(2, object.size()); + assertEquals(3.14, object.get("foo")); + assertEquals(42, object.get("bar")); + } + + @Test + public void testBinaryKeyInNestedObject() + throws Exception + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packArrayHeader(2); + packer.packMapHeader(1); + byte[] bytes = "bar".getBytes(); + packer.packBinaryHeader(bytes.length); + packer.writePayload(bytes); + packer.packInt(12); + packer.packInt(1); + packer.close(); + + ObjectMapper mapper = new MessagePackMapper(new MessagePackFactory()); + List objects = mapper.readValue(out.toByteArray(), new TypeReference>() {}); + assertEquals(2, objects.size()); + @SuppressWarnings(value = "unchecked") + Map map = (Map) objects.get(0); + assertEquals(1, map.size()); + assertEquals(12, map.get("bar")); + assertEquals(1, objects.get(1)); + } + + @Test + public void testByteArrayKey() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(out).packMapHeader(2); + byte[] k0 = new byte[] {0}; + byte[] k1 = new byte[] {1}; + messagePacker.packBinaryHeader(1).writePayload(k0).packInt(10); + messagePacker.packBinaryHeader(1).writePayload(k1).packInt(11); + messagePacker.close(); + + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule() + .addKeyDeserializer(byte[].class, new KeyDeserializer() + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws JacksonException + { + return key.getBytes(); + } + })) + .build(); + + Map map = objectMapper.readValue( + out.toByteArray(), new TypeReference>() {}); + assertEquals(2, map.size()); + for (Map.Entry entry : map.entrySet()) { + if (Arrays.equals(entry.getKey(), k0)) { + assertEquals((Integer) 10, entry.getValue()); + } + else if (Arrays.equals(entry.getKey(), k1)) { + assertEquals((Integer) 11, entry.getValue()); + } + } + } + + @Test + public void testIntegerKey() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(out).packMapHeader(2); + for (int i = 0; i < 2; i++) { + messagePacker.packInt(i).packInt(i + 10); + } + messagePacker.close(); + + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule() + .addKeyDeserializer(Integer.class, new KeyDeserializer() + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws JacksonException + { + return Integer.valueOf(key); + } + })) + .build(); + + Map map = objectMapper.readValue( + out.toByteArray(), new TypeReference>() {}); + assertEquals(2, map.size()); + assertEquals((Integer) 10, map.get(0)); + assertEquals((Integer) 11, map.get(1)); + } + + @Test + public void testFloatKey() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(out).packMapHeader(2); + for (int i = 0; i < 2; i++) { + messagePacker.packFloat(i).packInt(i + 10); + } + messagePacker.close(); + + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule() + .addKeyDeserializer(Float.class, new KeyDeserializer() + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws JacksonException + { + return Float.valueOf(key); + } + })) + .build(); + + Map map = objectMapper.readValue( + out.toByteArray(), new TypeReference>() {}); + assertEquals(2, map.size()); + assertEquals((Integer) 10, map.get(0f)); + assertEquals((Integer) 11, map.get(1f)); + } + + @Test + public void testBooleanKey() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(out).packMapHeader(2); + messagePacker.packBoolean(true).packInt(10); + messagePacker.packBoolean(false).packInt(11); + messagePacker.close(); + + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(new SimpleModule() + .addKeyDeserializer(Boolean.class, new KeyDeserializer() + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws JacksonException + { + return Boolean.valueOf(key); + } + })) + .build(); + + Map map = objectMapper.readValue( + out.toByteArray(), new TypeReference>() {}); + assertEquals(2, map.size()); + assertEquals((Integer) 10, map.get(true)); + assertEquals((Integer) 11, map.get(false)); + } + + @Test + public void extensionTypeCustomDeserializers() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packArrayHeader(3); + // 0: Integer + packer.packInt(42); + // 1: String + packer.packString("foo bar"); + // 2: ExtensionType + { + packer.packExtensionTypeHeader((byte) 31, 4); + packer.addPayload(new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}); + } + packer.close(); + + ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); + extTypeCustomDesers.addCustomDeser((byte) 31, new ExtensionTypeCustomDeserializers.Deser() { + @Override + public Object deserialize(byte[] data) + throws IOException + { + if (Arrays.equals(data, new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE})) { + return "Java"; + } + return "Not Java"; + } + } + ); + ObjectMapper objectMapper = + new MessagePackMapper(new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)); + + List values = objectMapper.readValue(new ByteArrayInputStream(out.toByteArray()), new TypeReference>() {}); + assertThat(values.size(), is(3)); + assertThat((Integer) values.get(0), is(42)); + assertThat((String) values.get(1), is("foo bar")); + assertThat((String) values.get(2), is("Java")); + } + + static class TripleBytesPojo + { + public byte first; + public byte second; + public byte third; + + public TripleBytesPojo(byte first, byte second, byte third) + { + this.first = first; + this.second = second; + this.third = third; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof TripleBytesPojo)) { + return false; + } + + TripleBytesPojo that = (TripleBytesPojo) o; + + if (first != that.first) { + return false; + } + if (second != that.second) { + return false; + } + return third == that.third; + } + + @Override + public int hashCode() + { + int result = first; + result = 31 * result + (int) second; + result = 31 * result + (int) third; + return result; + } + + @Override + public String toString() + { + return String.format("%d-%d-%d", first, second, third); + } + + static class Deserializer + extends StdDeserializer + { + protected Deserializer() + { + super(TripleBytesPojo.class); + } + + @Override + public TripleBytesPojo deserialize(JsonParser p, DeserializationContext ctxt) + throws JacksonException + { + return TripleBytesPojo.deserialize(p.getBinaryValue()); + } + } + + static class TripleBytesKeyDeserializer + extends KeyDeserializer + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws JacksonException + { + String[] values = key.split("-"); + return new TripleBytesPojo( + Byte.parseByte(values[0]), + Byte.parseByte(values[1]), + Byte.parseByte(values[2])); + } + } + + static byte[] serialize(TripleBytesPojo obj) + { + return new byte[] { obj.first, obj.second, obj.third }; + } + + static TripleBytesPojo deserialize(byte[] bytes) + { + return new TripleBytesPojo(bytes[0], bytes[1], bytes[2]); + } + } + + @Test + public void extensionTypeWithPojoInMap() + throws IOException + { + byte extTypeCode = 42; + + ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); + extTypeCustomDesers.addCustomDeser(extTypeCode, new ExtensionTypeCustomDeserializers.Deser() + { + @Override + public Object deserialize(byte[] value) + throws IOException + { + return TripleBytesPojo.deserialize(value); + } + }); + + SimpleModule module = new SimpleModule(); + module.addDeserializer(TripleBytesPojo.class, new TripleBytesPojo.Deserializer()); + module.addKeyDeserializer(TripleBytesPojo.class, new TripleBytesPojo.TripleBytesKeyDeserializer()); + ObjectMapper objectMapper = MessagePackMapper.builder( + new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)) + .addModule(module) + .build(); + + // Prepare serialized data + Map originalMap = new HashMap<>(); + byte[] serializedData; + { + ValueFactory.MapBuilder mapBuilder = ValueFactory.newMapBuilder(); + for (int i = 0; i < 4; i++) { + TripleBytesPojo keyObj = new TripleBytesPojo((byte) i, (byte) (i + 1), (byte) (i + 2)); + TripleBytesPojo valueObj = new TripleBytesPojo((byte) (i * 2), (byte) (i * 3), (byte) (i * 4)); + ExtensionValue k = ValueFactory.newExtension(extTypeCode, TripleBytesPojo.serialize(keyObj)); + ExtensionValue v = ValueFactory.newExtension(extTypeCode, TripleBytesPojo.serialize(valueObj)); + mapBuilder.put(k, v); + originalMap.put(keyObj, valueObj); + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(output); + MapValue mapValue = mapBuilder.build(); + mapValue.writeTo(packer); + packer.close(); + + serializedData = output.toByteArray(); + } + + Map deserializedMap = objectMapper.readValue(serializedData, + new TypeReference>() {}); + + assertEquals(originalMap.size(), deserializedMap.size()); + for (Map.Entry entry : originalMap.entrySet()) { + assertEquals(entry.getValue(), deserializedMap.get(entry.getKey())); + } + } + + @Test + public void extensionTypeWithUuidInMap() + throws IOException + { + byte extTypeCode = 42; + + ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); + extTypeCustomDesers.addCustomDeser(extTypeCode, new ExtensionTypeCustomDeserializers.Deser() + { + @Override + public Object deserialize(byte[] value) + throws IOException + { + return UUID.fromString(new String(value)); + } + }); + + ObjectMapper objectMapper = + new MessagePackMapper(new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)); + + // Prepare serialized data + Map originalMap = new HashMap<>(); + byte[] serializedData; + { + ValueFactory.MapBuilder mapBuilder = ValueFactory.newMapBuilder(); + for (int i = 0; i < 4; i++) { + UUID keyObj = UUID.randomUUID(); + UUID valueObj = UUID.randomUUID(); + ExtensionValue k = ValueFactory.newExtension(extTypeCode, keyObj.toString().getBytes()); + ExtensionValue v = ValueFactory.newExtension(extTypeCode, valueObj.toString().getBytes()); + mapBuilder.put(k, v); + originalMap.put(keyObj, valueObj); + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(output); + MapValue mapValue = mapBuilder.build(); + mapValue.writeTo(packer); + packer.close(); + + serializedData = output.toByteArray(); + } + + Map deserializedMap = objectMapper.readValue(serializedData, + new TypeReference>() {}); + + assertEquals(originalMap.size(), deserializedMap.size()); + for (Map.Entry entry : originalMap.entrySet()) { + assertEquals(entry.getValue(), deserializedMap.get(entry.getKey())); + } + } + + @Test + public void parserShouldReadStrAsBin() + throws IOException + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packMapHeader(2); + // #1 + packer.packString("s"); + packer.packString("foo"); + // #2 + packer.packString("b"); + packer.packString("bar"); + + packer.flush(); + + byte[] bytes = out.toByteArray(); + + BinKeyPojo binKeyPojo = objectMapper.readValue(bytes, BinKeyPojo.class); + assertEquals("foo", binKeyPojo.s); + assertArrayEquals("bar".getBytes(), binKeyPojo.b); + } + + @Test + public void deserializeStringAsInteger() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePack.newDefaultPacker(out).packString(String.valueOf(Integer.MAX_VALUE)).close(); + + Integer v = objectMapper.readValue(out.toByteArray(), Integer.class); + assertThat(v, is(Integer.MAX_VALUE)); + } + + @Test + public void deserializeStringAsLong() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePack.newDefaultPacker(out).packString(String.valueOf(Long.MIN_VALUE)).close(); + + Long v = objectMapper.readValue(out.toByteArray(), Long.class); + assertThat(v, is(Long.MIN_VALUE)); + } + + @Test + public void deserializeStringAsFloat() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePack.newDefaultPacker(out).packString(String.valueOf(Float.MAX_VALUE)).close(); + + Float v = objectMapper.readValue(out.toByteArray(), Float.class); + assertThat(v, is(Float.MAX_VALUE)); + } + + @Test + public void deserializeStringAsDouble() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePack.newDefaultPacker(out).packString(String.valueOf(Double.MIN_VALUE)).close(); + + Double v = objectMapper.readValue(out.toByteArray(), Double.class); + assertThat(v, is(Double.MIN_VALUE)); + } + + @Test + public void deserializeStringAsBigInteger() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BigInteger bi = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + MessagePack.newDefaultPacker(out).packString(bi.toString()).close(); + + BigInteger v = objectMapper.readValue(out.toByteArray(), BigInteger.class); + assertThat(v, is(bi)); + } + + @Test + public void deserializeStringAsBigDecimal() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BigDecimal bd = BigDecimal.valueOf(Double.MAX_VALUE); + MessagePack.newDefaultPacker(out).packString(bd.toString()).close(); + + BigDecimal v = objectMapper.readValue(out.toByteArray(), BigDecimal.class); + assertThat(v, is(bd)); + } + + @Test + public void handleMissingItemInArray() + throws IOException + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + packer.packArrayHeader(3); + packer.packString("one"); + packer.packString("two"); + packer.close(); + + assertThrows(UnexpectedEndOfInputException.class, () -> { + objectMapper.readValue(out.toByteArray(), new TypeReference>() {}); + }); + } + + @Test + public void handleMissingKeyValueInMap() + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + try { + packer.packMapHeader(3); + packer.packString("one"); + packer.packInt(1); + packer.packString("two"); + packer.packInt(2); + packer.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + assertThrows(JacksonException.class, () -> { + objectMapper.readValue(out.toByteArray(), new TypeReference>() {}); + }); + } + + @Test + public void handleMissingValueInMap() + { + MessagePacker packer = MessagePack.newDefaultPacker(out); + try { + packer.packMapHeader(3); + packer.packString("one"); + packer.packInt(1); + packer.packString("two"); + packer.packInt(2); + packer.packString("three"); + packer.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + assertThrows(JacksonException.class, () -> { + objectMapper.readValue(out.toByteArray(), new TypeReference>() {}); + }); + } + + @Test + public void testByteArrayThreadLocalClearedAfterClose() + throws IOException + { + ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory()); + + byte[] bytes = objectMapper.writeValueAsBytes(Arrays.asList(1, 2, 3)); + + // Parse once; this caches the byte array in the ThreadLocal + objectMapper.readValue(bytes, new TypeReference>() {}); + + // Parse again with the same byte array instance and AUTO_CLOSE_SOURCE enabled + // (default). The byte array reference should have been cleared from the + // ThreadLocal on close, so the second parse resets the unpacker and starts + // from the beginning rather than continuing from the end. + List result = objectMapper.readValue(bytes, new TypeReference>() {}); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testByteArrayReuseResetsUnpackerWhenAutoCloseSourceDisabled() + throws IOException + { + ObjectMapper objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .disable(tools.jackson.core.StreamReadFeature.AUTO_CLOSE_SOURCE) + .build(); + + byte[] bytes = objectMapper.writeValueAsBytes(Arrays.asList(1, 2, 3)); + + // First parse succeeds + List first = objectMapper.readValue(bytes, new TypeReference>() {}); + assertEquals(Arrays.asList(1, 2, 3), first); + + // Second parse with the same byte[] instance and AUTO_CLOSE_SOURCE disabled. + // The unpacker is not reset (src identity match, no AUTO_CLOSE_SOURCE trigger), + // so it continues from EOF and fails to produce the expected result. + List second = objectMapper.readValue(bytes, new TypeReference>() {}); + assertEquals(Arrays.asList(1, 2, 3), second); + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/TimestampExtensionModuleTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/TimestampExtensionModuleTest.java new file mode 100644 index 00000000..36f5c97f --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/TimestampExtensionModuleTest.java @@ -0,0 +1,217 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Test; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.core.MessageUnpacker; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Instant; + +import static org.junit.Assert.assertEquals; + +public class TimestampExtensionModuleTest +{ + private ObjectMapper objectMapper; + private final SingleInstant singleInstant = new SingleInstant(); + private final TripleInstants tripleInstants = new TripleInstants(); + + private static class SingleInstant + { + public Instant instant; + } + + private static class TripleInstants + { + public Instant a; + public Instant b; + public Instant c; + } + + @Before + public void setUp() + throws Exception + { + objectMapper = MessagePackMapper.builder(new MessagePackFactory()) + .addModule(TimestampExtensionModule.INSTANCE) + .build(); + } + + @Test + public void testSingleInstantPojo() + throws IOException + { + singleInstant.instant = Instant.now(); + byte[] bytes = objectMapper.writeValueAsBytes(singleInstant); + SingleInstant deserialized = objectMapper.readValue(bytes, SingleInstant.class); + assertEquals(singleInstant.instant, deserialized.instant); + } + + @Test + public void testTripleInstantsPojo() + throws IOException + { + Instant now = Instant.now(); + tripleInstants.a = now.minusSeconds(1); + tripleInstants.b = now; + tripleInstants.c = now.plusSeconds(1); + byte[] bytes = objectMapper.writeValueAsBytes(tripleInstants); + TripleInstants deserialized = objectMapper.readValue(bytes, TripleInstants.class); + assertEquals(now.minusSeconds(1), deserialized.a); + assertEquals(now, deserialized.b); + assertEquals(now.plusSeconds(1), deserialized.c); + } + + @Test + public void serialize32BitFormat() + throws IOException + { + singleInstant.instant = Instant.ofEpochSecond(Instant.now().getEpochSecond()); + + byte[] bytes = objectMapper.writeValueAsBytes(singleInstant); + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + assertEquals("instant", unpacker.unpackString()); + assertEquals(4, unpacker.unpackExtensionTypeHeader().getLength()); + } + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(singleInstant.instant, unpacker.unpackTimestamp()); + } + } + + @Test + public void serialize64BitFormat() + throws IOException + { + singleInstant.instant = Instant.ofEpochSecond(Instant.now().getEpochSecond(), 1234); + + byte[] bytes = objectMapper.writeValueAsBytes(singleInstant); + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + assertEquals("instant", unpacker.unpackString()); + assertEquals(8, unpacker.unpackExtensionTypeHeader().getLength()); + } + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(singleInstant.instant, unpacker.unpackTimestamp()); + } + } + + @Test + public void serialize96BitFormat() + throws IOException + { + singleInstant.instant = Instant.ofEpochSecond(19880866800L /* 2600-01-01 */, 1234); + + byte[] bytes = objectMapper.writeValueAsBytes(singleInstant); + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + assertEquals("instant", unpacker.unpackString()); + assertEquals(12, unpacker.unpackExtensionTypeHeader().getLength()); + } + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(singleInstant.instant, unpacker.unpackTimestamp()); + } + } + + @Test + public void deserialize32BitFormat() + throws IOException + { + Instant instant = Instant.ofEpochSecond(Instant.now().getEpochSecond()); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(os)) { + packer.packMapHeader(1) + .packString("instant") + .packTimestamp(instant); + } + + byte[] bytes = os.toByteArray(); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(4, unpacker.unpackExtensionTypeHeader().getLength()); + } + + SingleInstant deserialized = objectMapper.readValue(bytes, SingleInstant.class); + assertEquals(instant, deserialized.instant); + } + + @Test + public void deserialize64BitFormat() + throws IOException + { + Instant instant = Instant.ofEpochSecond(Instant.now().getEpochSecond(), 1234); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(os)) { + packer.packMapHeader(1) + .packString("instant") + .packTimestamp(instant); + } + + byte[] bytes = os.toByteArray(); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(8, unpacker.unpackExtensionTypeHeader().getLength()); + } + + SingleInstant deserialized = objectMapper.readValue(bytes, SingleInstant.class); + assertEquals(instant, deserialized.instant); + } + + @Test + public void deserialize96BitFormat() + throws IOException + { + Instant instant = Instant.ofEpochSecond(19880866800L /* 2600-01-01 */, 1234); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(os)) { + packer.packMapHeader(1) + .packString("instant") + .packTimestamp(instant); + } + + byte[] bytes = os.toByteArray(); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) { + unpacker.unpackMapHeader(); + unpacker.unpackString(); + assertEquals(12, unpacker.unpackExtensionTypeHeader().getLength()); + } + + SingleInstant deserialized = objectMapper.readValue(bytes, SingleInstant.class); + assertEquals(instant, deserialized.instant); + } +} diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md new file mode 100644 index 00000000..264fa225 --- /dev/null +++ b/plans/preexisting-issues.md @@ -0,0 +1,307 @@ +# Issues to address + +## msgpack-jackson3-specific + +### 1. `isClosed()` always returns false — FIXED + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` (close method) + +Fixed by delegating to `super.close()` (which sets `_closed = true` and calls +`_releaseBuffers()`). `AUTO_CLOSE_TARGET` handling was moved to `_closeInput()` so the +base-class lifecycle is used correctly. Our `close()` override calls `flush()` first to +drain pending nodes, then delegates the rest to `super.close()`. + +### 2. `MessagePackFactory.snapshot()` returns `this` — FIXED + +**File:** `msgpack-jackson3/.../MessagePackFactory.java` + +Fixed: `snapshot()` now delegates to `copy()`, and `rebuild()` is implemented via `MessagePackFactoryBuilder`. + +### 3. Build: `msgpack-jackson3` fails to compile locally on Java < 17 — FIXED + +`build.sbt` conditionally includes `msgpack-jackson3` in the root aggregate only when +running on Java 17+. Developers on older JDKs and CI on older JDK matrix entries +skip the module cleanly. + +### 4. `MessagePackGenerator` Jackson 3 generator contract — FIXED + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` + +Several issues fixed across two passes: + +**Pass 1** — `streamWriteContext()` returns null: +Added a `SimpleStreamWriteContext writeContext` field initialized to +`SimpleStreamWriteContext.createRootContext(null)`. `streamWriteContext()` returns +it; `writeStartArray/Object` push a child context; `endCurrentContainer` pops via +`clearAndGetParent()`; `currentValue()` and `assignCurrentValue()` delegate to it. + +Also fixed in the same pass: +- `version()` now returns `PackageVersion.VERSION` in generator, parser, and factory. + +**Pass 2** — Align with CBORGenerator pattern (reference: jackson-dataformats-binary 3.x): +- `_verifyValueWrite`: checks return value of `writeContext.writeValue()` and calls + `_reportError` on failure, so writing a value inside an object without a preceding + property name is detected at the Jackson API level. +- `writeName(String/SerializableString)`: checks return value of + `writeContext.writeName()` and calls `_reportError` when not in Object context or + when a name is written twice without an intervening value. +- `writeStartArray/writeStartObject`: call `_verifyValueWrite` before creating the + child context, matching the CBORGenerator pattern. +- `addValueNode`: calls `writeContext.writeValue()` to keep Jackson context state in + sync with our internal node-based state. This was the root cause of all `writeName` + failures: `GeneratorBase` does NOT auto-call `_verifyValueWrite` for abstract write + methods (writeString, writeNumber, etc.), so without this call `_gotPropertyId` was + never reset after the first `writeName`, making every subsequent `writeName` return + false. + +Note: `messageBufferOutputHolder` ThreadLocal OutputStream retention — FIXED (see +preexisting-issues section 4 below). `messageBufferOutputHolder.remove()` caused a 23% +regression; instead `_releaseBuffers()` now calls `messageBufferOutput.reset(null)` to +release the OutputStream reference while keeping the `OutputStreamBufferOutput` object +alive for reuse. + +### 5. `writeString(Reader, int)` len≥0 path allocates unbounded buffer — FIXED + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` (writeString(Reader, int)) + +The len≥0 path allocated `new char[len]` upfront — an unbounded allocation if the +caller passed a large hint value. Fixed by switching to chunked reading (8192-char +buffer) into a `StringBuilder`, matching the pattern already used in the len<0 branch. + +The len<0 path (StringBuilder + 1024-char chunk) still allocates an extra copy vs. +`CharArrayWriter`, but the overhead is minor and not worth optimizing at this layer. + +### 6. Node buffering: full-tree design — FYI, no action needed + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` (node buffering design) + +`MessagePackGenerator` buffers the entire document as a tree of `ValueNode` objects and +only serializes to MessagePack bytes when the root container closes (or on `flush()`). +This is required because MessagePack's array and map headers encode the element count +upfront (e.g. `fixarray 0x9X` or `array 16 0xdc`), and the count is not known until all +children have been written. + +A potential optimization would be to eagerly serialize completed sub-trees (e.g. flush a +completed nested array into bytes as soon as its `writeEndArray` is called). This reduces +the peak heap footprint for large flat nested collections, since the child nodes can be +GC'd once their bytes are emitted. However: + +- It does not eliminate the root-container buffering problem: the root map/array must + still buffer all its children until the root closes, because its own header depends on + the total count. +- It adds an extra copy per sub-tree (the bytes for each sub-tree must be merged into the + parent's payload at close time). +- In practice, the dominant cost is the root-level buffer, which this optimization does + not address. + +Decision: left as-is. Document as a known architectural constraint. + +### 7. Nested POJO generator inefficiency — FYI, no action needed + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` (writePOJO / nested generators) + +When a serializer calls `gen.writePOJO(someObject)`, Jackson internally creates a new +`JsonGenerator` wrapping the same output stream and serializes the nested object into it. +For `MessagePackGenerator` this means the nested generator also buffers a full node tree +and emits bytes only when it closes — triggering a `flush()` of its own root context. +The parent generator then reads those bytes back as a raw blob and re-wraps them in a +`RawValueNode`. + +The round-trip (buffer → bytes → re-buffer as raw) means one extra allocation and copy +per nested POJO. For deeply nested structures this can add up. + +Decision: this is an inherent consequence of the deferred-flush design (see item 6) and +cannot be fixed without a larger architectural change. Left as-is. + +### 8. `writeRaw*` / `writeRawValue*` — FYI, no action needed + +**File:** `msgpack-jackson3/.../MessagePackGenerator.java` + +CBORGenerator and SmileGenerator both throw `UnsupportedOperationException` for all +`writeRaw*` / `writeRawValue*` overloads. This was flagged in code review as something +MessagePackGenerator should match. + +After investigation, the current implementation (which delegates to `addValueNode`, same +as `writeString`) is correct and need not change: + +- The `writeRaw` contract requires: no escaping, no separators, content preserved + unchanged. All three are satisfied — MessagePack has no JSON-style escaping or + separators, and the text is packed verbatim via `packString`. +- CBOR/Smile throwing is a design choice, not a technical requirement. They could + equally implement it as "write as a text value" — they chose to be strict instead. +- Our lenient approach (degrade gracefully to `writeString` behaviour) is just as + defensible, and avoids breaking callers that use `writeRaw` as a `writeString` + synonym. + +--- + +## msgpack-jackson pre-existing issues + +These issues were identified during review of msgpack-jackson3 and confirmed to exist +identically in msgpack-jackson. They should be addressed in both modules together. + +## 1. `MessagePackParser`: Same byte-array input skips unpacker reset + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java:130` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java:108` — FIXED + +When `AUTO_CLOSE_SOURCE` is disabled and the same `byte[]` instance is parsed more +than once, the condition `messageUnpackerTuple.first() != src` is false, so the unpacker +is not reset. The second parse continues from where the first left off (typically EOF) +instead of from the beginning. + +Bug only manifests when all three conditions hold: (1) `reuseResourceInParser=true`, +(2) `AUTO_CLOSE_SOURCE` disabled (non-default), (3) the same `byte[]` reference reused. + +**Note on msgpack-jackson3:** In practice, issue #2's fix (clearing the `byte[]` from the +ThreadLocal on `close()`) incidentally prevents this bug from manifesting — after close, +the cached src becomes `null`, so the next parse always sees `null != bytes` and resets. +Fixed explicitly anyway (`|| src instanceof byte[]`) to make intent clear and remove the +subtle dependency between the two fixes. Regression test: +`MessagePackParserTest.testByteArrayReuseResetsUnpackerWhenAutoCloseSourceDisabled`. + +Needs the same explicit fix in msgpack-jackson (where issue #2 is also not yet fixed, so +both bugs compound). + +## 2. `MessagePackParser`: ThreadLocal retains last byte-array payload per thread + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java:135` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` — FIXED + +`messageUnpackerHolder` is never cleared on parser close. For byte-array inputs this +retains the entire last parsed payload for each thread in a pool indefinitely, which +can cause unbounded memory retention after large messages. + +Fixed in msgpack-jackson3: on `close()`, if the cached source is a `byte[]`, it is +replaced with `null` in the ThreadLocal (keeping the unpacker alive for reuse but +releasing the byte-array reference). InputStream sources are left unchanged because +they are needed to detect same-stream reuse. Needs the same fix in msgpack-jackson. + +## 3. `MessagePackGenerator`: `close()` does not set `isClosed()` to true + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` (close method) +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +The `close()` override never sets the closed flag, so `isClosed()` remains false. +Fixed in msgpack-jackson3 by setting `_closed = true` directly (calling `super.close()` +is not viable since it unconditionally closes the underlying stream, ignoring +`AUTO_CLOSE_TARGET`). Needs the same fix in msgpack-jackson. + +## 4. `MessagePackGenerator`: `writeString(Reader, int)` crashes on length -1 + +**File:** `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` (writeString(Reader, int)) + +`new char[len]` throws `NegativeArraySizeException` when `len` is -1, which is a +valid Jackson API usage meaning "unknown length, read until EOF". + +## 5. `MessagePackGenerator`: `writeNumber(String)` tries `Double.parseDouble` before `BigInteger` + +**File:** `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:699` + +Integer strings outside the `long` range are serialized as floating-point values, +losing precision, even though MessagePack can encode big integers exactly. The order +should try integer parsing first. + +## 6. `MessagePackGenerator`: Closing a container leaves stale `currentState` + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` (writeEndArray/writeEndObject) +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +After closing the root container, `currentState` was not reset to `IN_ROOT`. After +`flush()` clears `nodes`, any subsequent root-level value was treated as if inside the +old container. Fixed in msgpack-jackson3 by adding `currentState = IN_ROOT` in +`endCurrentContainer()`; needs the same fix in msgpack-jackson. + +## 7. `MessagePackSerializedString`: Most interface methods are stubs + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java:70` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java` — FIXED + +Most `SerializableString` methods return 0 or do nothing. Any Jackson code path +that calls these methods (e.g. for length or byte-copy operations) will silently +produce incorrect results. Fixed in msgpack-jackson3 by implementing all append/write/put +methods using the existing `asUnquotedUTF8()` / `asQuotedUTF8()` helpers. +Needs the same fix in msgpack-jackson. + +## 8. `MessagePackGenerator`: `getBytesIfAscii` writes to wrong index when offset > 0 + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:474` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +`bytes[i] = (byte) c` should be `bytes[i - offset] = (byte) c`. When offset > 0, `i` +starts above 0 but `bytes` is length `len`, so it throws `ArrayIndexOutOfBoundsException`. +Fixed in msgpack-jackson3; needs the same fix in msgpack-jackson. + +**Practical impact:** Low. `writeString(char[], offset, len)` is a low-level Jackson +streaming API used by Jackson's own internals and performance-sensitive custom serializers, +not by typical application code. Normal users call `writeString(String)` instead. + +## 9. `MessagePackGenerator`: `writeByteArrayTextValue` ASCII path ignores offset + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:512` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +`addValueNode(new AsciiCharString(text))` stores the entire backing array instead of +the requested slice `[offset, offset+len)`. Callers such as `writeRawUTF8String` and +`writeUTF8String` with non-zero offsets will serialize garbage bytes. +Fixed in msgpack-jackson3 using `System.arraycopy`; needs the same fix in msgpack-jackson. + +**Practical impact:** Low. `writeUTF8String(byte[], offset, len)` is a low-level API +called by Jackson's streaming infrastructure or custom serializers working with raw +byte buffers. Typical application code goes through ObjectMapper, which always passes +offset=0 for plain byte arrays. + +## 10. `MessagePackGenerator`: ByteBuffer serialization ignores `position()` + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:369` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +`writePayload(bb.array(), bb.arrayOffset(), len)` ignores `bb.position()`. For any +ByteBuffer with a non-zero position (e.g. from `ByteBuffer.wrap(data, offset, len)` or +after reads), this serializes bytes starting at the wrong offset. +Fixed in msgpack-jackson3 using `bb.arrayOffset() + bb.position()`; needs the same fix in msgpack-jackson. + +**Practical impact:** Moderate. This is the most realistic end-user scenario: a POJO +with a `ByteBuffer` field that was sliced or partially consumed will silently produce +corrupt serialized output. No exception is thrown. + +## 11. `MessagePackFactory`: byte-array parser copies the input slice unnecessarily + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java:143` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java` — FIXED + +`_createParser(byte[], int, int)` calls `Arrays.copyOfRange(data, offset, offset + len)` to +extract the slice, then wraps the copy in `InputStreamBufferInput`. `ArrayBufferInput` already +accepts `(data, offset, len)` directly and avoids the copy entirely. +Fixed in msgpack-jackson3 by constructing `ArrayBufferInput(data, offset, len)` and passing it +to the `MessageBufferInput`-taking constructor; needs the same fix in msgpack-jackson. + +**Practical impact:** Low for small payloads; proportional to payload size for large messages +since the entire slice is copied on every `readValue` call. + +## 12. `MessagePackGenerator`: non-array-backed ByteBuffer read advances caller's position + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:373` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +When serializing a `ByteBuffer` whose `hasArray()` is false (e.g. direct buffers), the +generator calls `bb.get(data)` which advances the buffer's position as a side effect. The +`hasArray()` fast path is non-destructive (`writePayload(bb.array(), bb.arrayOffset() + +bb.position(), len)`), so the two paths are inconsistent. +Fixed in msgpack-jackson3 using `bb.duplicate().get(data)` — `duplicate()` shares the +backing store without copying data (O(1)) but has its own independent position. +Needs the same fix in msgpack-jackson. + +**Practical impact:** Low. Direct ByteBuffers are uncommon in typical POJO fields, and the +mutation is only observable if the caller inspects or reuses the buffer after serialization. + diff --git a/project/plugins.sbt b/project/plugins.sbt index 2d12cde5..28f255ba 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,5 +6,6 @@ addSbtPlugin("org.xerial.sbt" % "sbt-jcheckstyle" % "0.2.1") addSbtPlugin("com.github.sbt" % "sbt-osgi" % "0.10.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.6.1") addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.1.1") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") scalacOptions ++= Seq("-deprecation", "-feature") From 6ecf5af3cfe776d25137739b0e928274b87f9332 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 24 May 2026 14:37:49 +0900 Subject: [PATCH 02/50] Fix nested generator resource leak in MessagePackGenerator Closing the nested generator via try-with-resources calls _releaseBuffers(), which resets the shared ThreadLocal OutputStreamBufferOutput to null. Re-attach the outer generator's output stream after the nested generator closes to restore correct state. Also document this as pre-existing issue #13 in msgpack-jackson. --- .../jackson/dataformat/MessagePackGenerator.java | 14 ++++++++++---- plans/preexisting-issues.md | 11 +++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 0527c0c3..97a0c86d 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -414,11 +414,17 @@ else if (v instanceof MessagePackExtensionType) { else { messagePacker.flush(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - MessagePackGenerator messagePackGenerator = new MessagePackGenerator( + try (MessagePackGenerator messagePackGenerator = new MessagePackGenerator( objectWriteContext(), _ioContext, _streamWriteFeatures, - outputStream, packerConfig, supportIntegerKeys); - objectWriteContext().writeValue(messagePackGenerator, v); - messagePackGenerator.flush(); + outputStream, packerConfig, supportIntegerKeys)) { + objectWriteContext().writeValue(messagePackGenerator, v); + } + // Closing the nested generator resets the shared ThreadLocal OutputStreamBufferOutput + // to null via _releaseBuffers(). Re-attach it to this generator's output stream. + OutputStreamBufferOutput bufferOutput = messageBufferOutputHolder.get(); + if (bufferOutput != null) { + bufferOutput.reset(output); + } output.write(outputStream.toByteArray()); } } diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index 264fa225..9ce166b1 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -305,3 +305,14 @@ Needs the same fix in msgpack-jackson. **Practical impact:** Low. Direct ByteBuffers are uncommon in typical POJO fields, and the mutation is only observable if the caller inspects or reuses the buffer after serialization. +## 13. `MessagePackGenerator`: nested generator not closed in `writePOJO` fallback + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java:387` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED + +In the `writePOJO` else-branch (unrecognised type), a new `MessagePackGenerator` is created +to serialize the nested object but never closed. Any resource cleanup in the generator's +`close()` (e.g. flushing pending nodes, releasing the `MessagePacker`) is skipped. +Fixed in msgpack-jackson3 using try-with-resources; needs the same fix in msgpack-jackson. + From 47be134cb9a297d9b293693066a29a17f96909ab Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 24 May 2026 19:27:01 +0900 Subject: [PATCH 03/50] Replace SimpleStreamWriteContext with lightweight MessagePackWriteContext SimpleStreamWriteContext carries state-machine overhead (dup detection, WritableTypeId tracking) that msgpack doesn't need. The custom inner class tracks only the gotName boolean for OBJECT alternation and currentName/ currentValue, matching what MessagePackReadContext already does on the read side. Measured +2.2% write throughput at steady-state JIT (20 warmup iters). --- .../dataformat/MessagePackGenerator.java | 89 +++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 97a0c86d..595ea196 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -18,7 +18,6 @@ import tools.jackson.core.Base64Variant; import tools.jackson.core.JacksonException; import tools.jackson.core.util.JacksonFeatureSet; -import tools.jackson.core.util.SimpleStreamWriteContext; import tools.jackson.core.JsonGenerator; import tools.jackson.core.ObjectWriteContext; import tools.jackson.core.SerializableString; @@ -58,7 +57,87 @@ public class MessagePackGenerator private int currentState = IN_ROOT; private final List nodes; private boolean isElementsClosed = false; - private SimpleStreamWriteContext writeContext; + private MessagePackWriteContext writeContext; + + private static final class MessagePackWriteContext extends TokenStreamContext + { + private final MessagePackWriteContext parent; + private String currentName; + private Object currentValue; + // For TYPE_OBJECT: true after writeName (expecting value), false after writeValue (expecting name) + private boolean gotName; + + private MessagePackWriteContext(int type, MessagePackWriteContext parent) + { + super(type, -1); + this.parent = parent; + } + + static MessagePackWriteContext createRootContext() + { + return new MessagePackWriteContext(TYPE_ROOT, null); + } + + MessagePackWriteContext createChildArrayContext(Object value) + { + MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_ARRAY, this); + ctx.currentValue = value; + return ctx; + } + + MessagePackWriteContext createChildObjectContext(Object value) + { + MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_OBJECT, this); + ctx.currentValue = value; + return ctx; + } + + @Override + public MessagePackWriteContext getParent() + { + return parent; + } + + boolean writeValue() + { + if (_type == TYPE_OBJECT) { + if (!gotName) { + return false; + } + gotName = false; + } + ++_index; + return true; + } + + boolean writeName(String name) + { + if (_type != TYPE_OBJECT || gotName) { + return false; + } + currentName = name; + gotName = true; + return true; + } + + @Override + public String currentName() + { + return currentName; + } + + @Override + public Object currentValue() + { + return currentValue; + } + + @Override + public void assignCurrentValue(Object v) + { + currentValue = v; + } + } private static final class RawUtf8String { @@ -199,7 +278,7 @@ private MessagePackGenerator( this.packerConfig = packerConfig; this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; - this.writeContext = SimpleStreamWriteContext.createRootContext(null); + this.writeContext = MessagePackWriteContext.createRootContext(); } public MessagePackGenerator( @@ -218,7 +297,7 @@ public MessagePackGenerator( this.packerConfig = packerConfig; this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; - this.writeContext = SimpleStreamWriteContext.createRootContext(null); + this.writeContext = MessagePackWriteContext.createRootContext(); } private MessageBufferOutput getMessageBufferOutputForOutputStream( @@ -339,7 +418,7 @@ public JsonGenerator writeEndObject() throws JacksonException private void endCurrentContainer() { - writeContext = writeContext.clearAndGetParent(); + writeContext = writeContext.getParent(); Node parent = nodes.get(currentParentElementIndex); if (currentParentElementIndex == 0) { isElementsClosed = true; From 9bf082d86f1a68c3522da7cc469193df5bd80e73 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 24 May 2026 23:43:30 +0900 Subject: [PATCH 04/50] Fix ThreadLocal ownership: non-owner nested generator no longer resets shared buffer Added `ownsThreadLocalBuffer` boolean set only in the public constructor when `reuseResourceInGenerator=true`. The private nested-generator constructor sets it `false`. `_releaseBuffers()` only calls `reset(null)` when this generator owns the ThreadLocal entry, so the outer generator's buffer reference is no longer corrupted. The re-attach workaround after nested generator close is deleted. --- .../jackson/dataformat/MessagePackGenerator.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 595ea196..9903d50e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -58,6 +58,7 @@ public class MessagePackGenerator private final List nodes; private boolean isElementsClosed = false; private MessagePackWriteContext writeContext; + private final boolean ownsThreadLocalBuffer; private static final class MessagePackWriteContext extends TokenStreamContext { @@ -279,6 +280,7 @@ private MessagePackGenerator( this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; this.writeContext = MessagePackWriteContext.createRootContext(); + this.ownsThreadLocalBuffer = false; } public MessagePackGenerator( @@ -298,6 +300,7 @@ public MessagePackGenerator( this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; this.writeContext = MessagePackWriteContext.createRootContext(); + this.ownsThreadLocalBuffer = reuseResourceInGenerator; } private MessageBufferOutput getMessageBufferOutputForOutputStream( @@ -498,12 +501,6 @@ else if (v instanceof MessagePackExtensionType) { outputStream, packerConfig, supportIntegerKeys)) { objectWriteContext().writeValue(messagePackGenerator, v); } - // Closing the nested generator resets the shared ThreadLocal OutputStreamBufferOutput - // to null via _releaseBuffers(). Re-attach it to this generator's output stream. - OutputStreamBufferOutput bufferOutput = messageBufferOutputHolder.get(); - if (bufferOutput != null) { - bufferOutput.reset(output); - } output.write(outputStream.toByteArray()); } } @@ -1050,10 +1047,9 @@ protected void _closeInput() throws IOException @Override protected void _releaseBuffers() { - OutputStreamBufferOutput messageBufferOutput = messageBufferOutputHolder.get(); - if (messageBufferOutput != null) { + if (ownsThreadLocalBuffer) { try { - messageBufferOutput.reset(null); + messageBufferOutputHolder.get().reset(null); } catch (IOException e) { throw _wrapIOFailure(e); From 9a06e003857f1c6a94cdf59a65b3a2f760682858 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 24 May 2026 23:56:31 +0900 Subject: [PATCH 05/50] Fix ThreadLocal ownership in MessagePackParser: non-owner skips close() cleanup Added `ownsThreadLocalUnpacker` boolean set to `true` only when the parser acquires the ThreadLocal entry (`reuseResourceInParser=true`), `false` when bypassing it. The byte[]-source nulling in `close()` is now gated on ownership, preventing a non-owner parser from corrupting another parser's ThreadLocal entry. --- .../msgpack/jackson/dataformat/MessagePackParser.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 3bd64f83..2b0646a2 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -56,6 +56,7 @@ public class MessagePackParser private long currentPosition; private final IOContext ioContext; private ExtensionTypeCustomDeserializers extTypeCustomDesers; + private final boolean ownsThreadLocalUnpacker; private enum Type { @@ -97,6 +98,7 @@ public MessagePackParser( streamReadContext = MessagePackReadContext.createRootContext(dups); if (!reuseResourceInParser) { messageUnpacker = MessagePack.newDefaultUnpacker(input); + ownsThreadLocalUnpacker = false; return; } @@ -111,6 +113,7 @@ public MessagePackParser( messageUnpacker = messageUnpackerTuple.second(); } messageUnpackerHolder.set(new Tuple<>(src, messageUnpacker)); + ownsThreadLocalUnpacker = true; } public void setExtensionTypeCustomDeserializers(ExtensionTypeCustomDeserializers extTypeCustomDesers) @@ -555,9 +558,11 @@ public void close() } finally { isClosed = true; - Tuple tuple = messageUnpackerHolder.get(); - if (tuple != null && tuple.first() instanceof byte[]) { - messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); + if (ownsThreadLocalUnpacker) { + Tuple tuple = messageUnpackerHolder.get(); + if (tuple != null && tuple.first() instanceof byte[]) { + messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); + } } } } From 9ca97a746df751deafdbde2fb250359af3a68202 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Mon, 25 May 2026 00:02:41 +0900 Subject: [PATCH 06/50] Fix artifactId mismatch and TimestampExtensionModule IOException handling - PackageVersion: artifactId corrected from "msgpack-jackson3" to "jackson-dataformat-msgpack3" to match the Maven artifact name - TimestampExtensionModule: replace UncheckedIOException with Jackson's _wrapIOFailure() in both InstantSerializer and InstantDeserializer, consistent with how the rest of the codebase handles IOExceptions --- .../java/org/msgpack/jackson/dataformat/PackageVersion.java | 2 +- .../msgpack/jackson/dataformat/TimestampExtensionModule.java | 5 ++--- .../msgpack/jackson/dataformat/MessagePackGeneratorTest.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java index 29d704cc..7d30bb29 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java @@ -20,7 +20,7 @@ public class PackageVersion implements Versioned { - public static final Version VERSION = new Version(0, 9, 12, null, "org.msgpack", "msgpack-jackson3"); + public static final Version VERSION = new Version(0, 9, 12, null, "org.msgpack", "jackson-dataformat-msgpack3"); @Override public Version version() diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java index e269fa49..40ec589c 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java @@ -14,7 +14,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UncheckedIOException; import java.time.Instant; public class TimestampExtensionModule @@ -51,7 +50,7 @@ public void serialize(Instant value, JsonGenerator gen, SerializationContext pro } } catch (IOException e) { - throw new UncheckedIOException(e); + throw _wrapIOFailure(provider, e); } } } @@ -78,7 +77,7 @@ public Instant deserialize(JsonParser p, DeserializationContext ctxt) } } catch (IOException e) { - throw new UncheckedIOException(e); + throw _wrapIOFailure(ctxt, e); } } } diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java index 2b79bbf8..933fd9d5 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java @@ -1184,7 +1184,7 @@ public void testVersion() { assertNotEquals(null, factory.version()); assertEquals("org.msgpack", factory.version().getGroupId()); - assertEquals("msgpack-jackson3", factory.version().getArtifactId()); + assertEquals("jackson-dataformat-msgpack3", factory.version().getArtifactId()); } @Test From 3b460de9c2e70508345de1defb7928e119047a50 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Tue, 26 May 2026 23:14:16 +0900 Subject: [PATCH 07/50] Remove redundant public InputStream constructor from MessagePackParser The constructor only wrapped InputStream into InputStreamBufferInput and delegated to the package-private constructor. MessagePackFactory (same package) now does the wrapping directly, making the public constructor unnecessary. --- .../jackson/dataformat/MessagePackFactory.java | 4 +++- .../jackson/dataformat/MessagePackParser.java | 13 ------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java index 78545a9e..e6b0bf01 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackFactory.java @@ -34,6 +34,7 @@ import org.msgpack.core.annotations.VisibleForTesting; import org.msgpack.core.buffer.ArrayBufferInput; +import org.msgpack.core.buffer.InputStreamBufferInput; import org.msgpack.core.buffer.MessageBufferInput; import java.io.DataInput; @@ -117,7 +118,8 @@ protected JsonParser _createParser(ObjectReadContext readCtxt, IOContext ioCtxt, { try { MessagePackParser parser = new MessagePackParser(readCtxt, ioCtxt, - readCtxt.getStreamReadFeatures(_streamReadFeatures), in, reuseResourceInParser); + readCtxt.getStreamReadFeatures(_streamReadFeatures), + new InputStreamBufferInput(in), in, reuseResourceInParser); if (extTypeCustomDesers != null) { parser.setExtensionTypeCustomDeserializers(extTypeCustomDesers); } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 2b0646a2..2308f2c1 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -31,12 +31,10 @@ import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; -import org.msgpack.core.buffer.InputStreamBufferInput; import org.msgpack.core.buffer.MessageBufferInput; import org.msgpack.value.ValueType; import java.io.IOException; -import java.io.InputStream; import java.math.BigDecimal; import java.math.BigInteger; @@ -71,17 +69,6 @@ private enum Type private BigInteger biValue; private MessagePackExtensionType extensionTypeValue; - public MessagePackParser( - ObjectReadContext readCtxt, - IOContext ioCtxt, - int streamReadFeatures, - InputStream in, - boolean reuseResourceInParser) - throws IOException - { - this(readCtxt, ioCtxt, streamReadFeatures, new InputStreamBufferInput(in), in, reuseResourceInParser); - } - MessagePackParser(ObjectReadContext readCtxt, IOContext ioCtxt, int streamReadFeatures, From 84c0955b0d56db6dcc09fa367a57cb3c42fb5c10 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 14:26:26 +0900 Subject: [PATCH 08/50] Move MessagePackWriteContext to top-level class and add unit tests Extracted the private inner class from MessagePackGenerator into its own package-level file, consistent with MessagePackReadContext. Added MessagePackWriteContextTest covering root/array/object context state, index and entry count tracking, parent references, and currentValue. --- .../dataformat/MessagePackGenerator.java | 80 ------- .../dataformat/MessagePackWriteContext.java | 98 +++++++++ .../MessagePackWriteContextTest.java | 201 ++++++++++++++++++ 3 files changed, 299 insertions(+), 80 deletions(-) create mode 100644 msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java create mode 100644 msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 9903d50e..e4fe9ce3 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -60,86 +60,6 @@ public class MessagePackGenerator private MessagePackWriteContext writeContext; private final boolean ownsThreadLocalBuffer; - private static final class MessagePackWriteContext extends TokenStreamContext - { - private final MessagePackWriteContext parent; - private String currentName; - private Object currentValue; - // For TYPE_OBJECT: true after writeName (expecting value), false after writeValue (expecting name) - private boolean gotName; - - private MessagePackWriteContext(int type, MessagePackWriteContext parent) - { - super(type, -1); - this.parent = parent; - } - - static MessagePackWriteContext createRootContext() - { - return new MessagePackWriteContext(TYPE_ROOT, null); - } - - MessagePackWriteContext createChildArrayContext(Object value) - { - MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_ARRAY, this); - ctx.currentValue = value; - return ctx; - } - - MessagePackWriteContext createChildObjectContext(Object value) - { - MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_OBJECT, this); - ctx.currentValue = value; - return ctx; - } - - @Override - public MessagePackWriteContext getParent() - { - return parent; - } - - boolean writeValue() - { - if (_type == TYPE_OBJECT) { - if (!gotName) { - return false; - } - gotName = false; - } - ++_index; - return true; - } - - boolean writeName(String name) - { - if (_type != TYPE_OBJECT || gotName) { - return false; - } - currentName = name; - gotName = true; - return true; - } - - @Override - public String currentName() - { - return currentName; - } - - @Override - public Object currentValue() - { - return currentValue; - } - - @Override - public void assignCurrentValue(Object v) - { - currentValue = v; - } - } - private static final class RawUtf8String { public final byte[] bytes; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java new file mode 100644 index 00000000..1b67154f --- /dev/null +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java @@ -0,0 +1,98 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.TokenStreamContext; + +class MessagePackWriteContext extends TokenStreamContext +{ + private final MessagePackWriteContext parent; + private String currentName; + private Object currentValue; + // For TYPE_OBJECT: true after writeName (expecting value), false after writeValue (expecting name) + private boolean gotName; + + private MessagePackWriteContext(int type, MessagePackWriteContext parent) + { + super(type, -1); + this.parent = parent; + } + + static MessagePackWriteContext createRootContext() + { + return new MessagePackWriteContext(TYPE_ROOT, null); + } + + MessagePackWriteContext createChildArrayContext(Object value) + { + MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_ARRAY, this); + ctx.currentValue = value; + return ctx; + } + + MessagePackWriteContext createChildObjectContext(Object value) + { + MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_OBJECT, this); + ctx.currentValue = value; + return ctx; + } + + @Override + public MessagePackWriteContext getParent() + { + return parent; + } + + boolean writeValue() + { + if (_type == TYPE_OBJECT) { + if (!gotName) { + return false; + } + gotName = false; + } + ++_index; + return true; + } + + boolean writeName(String name) + { + if (_type != TYPE_OBJECT || gotName) { + return false; + } + currentName = name; + gotName = true; + return true; + } + + @Override + public String currentName() + { + return currentName; + } + + @Override + public Object currentValue() + { + return currentValue; + } + + @Override + public void assignCurrentValue(Object v) + { + currentValue = v; + } +} diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java new file mode 100644 index 00000000..815da170 --- /dev/null +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java @@ -0,0 +1,201 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat; + +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.TokenStreamContext; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class MessagePackWriteContextTest +{ + private final MessagePackFactory factory = new MessagePackFactory(); + + @Test + public void testRootContext() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + TokenStreamContext ctx = gen.streamWriteContext(); + assertTrue(ctx.inRoot()); + assertFalse(ctx.inArray()); + assertFalse(ctx.inObject()); + assertNull(ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(0, ctx.getEntryCount()); + assertNull(ctx.getParent()); + gen.writeNumber(1); + } + } + + @Test + public void testArrayContext() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + gen.writeStartArray(); + TokenStreamContext ctx = gen.streamWriteContext(); + + assertFalse(ctx.inRoot()); + assertTrue(ctx.inArray()); + assertFalse(ctx.inObject()); + assertNull(ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(0, ctx.getEntryCount()); + assertNotNull(ctx.getParent()); + assertTrue(ctx.getParent().inRoot()); + + gen.writeNumber(1); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(1, ctx.getEntryCount()); + + gen.writeNumber(2); + assertEquals(1, ctx.getCurrentIndex()); + assertEquals(2, ctx.getEntryCount()); + + gen.writeNumber(3); + assertEquals(2, ctx.getCurrentIndex()); + assertEquals(3, ctx.getEntryCount()); + + gen.writeEndArray(); + assertTrue(gen.streamWriteContext().inRoot()); + } + } + + @Test + public void testObjectContext() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + gen.writeStartObject(); + TokenStreamContext ctx = gen.streamWriteContext(); + + assertFalse(ctx.inRoot()); + assertFalse(ctx.inArray()); + assertTrue(ctx.inObject()); + assertNull(ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(0, ctx.getEntryCount()); + + gen.writeName("first"); + assertEquals("first", ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(0, ctx.getEntryCount()); + + gen.writeNumber(1); + assertEquals("first", ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(1, ctx.getEntryCount()); + + gen.writeName("second"); + assertEquals("second", ctx.currentName()); + assertEquals(0, ctx.getCurrentIndex()); + assertEquals(1, ctx.getEntryCount()); + + gen.writeString("value"); + assertEquals(1, ctx.getCurrentIndex()); + assertEquals(2, ctx.getEntryCount()); + + gen.writeEndObject(); + assertTrue(gen.streamWriteContext().inRoot()); + } + } + + @Test + public void testNestedObjectInArray() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + gen.writeStartArray(); + TokenStreamContext arrayCtx = gen.streamWriteContext(); + assertTrue(arrayCtx.inArray()); + + gen.writeStartObject(); + TokenStreamContext objectCtx = gen.streamWriteContext(); + assertTrue(objectCtx.inObject()); + assertSame(arrayCtx, objectCtx.getParent()); + + gen.writeName("key"); + assertEquals("key", objectCtx.currentName()); + assertEquals("key", gen.streamWriteContext().currentName()); + + gen.writeNumber(42); + gen.writeEndObject(); + + assertSame(arrayCtx, gen.streamWriteContext()); + assertEquals(0, arrayCtx.getCurrentIndex()); + assertEquals(1, arrayCtx.getEntryCount()); + + gen.writeEndArray(); + } + } + + @Test + public void testCurrentValue() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + Object pojo = new Object(); + gen.writeStartObject(pojo); + TokenStreamContext ctx = gen.streamWriteContext(); + assertEquals(pojo, ctx.currentValue()); + + Object inner = new Object(); + gen.writeName("arr"); + gen.writeStartArray(inner); + assertEquals(inner, gen.streamWriteContext().currentValue()); + gen.writeEndArray(); + + gen.writeEndObject(); + } + } + + @Test + public void testAssignCurrentValue() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + gen.writeStartObject(); + TokenStreamContext ctx = gen.streamWriteContext(); + assertNull(ctx.currentValue()); + + Object v = new Object(); + gen.assignCurrentValue(v); + assertSame(v, ctx.currentValue()); + assertSame(v, gen.currentValue()); + + gen.writeName("k"); + gen.writeNumber(1); + gen.writeEndObject(); + } + } +} From bf4d998e394ae01909749ac714e0b1a192b30ffe Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 14:41:49 +0900 Subject: [PATCH 09/50] Deduplicate MessagePackSerializedString: quoted variants delegate to unquoted Binary formats have no quoting concept, so quoted and unquoted methods are equivalent. Each quoted variant now delegates to its unquoted counterpart instead of duplicating the same logic. --- .../MessagePackSerializedString.java | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java index c24c1559..619f68a0 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializedString.java @@ -67,23 +67,13 @@ public byte[] asQuotedUTF8() @Override public int appendQuotedUTF8(byte[] bytes, int i) { - byte[] utf8 = asUnquotedUTF8(); - if (utf8.length > bytes.length - i) { - return -1; - } - System.arraycopy(utf8, 0, bytes, i, utf8.length); - return utf8.length; + return appendUnquotedUTF8(bytes, i); } @Override public int appendQuoted(char[] chars, int i) { - char[] q = asQuotedChars(); - if (q.length > chars.length - i) { - return -1; - } - System.arraycopy(q, 0, chars, i, q.length); - return q.length; + return appendUnquoted(chars, i); } @Override @@ -111,9 +101,7 @@ public int appendUnquoted(char[] chars, int i) @Override public int writeQuotedUTF8(OutputStream outputStream) throws IOException { - byte[] utf8 = asUnquotedUTF8(); - outputStream.write(utf8); - return utf8.length; + return writeUnquotedUTF8(outputStream); } @Override @@ -127,12 +115,7 @@ public int writeUnquotedUTF8(OutputStream outputStream) throws IOException @Override public int putQuotedUTF8(ByteBuffer byteBuffer) { - byte[] utf8 = asUnquotedUTF8(); - if (utf8.length > byteBuffer.remaining()) { - return -1; - } - byteBuffer.put(utf8); - return utf8.length; + return putUnquotedUTF8(byteBuffer); } @Override From b41079b7414491a06545b7cf894d7ce5ca993423 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 14:50:46 +0900 Subject: [PATCH 10/50] Add child context recycling to MessagePackWriteContext Mirrors CBORWriteContext: each context holds a single reuse slot for its most recently popped child. On repeated container open/close (common in document serialization), the child context object is reset and reused instead of allocated fresh each time. --- .../dataformat/MessagePackWriteContext.java | 29 +++++++++++++++---- .../MessagePackWriteContextTest.java | 29 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java index 1b67154f..e639f641 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java @@ -20,6 +20,7 @@ class MessagePackWriteContext extends TokenStreamContext { private final MessagePackWriteContext parent; + private MessagePackWriteContext childToRecycle; private String currentName; private Object currentValue; // For TYPE_OBJECT: true after writeName (expecting value), false after writeValue (expecting name) @@ -31,6 +32,16 @@ private MessagePackWriteContext(int type, MessagePackWriteContext parent) this.parent = parent; } + private MessagePackWriteContext reset(int type, Object value) + { + _type = type; + _index = -1; + gotName = false; + currentName = null; + currentValue = value; + return this; + } + static MessagePackWriteContext createRootContext() { return new MessagePackWriteContext(TYPE_ROOT, null); @@ -38,16 +49,22 @@ static MessagePackWriteContext createRootContext() MessagePackWriteContext createChildArrayContext(Object value) { - MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_ARRAY, this); - ctx.currentValue = value; - return ctx; + MessagePackWriteContext ctx = childToRecycle; + if (ctx == null) { + ctx = new MessagePackWriteContext(TYPE_ARRAY, this); + childToRecycle = ctx; + } + return ctx.reset(TYPE_ARRAY, value); } MessagePackWriteContext createChildObjectContext(Object value) { - MessagePackWriteContext ctx = new MessagePackWriteContext(TYPE_OBJECT, this); - ctx.currentValue = value; - return ctx; + MessagePackWriteContext ctx = childToRecycle; + if (ctx == null) { + ctx = new MessagePackWriteContext(TYPE_OBJECT, this); + childToRecycle = ctx; + } + return ctx.reset(TYPE_OBJECT, value); } @Override diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java index 815da170..1f542fc7 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackWriteContextTest.java @@ -178,6 +178,35 @@ public void testCurrentValue() } } + @Test + public void testChildContextIsReused() + throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), baos)) { + gen.writeStartArray(); + + gen.writeStartObject(); + TokenStreamContext first = gen.streamWriteContext(); + gen.writeName("k"); + gen.writeNumber(1); + gen.writeEndObject(); + + gen.writeStartObject(); + TokenStreamContext second = gen.streamWriteContext(); + assertSame(first, second); + assertNull(second.currentName()); + assertEquals(0, second.getCurrentIndex()); + assertEquals(0, second.getEntryCount()); + + gen.writeName("k2"); + gen.writeNumber(2); + gen.writeEndObject(); + + gen.writeEndArray(); + } + } + @Test public void testAssignCurrentValue() throws IOException From 876cf75947596b87ae12b25e9f85a6279e7b7b43 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 14:56:59 +0900 Subject: [PATCH 11/50] Add DupDetector support to MessagePackWriteContext Mirrors CBORWriteContext: when STRICT_DUPLICATE_DETECTION is enabled, writeName() checks for duplicate property names via DupDetector and throws StreamWriteException on violation. No overhead when the feature is disabled (dups == null guards the check). --- .../dataformat/MessagePackGenerator.java | 9 ++++-- .../dataformat/MessagePackWriteContext.java | 29 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index e4fe9ce3..a5f44cd7 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -23,6 +23,7 @@ import tools.jackson.core.SerializableString; import tools.jackson.core.StreamWriteCapability; import tools.jackson.core.StreamWriteFeature; +import tools.jackson.core.json.DupDetector; import tools.jackson.core.TokenStreamContext; import tools.jackson.core.base.GeneratorBase; import tools.jackson.core.io.IOContext; @@ -199,7 +200,9 @@ private MessagePackGenerator( this.packerConfig = packerConfig; this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; - this.writeContext = MessagePackWriteContext.createRootContext(); + this.writeContext = MessagePackWriteContext.createRootContext( + StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) + ? DupDetector.rootDetector(this) : null); this.ownsThreadLocalBuffer = false; } @@ -219,7 +222,9 @@ public MessagePackGenerator( this.packerConfig = packerConfig; this.nodes = new ArrayList<>(); this.supportIntegerKeys = supportIntegerKeys; - this.writeContext = MessagePackWriteContext.createRootContext(); + this.writeContext = MessagePackWriteContext.createRootContext( + StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) + ? DupDetector.rootDetector(this) : null); this.ownsThreadLocalBuffer = reuseResourceInGenerator; } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java index e639f641..701f7e91 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java @@ -16,20 +16,24 @@ package org.msgpack.jackson.dataformat; import tools.jackson.core.TokenStreamContext; +import tools.jackson.core.exc.StreamWriteException; +import tools.jackson.core.json.DupDetector; class MessagePackWriteContext extends TokenStreamContext { private final MessagePackWriteContext parent; private MessagePackWriteContext childToRecycle; + private DupDetector dups; private String currentName; private Object currentValue; // For TYPE_OBJECT: true after writeName (expecting value), false after writeValue (expecting name) private boolean gotName; - private MessagePackWriteContext(int type, MessagePackWriteContext parent) + private MessagePackWriteContext(int type, MessagePackWriteContext parent, DupDetector dups) { super(type, -1); this.parent = parent; + this.dups = dups; } private MessagePackWriteContext reset(int type, Object value) @@ -39,19 +43,22 @@ private MessagePackWriteContext reset(int type, Object value) gotName = false; currentName = null; currentValue = value; + if (dups != null) { + dups.reset(); + } return this; } - static MessagePackWriteContext createRootContext() + static MessagePackWriteContext createRootContext(DupDetector dups) { - return new MessagePackWriteContext(TYPE_ROOT, null); + return new MessagePackWriteContext(TYPE_ROOT, null, dups); } MessagePackWriteContext createChildArrayContext(Object value) { MessagePackWriteContext ctx = childToRecycle; if (ctx == null) { - ctx = new MessagePackWriteContext(TYPE_ARRAY, this); + ctx = new MessagePackWriteContext(TYPE_ARRAY, this, dups == null ? null : dups.child()); childToRecycle = ctx; } return ctx.reset(TYPE_ARRAY, value); @@ -61,7 +68,7 @@ MessagePackWriteContext createChildObjectContext(Object value) { MessagePackWriteContext ctx = childToRecycle; if (ctx == null) { - ctx = new MessagePackWriteContext(TYPE_OBJECT, this); + ctx = new MessagePackWriteContext(TYPE_OBJECT, this, dups == null ? null : dups.child()); childToRecycle = ctx; } return ctx.reset(TYPE_OBJECT, value); @@ -85,16 +92,26 @@ boolean writeValue() return true; } - boolean writeName(String name) + boolean writeName(String name) throws StreamWriteException { if (_type != TYPE_OBJECT || gotName) { return false; } currentName = name; gotName = true; + if (dups != null) { + _checkDup(name); + } return true; } + private void _checkDup(String name) throws StreamWriteException + { + if (dups.isDup(name)) { + throw new StreamWriteException(null, "Duplicate Object property \"" + name + "\""); + } + } + @Override public String currentName() { From 9ba7072ea2599c08f156068a37edba5b5fe5dd34 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 15:22:24 +0900 Subject: [PATCH 12/50] Rename artifactId to jackson-dataformat-msgpack-jackson3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous name jackson-dataformat-msgpack3 was ambiguous — the '3' suffix reads as 'MessagePack version 3' rather than 'Jackson 3 support'. The new name jackson-dataformat-msgpack-jackson3 clearly conveys Jackson 3 integration. Updates PackageVersion.java, build.sbt, and the corresponding test assertion. --- build.sbt | 2 +- .../java/org/msgpack/jackson/dataformat/PackageVersion.java | 2 +- .../msgpack/jackson/dataformat/MessagePackGeneratorTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index f39cba7e..7d5215fc 100644 --- a/build.sbt +++ b/build.sbt @@ -184,7 +184,7 @@ lazy val msgpackJackson3 = Project(id = "msgpack-jackson3", base = file("msgpack .enablePlugins(SbtOsgi, JmhPlugin) .settings( buildSettings, - name := "jackson-dataformat-msgpack3", + name := "jackson-dataformat-msgpack-jackson3", description := "Jackson 3.x extension that adds support for MessagePack", OsgiKeys.bundleSymbolicName := "org.msgpack.msgpack-jackson3", OsgiKeys.exportPackage := Seq("org.msgpack.jackson", "org.msgpack.jackson.dataformat"), diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java index 7d30bb29..255ce1d6 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/PackageVersion.java @@ -20,7 +20,7 @@ public class PackageVersion implements Versioned { - public static final Version VERSION = new Version(0, 9, 12, null, "org.msgpack", "jackson-dataformat-msgpack3"); + public static final Version VERSION = new Version(0, 9, 12, null, "org.msgpack", "jackson-dataformat-msgpack-jackson3"); @Override public Version version() diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java index 933fd9d5..426bbb5e 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackGeneratorTest.java @@ -1184,7 +1184,7 @@ public void testVersion() { assertNotEquals(null, factory.version()); assertEquals("org.msgpack", factory.version().getGroupId()); - assertEquals("jackson-dataformat-msgpack3", factory.version().getArtifactId()); + assertEquals("jackson-dataformat-msgpack-jackson3", factory.version().getArtifactId()); } @Test From 7a58b583a5e069dc9d00cfce14a2ea21135bff3b Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 17:49:14 +0900 Subject: [PATCH 13/50] Add size-hint comments, WriteUTF8StringBenchmark, and preexisting-issues entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Comment writeStartArray/Object(Object,int) in jackson3 generator explaining why size is ignored: Jackson does not guarantee size>=0 when dynamic filters are active (Jackson author confirmed, msgpack-java#841) - Add WriteUTF8StringBenchmark to measure RawUtf8String vs new String impact - Add preexisting-issues #14 (RawUtf8String optimization for jackson2) and #15 (writeStartArray size hint — potential optimization with caveats) --- .../benchmark/WriteUTF8StringBenchmark.java | 71 +++++++++++++++++++ .../dataformat/MessagePackGenerator.java | 6 ++ plans/preexisting-issues.md | 44 ++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java new file mode 100644 index 00000000..4adee1e5 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java @@ -0,0 +1,71 @@ +package org.msgpack.jackson.dataformat.benchmark; + +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.ObjectWriteContext; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +/** + * Compares writeUTF8String throughput for RawUtf8String (current) vs new String(bytes, UTF_8). + * Run this benchmark twice: once with the current implementation, once after replacing + * writeByteArrayTextValue to use new String(text, offset, len, StandardCharsets.UTF_8). + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Fork(2) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class WriteUTF8StringBenchmark +{ + private static final int WRITES_PER_INVOCATION = 500; + + private final MessagePackFactory factory = new MessagePackFactory(); + + // ASCII-only: compact-string fast path applies for new String(bytes, UTF_8) + private final byte[] asciiBytes = + "Hello, World! This is a typical ASCII field value.".getBytes(StandardCharsets.UTF_8); + + // Non-ASCII: multi-byte UTF-8, compact-string fast path does NOT apply + private final byte[] nonAsciiBytes = + "東京は日本の首都です。This mixes CJK and ASCII.".getBytes(StandardCharsets.UTF_8); + + @Benchmark + public int writeUTF8StringAscii() throws Exception + { + NopOutputStream out = new NopOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), out)) { + gen.writeStartArray(); + for (int i = 0; i < WRITES_PER_INVOCATION; i++) { + gen.writeUTF8String(asciiBytes, 0, asciiBytes.length); + } + gen.writeEndArray(); + } + return out.size(); + } + + @Benchmark + public int writeUTF8StringNonAscii() throws Exception + { + NopOutputStream out = new NopOutputStream(); + try (JsonGenerator gen = factory.createGenerator(ObjectWriteContext.empty(), out)) { + gen.writeStartArray(); + for (int i = 0; i < WRITES_PER_INVOCATION; i++) { + gen.writeUTF8String(nonAsciiBytes, 0, nonAsciiBytes.length); + } + gen.writeEndArray(); + } + return out.size(); + } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index a5f44cd7..c4695636 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -274,6 +274,11 @@ public JsonGenerator writeStartArray(Object currentValue) throws JacksonExceptio return writeStartArray(currentValue, -1); } + // size is ignored: element count is determined at flush time from the actual child nodes. + // When size >= 0, it could in principle be used to skip buffering and write the array + // header immediately, but Jackson does not guarantee it — dynamic filters (Views, + // @JsonFilter) evaluate entries incrementally and will pass -1 even for known-size + // collections. Backends must handle both cases (Jackson author confirmed, see #841). @Override public JsonGenerator writeStartArray(Object currentValue, int size) throws JacksonException { @@ -315,6 +320,7 @@ public JsonGenerator writeStartObject(Object currentValue) throws JacksonExcepti return writeStartObject(currentValue, -1); } + // size is ignored: same reasoning as writeStartArray(Object, int) above. @Override public JsonGenerator writeStartObject(Object forValue, int size) throws JacksonException { diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index 9ce166b1..22cf469c 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -316,3 +316,47 @@ to serialize the nested object but never closed. Any resource cleanup in the gen `close()` (e.g. flushing pending nodes, releasing the `MessagePacker`) is skipped. Fixed in msgpack-jackson3 using try-with-resources; needs the same fix in msgpack-jackson. +## 14. `MessagePackGenerator`: `writeByteArrayTextValue` uses `AsciiCharString` instead of raw bytes + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` (writeByteArrayTextValue) +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — uses `RawUtf8String` + +The v2 generator's `writeByteArrayTextValue` (called from `writeUTF8String` and +`writeRawUTF8String`) stores the pre-encoded UTF-8 bytes as an `AsciiCharString` (or similar +wrapper that ultimately calls `packString(String)`). This forces a String decode + re-encode +roundtrip at flush time: + +1. `new String(bytes, offset, len, UTF_8)` — decode UTF-8 bytes to Java String +2. `s.getBytes(UTF_8)` inside `packString` — re-encode back to UTF-8 bytes +3. Copy into packer buffer + +The jackson3 version avoids this by using a `RawUtf8String` wrapper that writes the bytes +directly via `packRawStringHeader + writePayload`, skipping both decode and re-encode. + +**Benchmark impact (measured on jackson3):** +- ASCII strings: ~82K ops/s (RawUtf8String) vs ~57K ops/s (new String) → **-30%** +- Non-ASCII strings: ~81K ops/s (RawUtf8String) vs ~16K ops/s (new String) → **-80%** + +Port `RawUtf8String` and the corresponding `packNonContainer` branch to msgpack-jackson. + +## 15. `MessagePackGenerator`: `writeStartArray/Object(Object, int)` size hint unused — potential optimization + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` + +Both generators currently ignore the `size` parameter. When `size >= 0`, it could in principle +allow writing the array/map header immediately (skipping the node-buffering phase for that +container), reducing allocations. + +**Caveat (Jackson author confirmed, msgpack-java#841):** Jackson does not guarantee `size >= 0`. +Dynamic filters (`Views`, `@JsonFilter`) evaluate entries incrementally and will pass `-1` even +for collections whose size is statically known. Any implementation must fall back to the +buffered approach when `size == -1`, so the optimization only applies to a subset of +serialization paths and adds significant complexity. + +**Conclusion:** Not worth pursuing unless profiling shows the node-buffering overhead dominates. +Document the caveat and leave `size` ignored for now. If implemented in the future, both modules +should be updated together. + From 6a8b572e58092be56d6da783bcd45c16bed64958 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 18:40:34 +0900 Subject: [PATCH 14/50] Inline NodeEntryInObject construction into nodes.add() --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index c4695636..ea1acf88 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -477,8 +477,7 @@ private void addKeyNode(Object key) if (currentState != IN_OBJECT) { throw new IllegalStateException(); } - Node node = new NodeEntryInObject(currentParentElementIndex, key); - nodes.add(node); + nodes.add(new NodeEntryInObject(currentParentElementIndex, key)); } private void addValueNode(Object value) throws IOException From 0ab51805d1b13fe465b9808aaf04b1399bc914e7 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 18:54:54 +0900 Subject: [PATCH 15/50] Avoid defensive copy in writeByteArrayTextValue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store the caller's byte array reference directly in RawUtf8String instead of copying to a new slice. Safe for the typical ObjectMapper use case where Jackson owns the buffer for the full duration of serialization. Reduces writeUTF8String throughput by ~+20% (82K → 98K ops/s) by eliminating one byte[] allocation and arraycopy per call. --- .../jackson/dataformat/MessagePackGenerator.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index ea1acf88..5439600a 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -64,10 +64,14 @@ public class MessagePackGenerator private static final class RawUtf8String { public final byte[] bytes; + public final int offset; + public final int len; - public RawUtf8String(byte[] bytes) + public RawUtf8String(byte[] bytes, int offset, int len) { this.bytes = bytes; + this.offset = offset; + this.len = len; } } @@ -376,9 +380,9 @@ private void packNonContainer(Object v) messagePacker.packString((String) v); } else if (v instanceof RawUtf8String) { - byte[] bytes = ((RawUtf8String) v).bytes; - messagePacker.packRawStringHeader(bytes.length); - messagePacker.writePayload(bytes); + RawUtf8String raw = (RawUtf8String) v; + messagePacker.packRawStringHeader(raw.len); + messagePacker.writePayload(raw.bytes, raw.offset, raw.len); } else if (v instanceof Integer) { messagePacker.packInt((Integer) v); @@ -514,9 +518,7 @@ private void writeCharArrayTextValue(char[] text, int offset, int len) throws IO private void writeByteArrayTextValue(byte[] text, int offset, int len) throws IOException { - byte[] slice = new byte[len]; - System.arraycopy(text, offset, slice, 0, len); - addValueNode(new RawUtf8String(slice)); + addValueNode(new RawUtf8String(text, offset, len)); } @Override From 7cca5b8ec297c809db20cb6c1a443b501eaeb06f Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 21:35:41 +0900 Subject: [PATCH 16/50] Align error reporting with CBOR: use _reportError in parser/generator Replace IllegalStateException with _reportError in MessagePackParser, and align writeName/writePropertyId error messages with CBORGenerator. --- .../dataformat/MessagePackGenerator.java | 6 ++--- .../jackson/dataformat/MessagePackParser.java | 24 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 5439600a..c00a3c56 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -526,7 +526,7 @@ public JsonGenerator writePropertyId(long id) throws JacksonException { if (this.supportIntegerKeys) { if (!writeContext.writeName(String.valueOf(id))) { - _reportError("Cannot write property name, not in Object context"); + _reportError("Can not write a property id, expecting a value"); } addKeyNode(id); } @@ -546,7 +546,7 @@ public JacksonFeatureSet streamWriteCapabilities() public JsonGenerator writeName(String name) throws JacksonException { if (!writeContext.writeName(name)) { - _reportError("Cannot write property name, not in Object context"); + _reportError("Can not write a property name, expecting a value"); } addKeyNode(name); return this; @@ -557,7 +557,7 @@ public JsonGenerator writeName(SerializableString name) throws JacksonException { if (name instanceof MessagePackSerializedString) { if (!writeContext.writeName(name.getValue())) { - _reportError("Cannot write property name, not in Object context"); + _reportError("Can not write a property name, expecting a value"); } addKeyNode(((MessagePackSerializedString) name).getRawValue()); } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 2308f2c1..e425d643 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -268,7 +268,7 @@ else if (streamReadContext.inArray()) { } break; default: - throw new IllegalStateException("Shouldn't reach here"); + nextToken = _reportError("Unexpected MessagePack format type: " + type); } currentPosition = messageUnpacker.getTotalReadBytes(); @@ -306,7 +306,7 @@ public String getString() throw _wrapIOFailure(e); } default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -345,7 +345,7 @@ public byte[] getBinaryValue(Base64Variant b64variant) case EXT: return extensionTypeValue.getData(); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -362,7 +362,7 @@ public Number getNumberValue() case BIG_INT: return biValue; default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -379,7 +379,7 @@ public int getIntValue() case BIG_INT: return biValue.intValue(); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -396,7 +396,7 @@ public long getLongValue() case BIG_INT: return biValue.longValue(); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -413,7 +413,7 @@ public BigInteger getBigIntegerValue() case BIG_INT: return biValue; default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -430,7 +430,7 @@ public float getFloatValue() case BIG_INT: return biValue.floatValue(); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -447,7 +447,7 @@ public double getDoubleValue() case BIG_INT: return biValue.doubleValue(); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -464,7 +464,7 @@ public BigDecimal getDecimalValue() case BIG_INT: return new BigDecimal(biValue); default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -494,7 +494,7 @@ public Object getEmbeddedObject() throw _wrapIOFailure(e); } default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } @@ -511,7 +511,7 @@ public NumberType getNumberType() case BIG_INT: return NumberType.BIG_INTEGER; default: - throw new IllegalStateException("Invalid type=" + type); + return _reportError("Unexpected MessagePack value type: " + type); } } From a4a09c0f745c2710056ffc2655baf1e732c56338 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 21:41:29 +0900 Subject: [PATCH 17/50] Merge writeString(Reader) branches using long remaining sentinel --- .../dataformat/MessagePackGenerator.java | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index c00a3c56..a29523fd 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -595,30 +595,19 @@ public JsonGenerator writeString(char[] text, int offset, int len) throws Jackso public JsonGenerator writeString(Reader reader, int len) throws JacksonException { try { - if (len < 0) { - StringBuilder sb = new StringBuilder(); - char[] tmpBuf = new char[1024]; - int read; - while ((read = reader.read(tmpBuf)) >= 0) { - sb.append(tmpBuf, 0, read); + long remaining = len < 0 ? Long.MAX_VALUE : len; + int chunkSize = (int) Math.min(remaining, 8192); + StringBuilder sb = new StringBuilder(chunkSize); + char[] tmpBuf = new char[chunkSize]; + while (remaining > 0) { + int read = reader.read(tmpBuf, 0, (int) Math.min(remaining, tmpBuf.length)); + if (read < 0) { + break; } - addValueNode(sb.toString()); - } - else { - int chunkSize = Math.min(len, 8192); - StringBuilder sb = new StringBuilder(chunkSize); - char[] tmpBuf = new char[chunkSize]; - int remaining = len; - while (remaining > 0) { - int read = reader.read(tmpBuf, 0, Math.min(remaining, tmpBuf.length)); - if (read < 0) { - break; - } - sb.append(tmpBuf, 0, read); - remaining -= read; - } - addValueNode(sb.toString()); + sb.append(tmpBuf, 0, read); + remaining -= read; } + addValueNode(sb.toString()); } catch (IOException e) { throw _wrapIOFailure(e); From 8faff7d7ad9c73e1d15b30f111511f8fa648bd1d Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 21:56:08 +0900 Subject: [PATCH 18/50] Inline getMessagePacker().flush() in _flushBuffers --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index a29523fd..c4cd0edc 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -917,8 +917,7 @@ else if (node instanceof NodeArray) { private void flushMessagePacker() throws IOException { - MessagePacker messagePacker = getMessagePacker(); - messagePacker.flush(); + getMessagePacker().flush(); } @Override From cfa4d364253b68f5a4d6863a28d23f714d55e907 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 22:56:45 +0900 Subject: [PATCH 19/50] Address reviewer feedback: security fixes, license headers, minor corrections - Validate binary and extension payload lengths against StreamReadConstraints before allocating to prevent OOM from malicious large-length headers - Guard ByteBuffer hasArray() path with isReadOnly() to avoid ReadOnlyBufferException on read-only heap buffers - Fix error message in _nextToken default branch: use valueType not type - Escape backslashes and quotes in MessagePackReadContext.toString() - Add Apache 2.0 license headers to 5 files missing them --- .../dataformat/benchmark/NopOutputStream.java | 15 +++++++++++++++ .../jackson/dataformat/JsonArrayFormat.java | 15 +++++++++++++++ .../dataformat/MessagePackExtensionType.java | 15 +++++++++++++++ .../dataformat/MessagePackGenerator.java | 2 +- .../jackson/dataformat/MessagePackParser.java | 4 +++- .../dataformat/MessagePackReadContext.java | 17 ++++++++++++++++- .../dataformat/TimestampExtensionModule.java | 15 +++++++++++++++ 7 files changed, 80 insertions(+), 3 deletions(-) diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java index e3b932c5..983a27e2 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/NopOutputStream.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark; import java.io.OutputStream; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java index 84be3f5c..1b1e048b 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/JsonArrayFormat.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat; import tools.jackson.databind.cfg.MapperConfig; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java index 92519356..13041d4b 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat; import tools.jackson.core.JsonGenerator; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index c4cd0edc..35bd0bbe 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -411,7 +411,7 @@ else if (v instanceof Boolean) { else if (v instanceof ByteBuffer) { ByteBuffer bb = (ByteBuffer) v; int len = bb.remaining(); - if (bb.hasArray()) { + if (bb.hasArray() && !bb.isReadOnly()) { messagePacker.packBinaryHeader(len); messagePacker.writePayload(bb.array(), bb.arrayOffset() + bb.position(), len); } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index e425d643..69055e8d 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -238,6 +238,7 @@ else if (streamReadContext.inArray()) { case BINARY: type = Type.BYTES; int len = messageUnpacker.unpackBinaryHeader(); + _streamReadConstraints.validateStringLength(len); bytesValue = messageUnpacker.readPayload(len); if (isObjectValueSet) { streamReadContext.setCurrentName(new String(bytesValue, MessagePack.UTF8)); @@ -258,6 +259,7 @@ else if (streamReadContext.inArray()) { case EXTENSION: type = Type.EXT; ExtensionTypeHeader header = messageUnpacker.unpackExtensionTypeHeader(); + _streamReadConstraints.validateStringLength(header.getLength()); extensionTypeValue = new MessagePackExtensionType(header.getType(), messageUnpacker.readPayload(header.getLength())); if (isObjectValueSet) { streamReadContext.setCurrentName(deserializedExtensionTypeValue().toString()); @@ -268,7 +270,7 @@ else if (streamReadContext.inArray()) { } break; default: - nextToken = _reportError("Unexpected MessagePack format type: " + type); + nextToken = _reportError("Unexpected MessagePack format type: " + valueType); } currentPosition = messageUnpacker.getTotalReadBytes(); diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java index 1da2574c..f171e13b 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackReadContext.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat; import tools.jackson.core.TokenStreamContext; @@ -182,7 +197,7 @@ public String toString() sb.append('{'); if (currentName != null) { sb.append('"'); - sb.append(currentName); + sb.append(currentName.replace("\\", "\\\\").replace("\"", "\\\"")); sb.append('"'); } else { diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java index 40ec589c..54b41f77 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat; import tools.jackson.core.JsonGenerator; From 71e77d37e5878fa2ec270ab751ca0c14320fe1e7 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 23:16:14 +0900 Subject: [PATCH 20/50] Add missing StreamReadConstraints validations and fix addKeyNode error handling - Validate nesting depth on ARRAY/MAP to reject deeply nested documents - Validate string length after unpackString() for STRING values - Replace IllegalStateException in addKeyNode with _reportError for consistent Jackson error reporting --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 2 +- .../java/org/msgpack/jackson/dataformat/MessagePackParser.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 35bd0bbe..93733c9a 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -479,7 +479,7 @@ private void packArray(NodeArray container) private void addKeyNode(Object key) { if (currentState != IN_OBJECT) { - throw new IllegalStateException(); + _reportError("Can not write a property name, expecting a value"); } nodes.add(new NodeEntryInObject(currentParentElementIndex, key)); } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 69055e8d..2e4cf625 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -163,6 +163,7 @@ else if (streamReadContext.inArray()) { case STRING: type = Type.STRING; stringValue = unpackString(messageUnpacker); + _streamReadConstraints.validateStringLength(stringValue.length()); if (isObjectValueSet) { streamReadContext.setCurrentName(stringValue); nextToken = JsonToken.PROPERTY_NAME; @@ -251,10 +252,12 @@ else if (streamReadContext.inArray()) { case ARRAY: nextToken = JsonToken.START_ARRAY; streamReadContext = streamReadContext.createChildArrayContext(messageUnpacker.unpackArrayHeader()); + _streamReadConstraints.validateNestingDepth(streamReadContext.getNestingDepth()); break; case MAP: nextToken = JsonToken.START_OBJECT; streamReadContext = streamReadContext.createChildObjectContext(messageUnpacker.unpackMapHeader()); + _streamReadConstraints.validateNestingDepth(streamReadContext.getNestingDepth()); break; case EXTENSION: type = Type.EXT; From 44136349d2d0069bb6d4f043259bd976cc761594 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 27 May 2026 23:55:49 +0900 Subject: [PATCH 21/50] Fix BigDecimal serialization: guard against infinite double before BigDecimal.valueOf --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 93733c9a..d20eb590 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -455,7 +455,7 @@ private void packBigDecimal(BigDecimal decimal) if (failedToPackAsBI) { double doubleValue = decimal.doubleValue(); - if (decimal.compareTo(BigDecimal.valueOf(doubleValue)) != 0) { + if (Double.isInfinite(doubleValue) || decimal.compareTo(BigDecimal.valueOf(doubleValue)) != 0) { throw new IllegalArgumentException("MessagePack cannot serialize a BigDecimal that can't be represented as double. " + decimal); } messagePacker.packDouble(doubleValue); From 01654bc38d96dda5afac23f16aadab0666550363 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Thu, 28 May 2026 23:13:29 +0900 Subject: [PATCH 22/50] Add missing Apache 2.0 license headers to JMH benchmark files; fix stale preexisting-issues.md description --- .../dataformat/benchmark/BenchmarkState.java | 15 +++++++++++++++ .../benchmark/MsgpackReadBenchmark.java | 15 +++++++++++++++ .../benchmark/MsgpackWriteBenchmark.java | 15 +++++++++++++++ .../benchmark/WriteUTF8StringBenchmark.java | 15 +++++++++++++++ .../jackson/dataformat/benchmark/model/Image.java | 15 +++++++++++++++ .../dataformat/benchmark/model/MediaContent.java | 15 +++++++++++++++ .../dataformat/benchmark/model/MediaItem.java | 15 +++++++++++++++ .../dataformat/benchmark/model/MediaItems.java | 15 +++++++++++++++ .../dataformat/benchmark/model/Player.java | 15 +++++++++++++++ .../jackson/dataformat/benchmark/model/Size.java | 15 +++++++++++++++ plans/preexisting-issues.md | 7 ++++--- 11 files changed, 154 insertions(+), 3 deletions(-) diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java index 63d43064..8e47ceaa 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/BenchmarkState.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark; import org.msgpack.jackson.dataformat.MessagePackFactory; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java index 8bb1ec9d..f7860bc5 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackReadBenchmark.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark; import org.msgpack.jackson.dataformat.benchmark.model.MediaItem; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java index b534b833..9ed2bfa6 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/MsgpackWriteBenchmark.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark; import org.msgpack.jackson.dataformat.benchmark.model.MediaItem; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java index 4adee1e5..764a0efb 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/WriteUTF8StringBenchmark.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark; import org.msgpack.jackson.dataformat.MessagePackFactory; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java index 1d78c1e5..607d813c 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Image.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; public class Image diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java index 91e30846..9af96caa 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaContent.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; import java.util.ArrayList; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java index b3c3b5a2..0bed7a69 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItem.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; import com.fasterxml.jackson.annotation.JsonPropertyOrder; diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java index fa9fd41f..dd77700f 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/MediaItems.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; public class MediaItems diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java index 58477386..82740c66 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Player.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; public enum Player diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java index 43803743..36e56489 100644 --- a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/model/Size.java @@ -1,3 +1,18 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.jackson.dataformat.benchmark.model; public enum Size diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index 22cf469c..adb519b6 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -186,9 +186,10 @@ they are needed to detect same-stream reuse. Needs the same fix in msgpack-jacks - `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java` — FIXED The `close()` override never sets the closed flag, so `isClosed()` remains false. -Fixed in msgpack-jackson3 by setting `_closed = true` directly (calling `super.close()` -is not viable since it unconditionally closes the underlying stream, ignoring -`AUTO_CLOSE_TARGET`). Needs the same fix in msgpack-jackson. +Fixed in msgpack-jackson3 by delegating to `super.close()`, which sets `_closed` and +handles `AUTO_CLOSE_TARGET` correctly. Our override calls `flush()` first to drain +pending nodes, then delegates the rest to `super.close()`. Needs the same fix in +msgpack-jackson. ## 4. `MessagePackGenerator`: `writeString(Reader, int)` crashes on length -1 From 137926f19f5ed90a6fc4352fd24d48333ef76316 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 10:32:53 +0900 Subject: [PATCH 23/50] Fix null data in MessagePackExtensionType, stale type after NIL/BOOLEAN tokens, stale test comment --- .../jackson/dataformat/MessagePackExtensionType.java | 5 +++-- .../msgpack/jackson/dataformat/MessagePackParser.java | 11 +++++++++++ .../jackson/dataformat/MessagePackParserTest.java | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java index 13041d4b..1b62b80e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java @@ -21,6 +21,7 @@ import tools.jackson.databind.ser.std.StdSerializer; import java.util.Arrays; +import java.util.Objects; @JsonSerialize(using = MessagePackExtensionType.Serializer.class) public class MessagePackExtensionType @@ -31,7 +32,7 @@ public class MessagePackExtensionType public MessagePackExtensionType(byte type, byte[] data) { this.type = type; - this.data = data; + this.data = Objects.requireNonNull(data, "data"); } public byte getType() @@ -73,7 +74,7 @@ public int hashCode() @Override public String toString() { - return "MessagePackExtensionType(type=" + type + ", data.length=" + (data == null ? 0 : data.length) + ")"; + return "MessagePackExtensionType(type=" + type + ", data.length=" + data.length + ")"; } public static class Serializer extends StdSerializer diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 2e4cf625..56bfada4 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -212,10 +212,12 @@ else if (streamReadContext.inArray()) { } break; case NIL: + type = null; messageUnpacker.unpackNil(); nextToken = JsonToken.VALUE_NULL; break; case BOOLEAN: + type = null; boolean b = messageUnpacker.unpackBoolean(); if (isObjectValueSet) { streamReadContext.setCurrentName(Boolean.toString(b)); @@ -290,6 +292,15 @@ protected void _handleEOF() @Override public String getString() { + if (_currToken == JsonToken.VALUE_NULL) { + return null; + } + if (_currToken == JsonToken.VALUE_TRUE) { + return Boolean.TRUE.toString(); + } + if (_currToken == JsonToken.VALUE_FALSE) { + return Boolean.FALSE.toString(); + } switch (type) { case STRING: return stringValue; diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java index c2fd3ae0..adebada6 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java @@ -1124,8 +1124,8 @@ public void testByteArrayReuseResetsUnpackerWhenAutoCloseSourceDisabled() assertEquals(Arrays.asList(1, 2, 3), first); // Second parse with the same byte[] instance and AUTO_CLOSE_SOURCE disabled. - // The unpacker is not reset (src identity match, no AUTO_CLOSE_SOURCE trigger), - // so it continues from EOF and fails to produce the expected result. + // The byte[] source always triggers an unpacker reset (|| src instanceof byte[]), + // so the second parse succeeds and returns the correct result. List second = objectMapper.readValue(bytes, new TypeReference>() {}); assertEquals(Arrays.asList(1, 2, 3), second); } From 87bd4fa6a47d787ed54d24ad9ab73aa0ca373edc Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 10:52:21 +0900 Subject: [PATCH 24/50] Restore comments dropped from v2 during jackson3 port --- .../msgpack/jackson/dataformat/MessagePackGenerator.java | 9 +++++++++ .../msgpack/jackson/dataformat/MessagePackParser.java | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index d20eb590..e8342587 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -77,6 +77,7 @@ public RawUtf8String(byte[] bytes, int offset, int len) private abstract static class Node { + // Root containers have -1. final int parentIndex; public Node(int parentIndex) @@ -91,6 +92,7 @@ public Node(int parentIndex) private abstract static class NodeContainer extends Node { + // Only for containers. int childCount; public NodeContainer(int parentIndex) @@ -159,6 +161,7 @@ int currentStateAsParent() private static final class NodeEntryInObject extends Node { final Object key; + // Lazily initialized. Object value; public NodeEntryInObject(int parentIndex, Object key) @@ -446,6 +449,7 @@ private void packBigDecimal(BigDecimal decimal) MessagePacker messagePacker = getMessagePacker(); boolean failedToPackAsBI = false; try { + //Check to see if this BigDecimal can be converted to BigInteger BigInteger integer = decimal.toBigIntegerExact(); messagePacker.packBigInteger(integer); } @@ -455,6 +459,7 @@ private void packBigDecimal(BigDecimal decimal) if (failedToPackAsBI) { double doubleValue = decimal.doubleValue(); + //Check to make sure this BigDecimal can be represented as a double if (Double.isInfinite(doubleValue) || decimal.compareTo(BigDecimal.valueOf(doubleValue)) != 0) { throw new IllegalArgumentException("MessagePack cannot serialize a BigDecimal that can't be represented as double. " + decimal); } @@ -786,6 +791,9 @@ public JsonGenerator writeNumber(BigDecimal dec) throws JacksonException @Override public JsonGenerator writeNumber(String encodedValue) throws JacksonException { + // There is a room to improve this API's performance while the implementation is robust. + // If users can use other MessagePackGenerator#writeNumber APIs that accept + // proper numeric types not String, it's better to use the other APIs instead. try { try { long l = Long.parseLong(encodedValue); @@ -873,6 +881,7 @@ public void close() throws JacksonException public void flush() throws JacksonException { if (!isElementsClosed) { + // The whole elements are not closed yet. return; } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 56bfada4..53dd15b1 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -94,6 +94,10 @@ private enum Type messageUnpacker = MessagePack.newDefaultUnpacker(input); } else { + // Considering to reuse InputStream with StreamReadFeature.AUTO_CLOSE_SOURCE, + // MessagePackParser needs to use the MessageUnpacker that has the same InputStream + // since it has buffer which has loaded the InputStream data ahead. + // However, it needs to call MessageUnpacker#reset when the source is different from the previous one. if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || messageUnpackerTuple.first() != src || src instanceof byte[]) { messageUnpackerTuple.second().reset(input); } @@ -593,6 +597,7 @@ public TokenStreamLocation currentLocation() @Override public String currentName() { + // Simple, but need to look for START_OBJECT/ARRAY's "off-by-one" thing: if (_currToken == JsonToken.START_OBJECT || _currToken == JsonToken.START_ARRAY) { MessagePackReadContext parent = streamReadContext.getParent(); return parent.currentName(); From 5e20c63dc08329a50d269b38661ed90668b19d06 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 11:03:19 +0900 Subject: [PATCH 25/50] Use singleton MessagePackKeySerializer; fix enum assertion in testNormal --- .../jackson/dataformat/MessagePackSerializerFactory.java | 4 +++- .../jackson/dataformat/MessagePackDataformatForPojoTest.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java index 67ee2e2f..0aa72293 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackSerializerFactory.java @@ -34,9 +34,11 @@ public MessagePackSerializerFactory(SerializerFactoryConfig config) super(config); } + private static final MessagePackKeySerializer KEY_SERIALIZER = new MessagePackKeySerializer(); + @Override public ValueSerializer createKeySerializer(SerializationContext ctxt, JavaType keyType) { - return new MessagePackKeySerializer(); + return KEY_SERIALIZER; } } diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java index 417c1389..c8083189 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java @@ -45,7 +45,7 @@ public void testNormal() assertEquals(normalPojo.d, value.d, 0.000001f); assertArrayEquals(normalPojo.b, value.b); assertEquals(normalPojo.bi, value.bi); - assertEquals(normalPojo.suit, Suit.HEART); + assertEquals(normalPojo.suit, value.suit); assertEquals(normalPojo.sMultibyte, value.sMultibyte); } From 3880f00a0778fcc99b6087708d14ab1cf97325a0 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 11:08:55 +0900 Subject: [PATCH 26/50] Document pre-existing issues 16-18 in preexisting-issues.md --- plans/preexisting-issues.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index adb519b6..fa06d882 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -361,3 +361,36 @@ serialization paths and adds significant complexity. Document the caveat and leave `size` ignored for now. If implemented in the future, both modules should be updated together. +## 16. `MessagePackParser`: `getText()` returns stale value after NIL or BOOLEAN token + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` (getText) +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` — FIXED + +The `type` field is not reset for NIL or BOOLEAN tokens. After parsing a null or boolean value, +calling `getText()` / `getString()` returns stale data from the previous token instead of null / +"true" / "false". Fixed in msgpack-jackson3 by setting `type = null` in both cases and checking +`_currToken` at the top of `getString()`. Needs the same fix in msgpack-jackson. + +## 17. `MessagePackExtensionType`: constructor accepts null `data` + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackExtensionType.java` — FIXED + +The constructor stores `data` without a null check. A null `data` value causes NPE during +serialization in `MessagePackGenerator` (at `extData.length`) or produces a misleading string +from `toString()`. Fixed in msgpack-jackson3 by adding `Objects.requireNonNull(data, "data")`. +Needs the same fix in msgpack-jackson. + +## 18. `MessagePackDataformatForPojoTest`: enum assertion checks fixture, not deserialized value + +**Files:** +- `msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java:48` +- `msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackDataformatForPojoTest.java` — FIXED + +`assertEquals(normalPojo.suit, Suit.HEART)` compares the fixture to a constant, not the +deserialized `value.suit`. Enum serialization/deserialization could be broken without this test +detecting it. Fixed in msgpack-jackson3 to `assertEquals(normalPojo.suit, value.suit)`. +Needs the same fix in msgpack-jackson. + From f8ad97fbe9b6ae6c4b59cefb8c24a3919165b37f Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 11:27:08 +0900 Subject: [PATCH 27/50] Add NULL/BOOL enum values to Type; fix numeric accessor overflow/non-finite handling --- .../jackson/dataformat/MessagePackParser.java | 139 ++++++++++-- .../dataformat/MessagePackParserTest.java | 207 ++++++++++++++++++ 2 files changed, 324 insertions(+), 22 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 53dd15b1..a78829a1 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -58,12 +58,13 @@ public class MessagePackParser private enum Type { - INT, LONG, DOUBLE, STRING, BYTES, BIG_INT, EXT + INT, LONG, DOUBLE, STRING, BYTES, BOOL, BIG_INT, EXT, NULL } private Type type; private int intValue; private long longValue; private double doubleValue; + private boolean booleanValue; private byte[] bytesValue; private String stringValue; private BigInteger biValue; @@ -216,13 +217,14 @@ else if (streamReadContext.inArray()) { } break; case NIL: - type = null; + type = Type.NULL; messageUnpacker.unpackNil(); nextToken = JsonToken.VALUE_NULL; break; case BOOLEAN: - type = null; boolean b = messageUnpacker.unpackBoolean(); + type = Type.BOOL; + booleanValue = b; if (isObjectValueSet) { streamReadContext.setCurrentName(Boolean.toString(b)); nextToken = JsonToken.PROPERTY_NAME; @@ -296,15 +298,6 @@ protected void _handleEOF() @Override public String getString() { - if (_currToken == JsonToken.VALUE_NULL) { - return null; - } - if (_currToken == JsonToken.VALUE_TRUE) { - return Boolean.TRUE.toString(); - } - if (_currToken == JsonToken.VALUE_FALSE) { - return Boolean.FALSE.toString(); - } switch (type) { case STRING: return stringValue; @@ -316,6 +309,8 @@ public String getString() return String.valueOf(longValue); case DOUBLE: return String.valueOf(doubleValue); + case BOOL: + return Boolean.toString(booleanValue); case BIG_INT: return String.valueOf(biValue); case EXT: @@ -325,6 +320,8 @@ public String getString() catch (IOException e) { throw _wrapIOFailure(e); } + case NULL: + return "null"; default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -364,6 +361,13 @@ public byte[] getBinaryValue(Base64Variant b64variant) return stringValue.getBytes(MessagePack.UTF8); case EXT: return extensionTypeValue.getData(); + case INT: + case LONG: + case DOUBLE: + case BOOL: + case BIG_INT: + case NULL: + return _reportError("Current token (" + _currToken + ") not of binary type"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -381,6 +385,12 @@ public Number getNumberValue() return doubleValue; case BIG_INT: return biValue; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -393,11 +403,31 @@ public int getIntValue() case INT: return intValue; case LONG: + if (longValue < Integer.MIN_VALUE || longValue > Integer.MAX_VALUE) { + return _reportError("Numeric value (" + longValue + ") out of range for `int`"); + } return (int) longValue; case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `int`"); + } + if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) { + return _reportError("Numeric value (" + doubleValue + ") out of range for `int`"); + } return (int) doubleValue; case BIG_INT: - return biValue.intValue(); + try { + return biValue.intValueExact(); + } + catch (ArithmeticException e) { + return _reportError("Numeric value (" + biValue + ") out of range for `int`"); + } + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -412,9 +442,26 @@ public long getLongValue() case LONG: return longValue; case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `long`"); + } + if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { + return _reportError("Numeric value (" + doubleValue + ") out of range for `long`"); + } return (long) doubleValue; case BIG_INT: - return biValue.longValue(); + try { + return biValue.longValueExact(); + } + catch (ArithmeticException e) { + return _reportError("Numeric value (" + biValue + ") out of range for `long`"); + } + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -429,9 +476,18 @@ public BigInteger getBigIntegerValue() case LONG: return BigInteger.valueOf(longValue); case DOUBLE: - return BigInteger.valueOf((long) doubleValue); + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigInteger"); + } + return BigDecimal.valueOf(doubleValue).toBigInteger(); // truncates fractional part case BIG_INT: return biValue; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -440,6 +496,8 @@ public BigInteger getBigIntegerValue() @Override public float getFloatValue() { + // No bounds/range check: a finite double or large BigInteger may overflow to + // Float.POSITIVE_INFINITY. This is intentional — same as ParserBase and CBORParser. switch (type) { case INT: return (float) intValue; @@ -449,6 +507,12 @@ public float getFloatValue() return (float) doubleValue; case BIG_INT: return biValue.floatValue(); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -457,15 +521,23 @@ public float getFloatValue() @Override public double getDoubleValue() { - switch (type) { - case INT: - return intValue; + // No bounds/range check: large BigInteger may overflow to Double.POSITIVE_INFINITY, + // and large long values may lose precision. Intentional — same as ParserBase. + switch (type) { + case INT: + return intValue; case LONG: return (double) longValue; case DOUBLE: return doubleValue; case BIG_INT: return biValue.doubleValue(); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -474,15 +546,24 @@ public double getDoubleValue() @Override public BigDecimal getDecimalValue() { - switch (type) { - case INT: - return BigDecimal.valueOf(intValue); + switch (type) { + case INT: + return BigDecimal.valueOf(intValue); case LONG: return BigDecimal.valueOf(longValue); case DOUBLE: - return BigDecimal.valueOf(doubleValue); + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigDecimal"); + } + return BigDecimal.valueOf(doubleValue); case BIG_INT: return new BigDecimal(biValue); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -513,6 +594,14 @@ public Object getEmbeddedObject() catch (IOException e) { throw _wrapIOFailure(e); } + case INT: + case LONG: + case DOUBLE: + case BOOL: + case BIG_INT: + case STRING: + case NULL: + return _reportError("Current token (" + _currToken + ") not of embeddable type"); default: return _reportError("Unexpected MessagePack value type: " + type); } @@ -530,6 +619,12 @@ public NumberType getNumberType() return NumberType.DOUBLE; case BIG_INT: return NumberType.BIG_INTEGER; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return null; default: return _reportError("Unexpected MessagePack value type: " + type); } diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java index adebada6..5aab2565 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java @@ -53,8 +53,10 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.fail; public class MessagePackParserTest extends MessagePackDataformatTestBase @@ -1129,4 +1131,209 @@ public void testByteArrayReuseResetsUnpackerWhenAutoCloseSourceDisabled() List second = objectMapper.readValue(bytes, new TypeReference>() {}); assertEquals(Arrays.asList(1, 2, 3), second); } + + @Test + public void testGetStringOnNullToken() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packNil(); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_NULL, p.nextToken()); + assertEquals("null", p.getString()); + } + } + + @Test + public void testGetStringOnBoolToken() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packBoolean(true); + packer.packBoolean(false); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_TRUE, p.nextToken()); + assertEquals("true", p.getString()); + assertEquals(JsonToken.VALUE_FALSE, p.nextToken()); + assertEquals("false", p.getString()); + } + } + + @Test + public void testNumericAccessorsOnNullTokenThrow() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packNil(); + } + byte[] bytes = out.toByteArray(); + MessagePackFactory factory = new MessagePackFactory(); + + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getIntValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getLongValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getDoubleValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getFloatValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getBigIntegerValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getDecimalValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getNumberValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + } + + @Test + public void testNumericAccessorsOnBoolTokenThrow() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packBoolean(true); + } + byte[] bytes = out.toByteArray(); + MessagePackFactory factory = new MessagePackFactory(); + + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getIntValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getLongValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + p.nextToken(); + try { + p.getDoubleValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + } + } + + @Test + public void testGetNumberTypeOnNonNumericTokens() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packNil(); + packer.packBoolean(true); + packer.packString("hello"); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_NULL, p.nextToken()); + assertNull(p.getNumberType()); + assertEquals(JsonToken.VALUE_TRUE, p.nextToken()); + assertNull(p.getNumberType()); + assertEquals(JsonToken.VALUE_STRING, p.nextToken()); + assertNull(p.getNumberType()); + } + } + + @Test + public void testGetIntValueFromOutOfRangeDoubleThrows() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packDouble(1e30); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken()); + try { + p.getIntValue(); + fail("expected exception for out-of-range double"); + } + catch (JacksonException ignored) { } + } + } + + @Test + public void testGetLongValueFromOutOfRangeDoubleThrows() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packDouble(1e30); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken()); + try { + p.getLongValue(); + fail("expected exception for out-of-range double"); + } + catch (JacksonException ignored) { } + } + } + + @Test + public void testGetIntValueFromFractionalDoubleTruncates() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packDouble(3.7); + } + MessagePackFactory factory = new MessagePackFactory(); + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), out.toByteArray())) { + assertEquals(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken()); + assertEquals(3, p.getIntValue()); + } + } } From 499207b8b3ca31c5975e64d5a2fbedb3a7fc146c Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 12:15:41 +0900 Subject: [PATCH 28/50] Inherit ParserBase in MessagePackParser: remove custom numeric fields, add _parseNumericValue stubs --- .../jackson/dataformat/MessagePackParser.java | 333 +++++++----------- 1 file changed, 122 insertions(+), 211 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index a78829a1..d5159c21 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -23,7 +23,7 @@ import tools.jackson.core.TokenStreamContext; import tools.jackson.core.TokenStreamLocation; import tools.jackson.core.Version; -import tools.jackson.core.base.ParserMinimalBase; +import tools.jackson.core.base.ParserBase; import tools.jackson.core.exc.UnexpectedEndOfInputException; import tools.jackson.core.io.IOContext; import tools.jackson.core.json.DupDetector; @@ -39,7 +39,7 @@ import java.math.BigInteger; public class MessagePackParser - extends ParserMinimalBase + extends ParserBase { private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; @@ -49,7 +49,6 @@ public class MessagePackParser private MessagePackReadContext streamReadContext; - private boolean isClosed; private long tokenPosition; private long currentPosition; private final IOContext ioContext; @@ -61,13 +60,9 @@ private enum Type INT, LONG, DOUBLE, STRING, BYTES, BOOL, BIG_INT, EXT, NULL } private Type type; - private int intValue; - private long longValue; - private double doubleValue; private boolean booleanValue; private byte[] bytesValue; private String stringValue; - private BigInteger biValue; private MessagePackExtensionType extensionTypeValue; MessagePackParser(ObjectReadContext readCtxt, @@ -137,6 +132,7 @@ public JsonToken nextToken() throws JacksonException private JsonToken _nextToken() throws IOException { + _numTypesValid = NR_UNKNOWN; tokenPosition = messageUnpacker.getTotalReadBytes(); boolean isObjectValueSet = streamReadContext.inObject() && _currToken != JsonToken.PROPERTY_NAME; @@ -184,26 +180,30 @@ else if (streamReadContext.inArray()) { BigInteger bi = messageUnpacker.unpackBigInteger(); if (0 <= bi.compareTo(LONG_MIN) && bi.compareTo(LONG_MAX) <= 0) { type = Type.LONG; - longValue = bi.longValue(); - v = longValue; + _numberLong = bi.longValue(); + _numTypesValid = NR_LONG; + v = _numberLong; } else { type = Type.BIG_INT; - biValue = bi; - v = biValue; + _numberBigInt = bi; + _numTypesValid = NR_BIGINT; + v = _numberBigInt; } break; default: long l = messageUnpacker.unpackLong(); if (Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE) { type = Type.INT; - intValue = (int) l; - v = intValue; + _numberInt = (int) l; + _numTypesValid = NR_INT; + v = _numberInt; } else { type = Type.LONG; - longValue = l; - v = longValue; + _numberLong = l; + _numTypesValid = NR_LONG; + v = _numberLong; } break; } @@ -235,9 +235,10 @@ else if (streamReadContext.inArray()) { break; case FLOAT: type = Type.DOUBLE; - doubleValue = messageUnpacker.unpackDouble(); + _numberDouble = messageUnpacker.unpackDouble(); + _numTypesValid = NR_DOUBLE; if (isObjectValueSet) { - streamReadContext.setCurrentName(String.valueOf(doubleValue)); + streamReadContext.setCurrentName(String.valueOf(_numberDouble)); nextToken = JsonToken.PROPERTY_NAME; } else { @@ -304,15 +305,15 @@ public String getString() case BYTES: return new String(bytesValue, MessagePack.UTF8); case INT: - return String.valueOf(intValue); + return String.valueOf(_numberInt); case LONG: - return String.valueOf(longValue); + return String.valueOf(_numberLong); case DOUBLE: - return String.valueOf(doubleValue); + return String.valueOf(_numberDouble); case BOOL: return Boolean.toString(booleanValue); case BIG_INT: - return String.valueOf(biValue); + return String.valueOf(_numberBigInt); case EXT: try { return deserializedExtensionTypeValue().toString(); @@ -373,200 +374,101 @@ public byte[] getBinaryValue(Base64Variant b64variant) } } - @Override - public Number getNumberValue() - { - switch (type) { - case INT: - return intValue; - case LONG: - return longValue; - case DOUBLE: - return doubleValue; - case BIG_INT: - return biValue; - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); - } - } + // getNumberValue(), getFloatValue(), getDoubleValue() are inherited from ParserBase. @Override - public int getIntValue() + public int getIntValue() throws JacksonException { - switch (type) { - case INT: - return intValue; - case LONG: - if (longValue < Integer.MIN_VALUE || longValue > Integer.MAX_VALUE) { - return _reportError("Numeric value (" + longValue + ") out of range for `int`"); - } - return (int) longValue; - case DOUBLE: - if (!Double.isFinite(doubleValue)) { - return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `int`"); - } - if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) { - return _reportError("Numeric value (" + doubleValue + ") out of range for `int`"); - } - return (int) doubleValue; - case BIG_INT: - try { - return biValue.intValueExact(); + if ((_numTypesValid & NR_INT) != 0) { + return _numberInt; + } + if (_numTypesValid == NR_UNKNOWN) { + _parseNumericValue(NR_INT); + } + if ((_numTypesValid & NR_INT) == 0) { + // Conversion from LONG, DOUBLE, or BIGINT — need NaN/range checks. + if ((_numTypesValid & NR_DOUBLE) != 0) { + if (!Double.isFinite(_numberDouble)) { + return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to `int`"); } - catch (ArithmeticException e) { - return _reportError("Numeric value (" + biValue + ") out of range for `int`"); + if (_numberDouble < Integer.MIN_VALUE || _numberDouble > Integer.MAX_VALUE) { + return _reportError("Numeric value (" + _numberDouble + ") out of range for `int`"); } - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + } + convertNumberToInt(); } + return _numberInt; } @Override - public long getLongValue() + public long getLongValue() throws JacksonException { - switch (type) { - case INT: - return intValue; - case LONG: - return longValue; - case DOUBLE: - if (!Double.isFinite(doubleValue)) { - return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `long`"); - } - if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { - return _reportError("Numeric value (" + doubleValue + ") out of range for `long`"); - } - return (long) doubleValue; - case BIG_INT: - try { - return biValue.longValueExact(); + if ((_numTypesValid & NR_LONG) != 0) { + return _numberLong; + } + if (_numTypesValid == NR_UNKNOWN) { + _parseNumericValue(NR_LONG); + } + if ((_numTypesValid & NR_LONG) == 0) { + if ((_numTypesValid & NR_DOUBLE) != 0) { + if (!Double.isFinite(_numberDouble)) { + return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to `long`"); } - catch (ArithmeticException e) { - return _reportError("Numeric value (" + biValue + ") out of range for `long`"); + if (_numberDouble < Long.MIN_VALUE || _numberDouble > Long.MAX_VALUE) { + return _reportError("Numeric value (" + _numberDouble + ") out of range for `long`"); } - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + } + convertNumberToLong(); } + return _numberLong; } @Override - public BigInteger getBigIntegerValue() + public BigInteger getBigIntegerValue() throws JacksonException { - switch (type) { - case INT: - return BigInteger.valueOf(intValue); - case LONG: - return BigInteger.valueOf(longValue); - case DOUBLE: - if (!Double.isFinite(doubleValue)) { - return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigInteger"); + if ((_numTypesValid & NR_BIGINT) != 0) { + return _numberBigInt; + } + if (_numTypesValid == NR_UNKNOWN) { + _parseNumericValue(NR_BIGINT); + } + if ((_numTypesValid & NR_BIGINT) == 0) { + if ((_numTypesValid & NR_DOUBLE) != 0) { + if (!Double.isFinite(_numberDouble)) { + return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to BigInteger"); } - return BigDecimal.valueOf(doubleValue).toBigInteger(); // truncates fractional part - case BIG_INT: - return biValue; - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + // truncates fractional part + _numberBigInt = BigDecimal.valueOf(_numberDouble).toBigInteger(); + _numTypesValid |= NR_BIGINT; + return _numberBigInt; + } + convertNumberToBigInteger(); } + return _numberBigInt; } @Override - public float getFloatValue() + public BigDecimal getDecimalValue() throws JacksonException { - // No bounds/range check: a finite double or large BigInteger may overflow to - // Float.POSITIVE_INFINITY. This is intentional — same as ParserBase and CBORParser. - switch (type) { - case INT: - return (float) intValue; - case LONG: - return (float) longValue; - case DOUBLE: - return (float) doubleValue; - case BIG_INT: - return biValue.floatValue(); - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + if ((_numTypesValid & NR_BIGDECIMAL) != 0) { + return _numberBigDecimal; } - } - - @Override - public double getDoubleValue() - { - // No bounds/range check: large BigInteger may overflow to Double.POSITIVE_INFINITY, - // and large long values may lose precision. Intentional — same as ParserBase. - switch (type) { - case INT: - return intValue; - case LONG: - return (double) longValue; - case DOUBLE: - return doubleValue; - case BIG_INT: - return biValue.doubleValue(); - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + if (_numTypesValid == NR_UNKNOWN) { + _parseNumericValue(NR_BIGDECIMAL); } - } - - @Override - public BigDecimal getDecimalValue() - { - switch (type) { - case INT: - return BigDecimal.valueOf(intValue); - case LONG: - return BigDecimal.valueOf(longValue); - case DOUBLE: - if (!Double.isFinite(doubleValue)) { - return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigDecimal"); + if ((_numTypesValid & NR_BIGDECIMAL) == 0) { + if ((_numTypesValid & NR_DOUBLE) != 0) { + if (!Double.isFinite(_numberDouble)) { + return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to BigDecimal"); } - return BigDecimal.valueOf(doubleValue); - case BIG_INT: - return new BigDecimal(biValue); - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); - default: - return _reportError("Unexpected MessagePack value type: " + type); + // bypass ParserBase's text-based conversion to avoid parsing "NaN"/"Infinity" + _numberBigDecimal = BigDecimal.valueOf(_numberDouble); + _numTypesValid |= NR_BIGDECIMAL; + return _numberBigDecimal; + } + convertNumberToBigDecimal(); } + return _numberBigDecimal; } private Object deserializedExtensionTypeValue() @@ -610,43 +512,52 @@ public Object getEmbeddedObject() @Override public NumberType getNumberType() { - switch (type) { - case INT: + // Check _currToken directly (like CBORParser) to avoid _parseNumericValue() + // being called for non-numeric tokens. + if (_currToken == JsonToken.VALUE_NUMBER_INT) { + if ((_numTypesValid & NR_INT) != 0) { return NumberType.INT; - case LONG: + } + if ((_numTypesValid & NR_LONG) != 0) { return NumberType.LONG; - case DOUBLE: - return NumberType.DOUBLE; - case BIG_INT: - return NumberType.BIG_INTEGER; - case NULL: - case BOOL: - case STRING: - case BYTES: - case EXT: - return null; - default: - return _reportError("Unexpected MessagePack value type: " + type); + } + return NumberType.BIG_INTEGER; } + if (_currToken == JsonToken.VALUE_NUMBER_FLOAT) { + return NumberType.DOUBLE; + } + return null; } @Override - protected void _closeInput() throws IOException + protected void _parseNumericValue(int expType) throws JacksonException { - if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { - messageUnpacker.close(); + // No lazy decoding — numbers are fully parsed eagerly in _nextToken(). + // If we get here the current token is not numeric. + if (_currToken != JsonToken.VALUE_NUMBER_INT && _currToken != JsonToken.VALUE_NUMBER_FLOAT) { + _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); } } @Override - protected void _releaseBuffers() + protected int _parseIntValue() throws JacksonException { + _parseNumericValue(NR_INT); + return 0; // unreachable } @Override - public boolean isClosed() + protected void _closeInput() throws IOException + { + if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { + messageUnpacker.close(); + } + } + + @Override + protected void _releaseBuffers() { - return isClosed; + super._releaseBuffers(); } @Override @@ -659,7 +570,7 @@ public void close() throw _wrapIOFailure(e); } finally { - isClosed = true; + _closed = true; if (ownsThreadLocalUnpacker) { Tuple tuple = messageUnpackerHolder.get(); if (tuple != null && tuple.first() instanceof byte[]) { @@ -722,7 +633,7 @@ public void assignCurrentValue(Object v) public boolean isNaN() { if (type == Type.DOUBLE) { - return Double.isNaN(doubleValue) || Double.isInfinite(doubleValue); + return !Double.isFinite(_numberDouble); } return false; } From ce02fcbfb834357e54c8a662ddf66616042a84e9 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 13:02:49 +0900 Subject: [PATCH 29/50] Revert "Inherit ParserBase in MessagePackParser: remove custom numeric fields, add _parseNumericValue stubs" This reverts commit 499207b8b3ca31c5975e64d5a2fbedb3a7fc146c. --- .../jackson/dataformat/MessagePackParser.java | 333 +++++++++++------- 1 file changed, 211 insertions(+), 122 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index d5159c21..a78829a1 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -23,7 +23,7 @@ import tools.jackson.core.TokenStreamContext; import tools.jackson.core.TokenStreamLocation; import tools.jackson.core.Version; -import tools.jackson.core.base.ParserBase; +import tools.jackson.core.base.ParserMinimalBase; import tools.jackson.core.exc.UnexpectedEndOfInputException; import tools.jackson.core.io.IOContext; import tools.jackson.core.json.DupDetector; @@ -39,7 +39,7 @@ import java.math.BigInteger; public class MessagePackParser - extends ParserBase + extends ParserMinimalBase { private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; @@ -49,6 +49,7 @@ public class MessagePackParser private MessagePackReadContext streamReadContext; + private boolean isClosed; private long tokenPosition; private long currentPosition; private final IOContext ioContext; @@ -60,9 +61,13 @@ private enum Type INT, LONG, DOUBLE, STRING, BYTES, BOOL, BIG_INT, EXT, NULL } private Type type; + private int intValue; + private long longValue; + private double doubleValue; private boolean booleanValue; private byte[] bytesValue; private String stringValue; + private BigInteger biValue; private MessagePackExtensionType extensionTypeValue; MessagePackParser(ObjectReadContext readCtxt, @@ -132,7 +137,6 @@ public JsonToken nextToken() throws JacksonException private JsonToken _nextToken() throws IOException { - _numTypesValid = NR_UNKNOWN; tokenPosition = messageUnpacker.getTotalReadBytes(); boolean isObjectValueSet = streamReadContext.inObject() && _currToken != JsonToken.PROPERTY_NAME; @@ -180,30 +184,26 @@ else if (streamReadContext.inArray()) { BigInteger bi = messageUnpacker.unpackBigInteger(); if (0 <= bi.compareTo(LONG_MIN) && bi.compareTo(LONG_MAX) <= 0) { type = Type.LONG; - _numberLong = bi.longValue(); - _numTypesValid = NR_LONG; - v = _numberLong; + longValue = bi.longValue(); + v = longValue; } else { type = Type.BIG_INT; - _numberBigInt = bi; - _numTypesValid = NR_BIGINT; - v = _numberBigInt; + biValue = bi; + v = biValue; } break; default: long l = messageUnpacker.unpackLong(); if (Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE) { type = Type.INT; - _numberInt = (int) l; - _numTypesValid = NR_INT; - v = _numberInt; + intValue = (int) l; + v = intValue; } else { type = Type.LONG; - _numberLong = l; - _numTypesValid = NR_LONG; - v = _numberLong; + longValue = l; + v = longValue; } break; } @@ -235,10 +235,9 @@ else if (streamReadContext.inArray()) { break; case FLOAT: type = Type.DOUBLE; - _numberDouble = messageUnpacker.unpackDouble(); - _numTypesValid = NR_DOUBLE; + doubleValue = messageUnpacker.unpackDouble(); if (isObjectValueSet) { - streamReadContext.setCurrentName(String.valueOf(_numberDouble)); + streamReadContext.setCurrentName(String.valueOf(doubleValue)); nextToken = JsonToken.PROPERTY_NAME; } else { @@ -305,15 +304,15 @@ public String getString() case BYTES: return new String(bytesValue, MessagePack.UTF8); case INT: - return String.valueOf(_numberInt); + return String.valueOf(intValue); case LONG: - return String.valueOf(_numberLong); + return String.valueOf(longValue); case DOUBLE: - return String.valueOf(_numberDouble); + return String.valueOf(doubleValue); case BOOL: return Boolean.toString(booleanValue); case BIG_INT: - return String.valueOf(_numberBigInt); + return String.valueOf(biValue); case EXT: try { return deserializedExtensionTypeValue().toString(); @@ -374,101 +373,200 @@ public byte[] getBinaryValue(Base64Variant b64variant) } } - // getNumberValue(), getFloatValue(), getDoubleValue() are inherited from ParserBase. - @Override - public int getIntValue() throws JacksonException + public Number getNumberValue() { - if ((_numTypesValid & NR_INT) != 0) { - return _numberInt; - } - if (_numTypesValid == NR_UNKNOWN) { - _parseNumericValue(NR_INT); + switch (type) { + case INT: + return intValue; + case LONG: + return longValue; + case DOUBLE: + return doubleValue; + case BIG_INT: + return biValue; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - if ((_numTypesValid & NR_INT) == 0) { - // Conversion from LONG, DOUBLE, or BIGINT — need NaN/range checks. - if ((_numTypesValid & NR_DOUBLE) != 0) { - if (!Double.isFinite(_numberDouble)) { - return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to `int`"); + } + + @Override + public int getIntValue() + { + switch (type) { + case INT: + return intValue; + case LONG: + if (longValue < Integer.MIN_VALUE || longValue > Integer.MAX_VALUE) { + return _reportError("Numeric value (" + longValue + ") out of range for `int`"); } - if (_numberDouble < Integer.MIN_VALUE || _numberDouble > Integer.MAX_VALUE) { - return _reportError("Numeric value (" + _numberDouble + ") out of range for `int`"); + return (int) longValue; + case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `int`"); } - } - convertNumberToInt(); + if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) { + return _reportError("Numeric value (" + doubleValue + ") out of range for `int`"); + } + return (int) doubleValue; + case BIG_INT: + try { + return biValue.intValueExact(); + } + catch (ArithmeticException e) { + return _reportError("Numeric value (" + biValue + ") out of range for `int`"); + } + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - return _numberInt; } @Override - public long getLongValue() throws JacksonException + public long getLongValue() { - if ((_numTypesValid & NR_LONG) != 0) { - return _numberLong; - } - if (_numTypesValid == NR_UNKNOWN) { - _parseNumericValue(NR_LONG); - } - if ((_numTypesValid & NR_LONG) == 0) { - if ((_numTypesValid & NR_DOUBLE) != 0) { - if (!Double.isFinite(_numberDouble)) { - return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to `long`"); + switch (type) { + case INT: + return intValue; + case LONG: + return longValue; + case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to `long`"); } - if (_numberDouble < Long.MIN_VALUE || _numberDouble > Long.MAX_VALUE) { - return _reportError("Numeric value (" + _numberDouble + ") out of range for `long`"); + if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { + return _reportError("Numeric value (" + doubleValue + ") out of range for `long`"); } - } - convertNumberToLong(); + return (long) doubleValue; + case BIG_INT: + try { + return biValue.longValueExact(); + } + catch (ArithmeticException e) { + return _reportError("Numeric value (" + biValue + ") out of range for `long`"); + } + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - return _numberLong; } @Override - public BigInteger getBigIntegerValue() throws JacksonException + public BigInteger getBigIntegerValue() { - if ((_numTypesValid & NR_BIGINT) != 0) { - return _numberBigInt; - } - if (_numTypesValid == NR_UNKNOWN) { - _parseNumericValue(NR_BIGINT); - } - if ((_numTypesValid & NR_BIGINT) == 0) { - if ((_numTypesValid & NR_DOUBLE) != 0) { - if (!Double.isFinite(_numberDouble)) { - return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to BigInteger"); + switch (type) { + case INT: + return BigInteger.valueOf(intValue); + case LONG: + return BigInteger.valueOf(longValue); + case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigInteger"); } - // truncates fractional part - _numberBigInt = BigDecimal.valueOf(_numberDouble).toBigInteger(); - _numTypesValid |= NR_BIGINT; - return _numberBigInt; - } - convertNumberToBigInteger(); + return BigDecimal.valueOf(doubleValue).toBigInteger(); // truncates fractional part + case BIG_INT: + return biValue; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - return _numberBigInt; } @Override - public BigDecimal getDecimalValue() throws JacksonException + public float getFloatValue() { - if ((_numTypesValid & NR_BIGDECIMAL) != 0) { - return _numberBigDecimal; + // No bounds/range check: a finite double or large BigInteger may overflow to + // Float.POSITIVE_INFINITY. This is intentional — same as ParserBase and CBORParser. + switch (type) { + case INT: + return (float) intValue; + case LONG: + return (float) longValue; + case DOUBLE: + return (float) doubleValue; + case BIG_INT: + return biValue.floatValue(); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - if (_numTypesValid == NR_UNKNOWN) { - _parseNumericValue(NR_BIGDECIMAL); + } + + @Override + public double getDoubleValue() + { + // No bounds/range check: large BigInteger may overflow to Double.POSITIVE_INFINITY, + // and large long values may lose precision. Intentional — same as ParserBase. + switch (type) { + case INT: + return intValue; + case LONG: + return (double) longValue; + case DOUBLE: + return doubleValue; + case BIG_INT: + return biValue.doubleValue(); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - if ((_numTypesValid & NR_BIGDECIMAL) == 0) { - if ((_numTypesValid & NR_DOUBLE) != 0) { - if (!Double.isFinite(_numberDouble)) { - return _reportError("Cannot convert non-finite double (" + _numberDouble + ") to BigDecimal"); + } + + @Override + public BigDecimal getDecimalValue() + { + switch (type) { + case INT: + return BigDecimal.valueOf(intValue); + case LONG: + return BigDecimal.valueOf(longValue); + case DOUBLE: + if (!Double.isFinite(doubleValue)) { + return _reportError("Cannot convert non-finite double (" + doubleValue + ") to BigDecimal"); } - // bypass ParserBase's text-based conversion to avoid parsing "NaN"/"Infinity" - _numberBigDecimal = BigDecimal.valueOf(_numberDouble); - _numTypesValid |= NR_BIGDECIMAL; - return _numberBigDecimal; - } - convertNumberToBigDecimal(); + return BigDecimal.valueOf(doubleValue); + case BIG_INT: + return new BigDecimal(biValue); + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + default: + return _reportError("Unexpected MessagePack value type: " + type); } - return _numberBigDecimal; } private Object deserializedExtensionTypeValue() @@ -512,40 +610,26 @@ public Object getEmbeddedObject() @Override public NumberType getNumberType() { - // Check _currToken directly (like CBORParser) to avoid _parseNumericValue() - // being called for non-numeric tokens. - if (_currToken == JsonToken.VALUE_NUMBER_INT) { - if ((_numTypesValid & NR_INT) != 0) { + switch (type) { + case INT: return NumberType.INT; - } - if ((_numTypesValid & NR_LONG) != 0) { + case LONG: return NumberType.LONG; - } - return NumberType.BIG_INTEGER; - } - if (_currToken == JsonToken.VALUE_NUMBER_FLOAT) { - return NumberType.DOUBLE; - } - return null; - } - - @Override - protected void _parseNumericValue(int expType) throws JacksonException - { - // No lazy decoding — numbers are fully parsed eagerly in _nextToken(). - // If we get here the current token is not numeric. - if (_currToken != JsonToken.VALUE_NUMBER_INT && _currToken != JsonToken.VALUE_NUMBER_FLOAT) { - _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + case DOUBLE: + return NumberType.DOUBLE; + case BIG_INT: + return NumberType.BIG_INTEGER; + case NULL: + case BOOL: + case STRING: + case BYTES: + case EXT: + return null; + default: + return _reportError("Unexpected MessagePack value type: " + type); } } - @Override - protected int _parseIntValue() throws JacksonException - { - _parseNumericValue(NR_INT); - return 0; // unreachable - } - @Override protected void _closeInput() throws IOException { @@ -557,7 +641,12 @@ protected void _closeInput() throws IOException @Override protected void _releaseBuffers() { - super._releaseBuffers(); + } + + @Override + public boolean isClosed() + { + return isClosed; } @Override @@ -570,7 +659,7 @@ public void close() throw _wrapIOFailure(e); } finally { - _closed = true; + isClosed = true; if (ownsThreadLocalUnpacker) { Tuple tuple = messageUnpackerHolder.get(); if (tuple != null && tuple.first() instanceof byte[]) { @@ -633,7 +722,7 @@ public void assignCurrentValue(Object v) public boolean isNaN() { if (type == Type.DOUBLE) { - return !Double.isFinite(_numberDouble); + return Double.isNaN(doubleValue) || Double.isInfinite(doubleValue); } return false; } From ee7ece4b4dd8612247d447b844f60e6897d133d6 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 17:00:32 +0900 Subject: [PATCH 30/50] Address feedback --- .../jackson/dataformat/MessagePackGenerator.java | 8 ++++++-- .../msgpack/jackson/dataformat/MessagePackParser.java | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index e8342587..329a60f8 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -872,8 +872,12 @@ public void writeExtensionType(MessagePackExtensionType extensionType) public void close() throws JacksonException { if (!_closed) { - flush(); - super.close(); + try { + flush(); + } + finally { + super.close(); + } } } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index a78829a1..8722021c 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -137,6 +137,7 @@ public JsonToken nextToken() throws JacksonException private JsonToken _nextToken() throws IOException { + type = null; tokenPosition = messageUnpacker.getTotalReadBytes(); boolean isObjectValueSet = streamReadContext.inObject() && _currToken != JsonToken.PROPERTY_NAME; @@ -298,6 +299,9 @@ protected void _handleEOF() @Override public String getString() { + if (type == null) { + return _currToken == null ? null : _currToken.asString(); + } switch (type) { case STRING: return stringValue; @@ -663,6 +667,12 @@ public void close() if (ownsThreadLocalUnpacker) { Tuple tuple = messageUnpackerHolder.get(); if (tuple != null && tuple.first() instanceof byte[]) { + try { + tuple.second().close(); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); } } From 00daf6c0b18d50e7f94a1cc1a8d34d5453a9bec6 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 17:11:02 +0900 Subject: [PATCH 31/50] Address feedback --- .../dataformat/MessagePackGenerator.java | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 329a60f8..fc86bd95 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -812,19 +812,28 @@ public JsonGenerator writeNumber(String encodedValue) throws JacksonException } try { - double d = Double.parseDouble(encodedValue); - addValueNode(d); - return this; - } - catch (NumberFormatException ignored) { - } + BigDecimal bd = new BigDecimal(encodedValue); + double d = bd.doubleValue(); - try { - BigDecimal bc = new BigDecimal(encodedValue); - addValueNode(bc); + // Check if the double can perfectly represent the exact decimal value. + if (bd.compareTo(new BigDecimal(String.valueOf(d))) == 0) { + // It's a safe ordinary floating-point number. + addValueNode(d); + } + else { + // It has more precision than a double can handle. + addValueNode(bd); + } return this; } - catch (NumberFormatException ignored) { + catch (NumberFormatException e) { + // Fall back for NaN, Infinity, -Infinity which BigDecimal rejects. + try { + double d = Double.parseDouble(encodedValue); + addValueNode(d); + } + catch (NumberFormatException ignored) { + } } throw new NumberFormatException(encodedValue); From 9183b320ba5c94ea59bab92634fdaee82a53bd6e Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 19:02:25 +0900 Subject: [PATCH 32/50] Detect writeName-without-value before writeEndObject Add isExpectingValue() to MessagePackWriteContext and check it in writeEndObject(). Previously, calling writeName then immediately writeEndObject silently produced a corrupt MessagePack map (header count off by one). Now it throws a StreamWriteException, consistent with CBORGenerator's canCloseObject() check. --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 3 +++ .../msgpack/jackson/dataformat/MessagePackWriteContext.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index fc86bd95..4afd8859 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -353,6 +353,9 @@ public JsonGenerator writeEndObject() throws JacksonException if (currentState != IN_OBJECT) { _reportError("Current context not an object but " + currentStateStr()); } + if (writeContext.isExpectingValue()) { + _reportError("Cannot close Object, property name written but no value"); + } endCurrentContainer(); return this; } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java index 701f7e91..cf8fbc5e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackWriteContext.java @@ -80,6 +80,11 @@ public MessagePackWriteContext getParent() return parent; } + boolean isExpectingValue() + { + return _type == TYPE_OBJECT && gotName; + } + boolean writeValue() { if (_type == TYPE_OBJECT) { From b3c442fc2c07ccf6c109143a1d028ceb148c5efe Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 19:13:14 +0900 Subject: [PATCH 33/50] Add inline comments to pre-empt recurring AI review suggestions - writeString(Reader,int): explain why chunk size is capped rather than pre-allocating len (OOM risk for large caller-provided hints) - _releaseBuffers(): explain why no null check on ThreadLocal.get() (single-threaded contract; NPE intentionally surfaces cross-thread misuse) - MessagePackParser.close(): note single-threaded contract - TimestampExtensionModule: note per-call allocation is a known limitation carried from v2, deferred as future optimization - build.sbt isJava17Plus: note that getOrElse(false) is intentionally conservative for non-numeric version strings (e.g. early-access builds) --- build.sbt | 2 ++ .../msgpack/jackson/dataformat/MessagePackGenerator.java | 6 ++++++ .../org/msgpack/jackson/dataformat/MessagePackParser.java | 2 ++ .../jackson/dataformat/TimestampExtensionModule.java | 4 ++++ 4 files changed, 14 insertions(+) diff --git a/build.sbt b/build.sbt index 7d5215fc..0f3c280e 100644 --- a/build.sbt +++ b/build.sbt @@ -103,6 +103,8 @@ val junitInterface = "com.github.sbt" % "junit-interface" % "0.13.3" % " // Project settings val isJava17Plus: Boolean = { val v = sys.props.getOrElse("java.specification.version", "1.8") + // getOrElse(false): non-numeric versions (e.g. early-access "17-ea") fail safe + // by not compiling the jackson3 module rather than making an optimistic guess. if (v.startsWith("1.")) false else scala.util.Try(v.toInt >= 17).getOrElse(false) } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 4afd8859..81be9dd2 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -604,6 +604,9 @@ public JsonGenerator writeString(Reader reader, int len) throws JacksonException { try { long remaining = len < 0 ? Long.MAX_VALUE : len; + // Cap chunk size: len is a caller hint and can be arbitrarily large. + // Pre-allocating new StringBuilder(len) would reserve len*2 bytes upfront, + // which is an OOM risk for large inputs. The StringBuilder grows as needed. int chunkSize = (int) Math.min(remaining, 8192); StringBuilder sb = new StringBuilder(chunkSize); char[] tmpBuf = new char[chunkSize]; @@ -992,6 +995,9 @@ protected void _closeInput() throws IOException @Override protected void _releaseBuffers() { + // No null check on get(): generators are single-threaded by contract so this + // ThreadLocal is always set on the calling thread. A null here would indicate + // cross-thread misuse; letting it NPE surfaces that bug immediately. if (ownsThreadLocalBuffer) { try { messageBufferOutputHolder.get().reset(null); diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 8722021c..9f337d24 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -656,6 +656,8 @@ public boolean isClosed() @Override public void close() { + // Parsers are single-threaded by contract; close() is expected on the same + // thread that created the parser. Cross-thread close is not a supported use case. try { _closeInput(); } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java index 54b41f77..a1df18e4 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/TimestampExtensionModule.java @@ -52,6 +52,9 @@ protected InstantSerializer(Class t) public void serialize(Instant value, JsonGenerator gen, SerializationContext provider) { try { + // Per-call allocation is a known limitation carried from the v2 module. + // Manually encoding the timestamp bytes would avoid it but duplicates + // msgpack-core's timestamp logic. Tracked as a future optimization. ByteArrayOutputStream os = new ByteArrayOutputStream(); try (MessagePacker packer = MessagePack.newDefaultPacker(os)) { packer.packTimestamp(value); @@ -81,6 +84,7 @@ protected InstantDeserializer(Class vc) public Instant deserialize(JsonParser p, DeserializationContext ctxt) { try { + // Per-call allocation is a known limitation — see serialize() above. MessagePackExtensionType ext = p.readValueAs(MessagePackExtensionType.class); if (ext.getType() != EXT_TYPE) { ctxt.reportInputMismatch(Instant.class, From ef84b4095820ee336c730ca6a168b18cdc7e3b5f Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 19:17:12 +0900 Subject: [PATCH 34/50] Add issue #19: nil map key emitted as VALUE_NULL instead of PROPERTY_NAME --- plans/preexisting-issues.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index fa06d882..fdd23428 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -394,3 +394,20 @@ deserialized `value.suit`. Enum serialization/deserialization could be broken wi detecting it. Fixed in msgpack-jackson3 to `assertEquals(normalPojo.suit, value.suit)`. Needs the same fix in msgpack-jackson. +## 19. `MessagePackParser`: nil map key emitted as `VALUE_NULL` instead of `PROPERTY_NAME` + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` + +When a MessagePack map contains a nil key, the NIL case in `_nextToken()` unconditionally +returns `VALUE_NULL` regardless of whether the parser is reading a key or a value. When +reading a key (`isObjectValueSet == true`), it should call +`streamReadContext.setCurrentName(...)` and return `PROPERTY_NAME` instead, matching the +behaviour of the INTEGER, FLOAT, BOOLEAN, STRING, and EXTENSION key branches. The +desynchronisation causes the subsequent value to be misinterpreted and the object's entry +count to be wrong. + +**Practical impact:** Low. Nil keys in MessagePack maps are unusual, and most real-world +data uses string or integer keys. Affects both modules identically. + From 001afc582e65eb545a8416393f9e93496a950c63 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 19:23:19 +0900 Subject: [PATCH 35/50] Add README for msgpack-jackson3 module Mirrors msgpack-jackson README with jackson3-specific changes: - Artifact ID jackson-dataformat-msgpack-jackson3 - Java 17+ / Jackson 3.x requirement note - Note on tools.jackson.* vs com.fasterxml.jackson.annotation namespaces - StreamWriteFeature/StreamReadFeature replacing JsonGenerator/JsonParser Feature - tools.jackson.databind.KeyDeserializer in extension type examples - Dropped msgpack-java 0.6 POJO array compatibility section (not applicable) --- msgpack-jackson3/README.md | 475 +++++++++++++++++++++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 msgpack-jackson3/README.md diff --git a/msgpack-jackson3/README.md b/msgpack-jackson3/README.md new file mode 100644 index 00000000..ae971687 --- /dev/null +++ b/msgpack-jackson3/README.md @@ -0,0 +1,475 @@ +# jackson-dataformat-msgpack-jackson3 + +This Jackson 3.x extension library is a component to easily read and write [MessagePack](http://msgpack.org/) encoded data through jackson-databind API. + +It extends standard Jackson streaming API (`JsonFactory`, `JsonParser`, `JsonGenerator`), and as such works seamlessly with all the higher level data abstractions (data binding, tree model, and pluggable extensions). + +**Requirements:** Java 17+ and Jackson 3.x. For the Jackson 2.x compatible version, see [`msgpack-jackson`](../msgpack-jackson/). + +**Note on imports:** Jackson 3 moved its core and databind packages from `com.fasterxml.jackson` to `tools.jackson`. User-facing annotations (`@JsonProperty`, `@JsonFormat`, etc.) remain in `com.fasterxml.jackson.annotation` for backward compatibility. + +## Install + +### Maven + +```xml + + org.msgpack + jackson-dataformat-msgpack-jackson3 + (version) + +``` + +### Sbt + +```scala +libraryDependencies += "org.msgpack" % "jackson-dataformat-msgpack-jackson3" % "(version)" +``` + +### Gradle + +```groovy +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.msgpack:jackson-dataformat-msgpack-jackson3:(version)' +} +``` + +## Basic usage + +### Serialization/Deserialization of POJO + +Only thing you need to do is to instantiate `MessagePackFactory` and pass it to the constructor of `tools.jackson.databind.ObjectMapper`. And then, you can use it for MessagePack format data in the same way as jackson-databind. + +```java +// Instantiate ObjectMapper for MessagePack +ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + +// Serialize a Java object to byte array +ExamplePojo pojo = new ExamplePojo("komamitsu"); +byte[] bytes = objectMapper.writeValueAsBytes(pojo); + +// Deserialize the byte array to a Java object +ExamplePojo deserialized = objectMapper.readValue(bytes, ExamplePojo.class); +System.out.println(deserialized.getName()); // => komamitsu +``` + +Or more easily: + +```java +ObjectMapper objectMapper = new MessagePackMapper(); +``` + +We strongly recommend to call `MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` if you serialize and/or deserialize BigInteger/BigDecimal values. See [Serialize and deserialize BigDecimal as str type internally in MessagePack format](#serialize-and-deserialize-bigdecimal-as-str-type-internally-in-messagepack-format) for details. + +```java +ObjectMapper objectMapper = new MessagePackMapper().handleBigIntegerAndBigDecimalAsString(); +``` + +### Serialization/Deserialization of List + +```java +// Instantiate ObjectMapper for MessagePack +ObjectMapper objectMapper = new MessagePackMapper(); + +// Serialize a List to byte array +List list = new ArrayList<>(); +list.add("Foo"); +list.add("Bar"); +list.add(42); +byte[] bytes = objectMapper.writeValueAsBytes(list); + +// Deserialize the byte array to a List +List deserialized = objectMapper.readValue(bytes, new TypeReference>() {}); +System.out.println(deserialized); // => [Foo, Bar, 42] +``` + +### Serialization/Deserialization of Map + +```java +// Instantiate ObjectMapper for MessagePack +ObjectMapper objectMapper = new MessagePackMapper(); + +// Serialize a Map to byte array +Map map = new HashMap<>(); +map.put("name", "komamitsu"); +map.put("age", 42); +byte[] bytes = objectMapper.writeValueAsBytes(map); + +// Deserialize the byte array to a Map +Map deserialized = objectMapper.readValue(bytes, new TypeReference>() {}); +System.out.println(deserialized); // => {name=komamitsu, age=42} +``` + +### Example of Serialization/Deserialization over multiple languages + +Java + +```java +// Serialize +Map obj = new HashMap(); +obj.put("foo", "hello"); +obj.put("bar", "world"); +byte[] bs = objectMapper.writeValueAsBytes(obj); +// bs => [-126, -93, 102, 111, 111, -91, 104, 101, 108, 108, 111, +// -93, 98, 97, 114, -91, 119, 111, 114, 108, 100] +``` + +Ruby + +```ruby +require 'msgpack' + +# Deserialize +xs = [-126, -93, 102, 111, 111, -91, 104, 101, 108, 108, 111, + -93, 98, 97, 114, -91, 119, 111, 114, 108, 100] +MessagePack.unpack(xs.pack("C*")) +# => {"foo"=>"hello", "bar"=>"world"} + +# Serialize +["zero", 1, 2.0, nil].to_msgpack.unpack('C*') +# => [148, 164, 122, 101, 114, 111, 1, 203, 64, 0, 0, 0, 0, 0, 0, 0, 192] +``` + +Java + +```java +// Deserialize +bs = new byte[] {(byte) 148, (byte) 164, 122, 101, 114, 111, 1, + (byte) 203, 64, 0, 0, 0, 0, 0, 0, 0, (byte) 192}; +TypeReference> typeReference = new TypeReference>(){}; +List xs = objectMapper.readValue(bs, typeReference); +// xs => [zero, 1, 2.0, null] +``` + +## Advanced usage + +### Serialize multiple values without closing an output stream + +`tools.jackson.databind.ObjectMapper` closes an output stream by default after it writes a value. If you want to serialize multiple values in a row without closing an output stream, disable `StreamWriteFeature.AUTO_CLOSE_TARGET`. + +```java +OutputStream out = new FileOutputStream(tempFile); +ObjectMapper objectMapper = new MessagePackMapper(); +objectMapper.disable(StreamWriteFeature.AUTO_CLOSE_TARGET); + +objectMapper.writeValue(out, 1); +objectMapper.writeValue(out, "two"); +objectMapper.writeValue(out, 3.14); +out.close(); + +MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new FileInputStream(tempFile)); +System.out.println(unpacker.unpackInt()); // => 1 +System.out.println(unpacker.unpackString()); // => two +System.out.println(unpacker.unpackFloat()); // => 3.14 +``` + +### Deserialize multiple values without closing an input stream + +`tools.jackson.databind.ObjectMapper` closes an input stream by default after it reads a value. If you want to deserialize multiple values in a row without closing an input stream, disable `StreamReadFeature.AUTO_CLOSE_SOURCE`. + +```java +MessagePacker packer = MessagePack.newDefaultPacker(new FileOutputStream(tempFile)); +packer.packInt(42); +packer.packString("Hello"); +packer.close(); + +FileInputStream in = new FileInputStream(tempFile); +ObjectMapper objectMapper = new MessagePackMapper(); +objectMapper.disable(StreamReadFeature.AUTO_CLOSE_SOURCE); +System.out.println(objectMapper.readValue(in, Integer.class)); +System.out.println(objectMapper.readValue(in, String.class)); +in.close(); +``` + +### Serialize not using str8 type + +Old msgpack-java (e.g 0.6.7) doesn't support MessagePack str8 type. When your application needs to communicate with such an old MessagePack library, you can disable the data type like this: + +```java +MessagePack.PackerConfig config = new MessagePack.PackerConfig().withStr8FormatSupport(false); +ObjectMapper objectMapper = new MessagePackMapper(new MessagePackFactory(config)); +// This string is serialized as bin8 type +byte[] resultWithoutStr8Format = objectMapper.writeValueAsBytes(str8LengthString); +``` + +### Serialize using non-String as a key of Map + +When you want to use non-String value as a key of Map, use `MessagePackKeySerializer` for key serialization. + +```java +@JsonSerialize(keyUsing = MessagePackKeySerializer.class) +private Map intMap = new HashMap<>(); + + : + +intMap.put(42, "Hello"); + +ObjectMapper objectMapper = new MessagePackMapper(); +byte[] bytes = objectMapper.writeValueAsBytes(intMap); + +Map deserialized = objectMapper.readValue(bytes, new TypeReference>() {}); +System.out.println(deserialized); // => {42=Hello} +``` + +### Serialize and deserialize BigDecimal as str type internally in MessagePack format + +`jackson-dataformat-msgpack-jackson3` represents BigDecimal values as float type in MessagePack format by default for backward compatibility. But the default behavior could fail when handling too large value for `double` type. So we strongly recommend to call `MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` to internally handle BigDecimal values as String. + +```java +ObjectMapper objectMapper = new MessagePackMapper().handleBigIntegerAndBigDecimalAsString(); + +Pojo obj = new Pojo(); +// This value is too large to be serialized as double +obj.value = new BigDecimal("1234567890.98765432100"); + +byte[] converted = objectMapper.writeValueAsBytes(obj); + +System.out.println(objectMapper.readValue(converted, Pojo.class)); // => Pojo{value=1234567890.98765432100} +``` + +`MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` is equivalent to the following configuration. + +```java +ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); +objectMapper.configOverride(BigInteger.class).setFormat(JsonFormat.Value.forShape(JsonFormat.Shape.STRING)); +objectMapper.configOverride(BigDecimal.class).setFormat(JsonFormat.Value.forShape(JsonFormat.Shape.STRING)); +``` + +### Serialize and deserialize Instant instances as MessagePack extension type + +`timestamp` extension type is defined in MessagePack as type:-1. Registering `TimestampExtensionModule.INSTANCE` module enables automatic serialization and deserialization of `java.time.Instant` to/from the MessagePack extension type. + +```java +ObjectMapper objectMapper = new MessagePackMapper() + .registerModule(TimestampExtensionModule.INSTANCE); +Pojo pojo = new Pojo(); +// The type of `timestamp` variable is Instant +pojo.timestamp = Instant.now(); +byte[] bytes = objectMapper.writeValueAsBytes(pojo); + +// The Instant instance is serialized as MessagePack extension type (type: -1) + +Pojo deserialized = objectMapper.readValue(bytes, Pojo.class); +System.out.println(deserialized); // "2022-09-14T08:47:24.922Z" +``` + +### Deserialize extension types with ExtensionTypeCustomDeserializers + +`ExtensionTypeCustomDeserializers` helps you to deserialize your own custom extension types easily. + +#### Deserialize extension type value directly + +```java +// In this application, extension type 59 is used for byte[] +byte[] bytes; +{ + // This ObjectMapper is just for temporary serialization + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(outputStream); + + packer.packExtensionTypeHeader((byte) 59, hexspeak.length); + packer.addPayload(hexspeak); + packer.close(); + + bytes = outputStream.toByteArray(); +} + +// Register the type and a deserializer to ExtensionTypeCustomDeserializers +ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); +extTypeCustomDesers.addCustomDeser((byte) 59, data -> { + if (Arrays.equals(data, + new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE})) { + return "Java"; + } + return "Not Java"; +}); + +ObjectMapper objectMapper = new ObjectMapper( + new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)); + +System.out.println(objectMapper.readValue(bytes, Object.class)); + // => Java +``` + +#### Use extension type as Map key + +```java +static class TripleBytesPojo +{ + public byte first; + public byte second; + public byte third; + + public TripleBytesPojo(byte first, byte second, byte third) + { + this.first = first; + this.second = second; + this.third = third; + } + + @Override + public boolean equals(Object o) + { + : + } + + @Override + public int hashCode() + { + : + } + + @Override + public String toString() + { + // This key format is used when serialized as map key + return String.format("%d-%d-%d", first, second, third); + } + + static class KeyDeserializer + extends tools.jackson.databind.KeyDeserializer + { + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + { + String[] values = key.split("-"); + return new TripleBytesPojo(Byte.parseByte(values[0]), Byte.parseByte(values[1]), Byte.parseByte(values[2])); + } + } + + static TripleBytesPojo deserialize(byte[] bytes) + { + return new TripleBytesPojo(bytes[0], bytes[1], bytes[2]); + } +} + +: + +byte extTypeCode = 42; + +ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); +extTypeCustomDesers.addCustomDeser(extTypeCode, new ExtensionTypeCustomDeserializers.Deser() +{ + @Override + public Object deserialize(byte[] value) + throws IOException + { + return TripleBytesPojo.deserialize(value); + } +}); + +SimpleModule module = new SimpleModule(); +module.addKeyDeserializer(TripleBytesPojo.class, new TripleBytesPojo.KeyDeserializer()); +ObjectMapper objectMapper = new ObjectMapper( + new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)) + .registerModule(module); + +Map deserializedMap = + objectMapper.readValue(serializedData, + new TypeReference>() {}); +``` + +#### Use extension type as Map value + +```java +static class TripleBytesPojo +{ + public byte first; + public byte second; + public byte third; + + public TripleBytesPojo(byte first, byte second, byte third) + { + this.first = first; + this.second = second; + this.third = third; + } + + static class Deserializer + extends StdDeserializer + { + protected Deserializer() + { + super(TripleBytesPojo.class); + } + + @Override + public TripleBytesPojo deserialize(JsonParser p, DeserializationContext ctxt) + { + return TripleBytesPojo.deserialize(p.getBinaryValue()); + } + } + + static TripleBytesPojo deserialize(byte[] bytes) + { + return new TripleBytesPojo(bytes[0], bytes[1], bytes[2]); + } +} + +: + +byte extTypeCode = 42; + +ExtensionTypeCustomDeserializers extTypeCustomDesers = new ExtensionTypeCustomDeserializers(); +extTypeCustomDesers.addCustomDeser(extTypeCode, new ExtensionTypeCustomDeserializers.Deser() +{ + @Override + public Object deserialize(byte[] value) + throws IOException + { + return TripleBytesPojo.deserialize(value); + } +}); + +SimpleModule module = new SimpleModule(); +module.addDeserializer(TripleBytesPojo.class, new TripleBytesPojo.Deserializer()); +ObjectMapper objectMapper = new ObjectMapper( + new MessagePackFactory().setExtTypeCustomDesers(extTypeCustomDesers)) + .registerModule(module); + +Map deserializedMap = + objectMapper.readValue(serializedData, + new TypeReference>() {}); +``` + +### Serialize a nested object that also serializes + +When you serialize an object that has a nested object also serializing with ObjectMapper and MessagePackFactory like the following code, it throws NullPointerException since the nested MessagePackFactory modifies a shared state stored in ThreadLocal. + +```java +@Test +public void testNestedSerialization() throws Exception +{ + ObjectMapper objectMapper = new MessagePackMapper(); + objectMapper.writeValueAsBytes(new OuterClass()); +} + +public class OuterClass +{ + public String getInner() throws JacksonException + { + ObjectMapper m = new MessagePackMapper(); + m.writeValueAsBytes(new InnerClass()); + return "EFG"; + } +} + +public class InnerClass +{ + public String getName() + { + return "ABC"; + } +} +``` + +There are a few options to fix this issue, but they introduce performance degradations while this usage is a corner case. A workaround that doesn't affect performance is to call `MessagePackFactory#setReuseResourceInGenerator(false)`. It might be inconvenient to call the API for users, but it's a reasonable tradeoff with performance for now. + +```java +ObjectMapper objectMapper = new ObjectMapper( + new MessagePackFactory().setReuseResourceInGenerator(false)); +``` From ccee0c336b430b3ff3ad9928a2b9b8ce947a005b Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 20:06:37 +0900 Subject: [PATCH 36/50] Fix NPE in numeric accessors on structural tokens Add if (type == null) guards to getNumberValue(), getIntValue(), getLongValue(), getFloatValue(), getDoubleValue(), getBigIntegerValue(), getDecimalValue(), and getNumberType(). When type is null the parser is on a structural token (START_OBJECT, END_ARRAY, etc.); a null enum in a switch throws NPE instead of a useful error. The fix mirrors CBORParser's _checkNumericValue() behaviour via ParserBase: throw a descriptive StreamReadException. getNumberType() returns null for non-numeric tokens, consistent with CBORParser. Add testNumericAccessorsOnStructuralTokenThrow to cover this path. --- .../jackson/dataformat/MessagePackParser.java | 24 +++++++++ .../dataformat/MessagePackParserTest.java | 51 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 9f337d24..52b7771e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -380,6 +380,9 @@ public byte[] getBinaryValue(Base64Variant b64variant) @Override public Number getNumberValue() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return intValue; @@ -403,6 +406,9 @@ public Number getNumberValue() @Override public int getIntValue() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return intValue; @@ -440,6 +446,9 @@ public int getIntValue() @Override public long getLongValue() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return intValue; @@ -474,6 +483,9 @@ public long getLongValue() @Override public BigInteger getBigIntegerValue() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return BigInteger.valueOf(intValue); @@ -502,6 +514,9 @@ public float getFloatValue() { // No bounds/range check: a finite double or large BigInteger may overflow to // Float.POSITIVE_INFINITY. This is intentional — same as ParserBase and CBORParser. + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return (float) intValue; @@ -527,6 +542,9 @@ public double getDoubleValue() { // No bounds/range check: large BigInteger may overflow to Double.POSITIVE_INFINITY, // and large long values may lose precision. Intentional — same as ParserBase. + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return intValue; @@ -550,6 +568,9 @@ public double getDoubleValue() @Override public BigDecimal getDecimalValue() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not numeric, cannot use numeric value accessors"); + } switch (type) { case INT: return BigDecimal.valueOf(intValue); @@ -614,6 +635,9 @@ public Object getEmbeddedObject() @Override public NumberType getNumberType() { + if (type == null) { + return null; + } switch (type) { case INT: return NumberType.INT; diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java index 5aab2565..babbc866 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java @@ -1323,6 +1323,57 @@ public void testGetLongValueFromOutOfRangeDoubleThrows() throws IOException } } + @Test + public void testNumericAccessorsOnStructuralTokenThrow() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (MessagePacker packer = MessagePack.newDefaultPacker(out)) { + packer.packMapHeader(0); + } + byte[] bytes = out.toByteArray(); + MessagePackFactory factory = new MessagePackFactory(); + + try (JsonParser p = factory.createParser(tools.jackson.core.ObjectReadContext.empty(), bytes)) { + assertEquals(JsonToken.START_OBJECT, p.nextToken()); + try { + p.getIntValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getLongValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getDoubleValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getFloatValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getBigIntegerValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getDecimalValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + try { + p.getNumberValue(); + fail("expected exception"); + } + catch (JacksonException ignored) { } + assertNull(p.getNumberType()); + } + } + @Test public void testGetIntValueFromFractionalDoubleTruncates() throws IOException { From 9e295e1e4503e2cc74ad255cf455cdeb666fb2d5 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 21:38:10 +0900 Subject: [PATCH 37/50] Use WeakReference for ThreadLocal cached objects in generator and parser Generator: wrap OutputStreamBufferOutput in WeakReference so GC can reclaim the buffer (including its internal byte[]) under memory pressure between uses on a thread-pool thread, instead of retaining it indefinitely. Parser: wrap the cached source object in WeakReference so GC can reclaim large byte[] payloads or InputStream references once the caller drops them, without disrupting same-stream reuse detection (a live InputStream is still strongly referenced by the caller, keeping the WeakReference valid). --- .../dataformat/MessagePackGenerator.java | 22 ++++++++++++------- .../jackson/dataformat/MessagePackParser.java | 16 ++++++++------ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 81be9dd2..04ce03b0 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Reader; +import java.lang.ref.WeakReference; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -49,7 +50,7 @@ public class MessagePackGenerator private static final int IN_OBJECT = 1; private static final int IN_ARRAY = 2; private final MessagePacker messagePacker; - private static final ThreadLocal messageBufferOutputHolder = new ThreadLocal<>(); + private static final ThreadLocal> messageBufferOutputHolder = new ThreadLocal<>(); private final OutputStream output; private final MessagePack.PackerConfig packerConfig; private final boolean supportIntegerKeys; @@ -242,10 +243,11 @@ private MessageBufferOutput getMessageBufferOutputForOutputStream( { OutputStreamBufferOutput messageBufferOutput; if (reuseResourceInGenerator) { - messageBufferOutput = messageBufferOutputHolder.get(); + WeakReference ref = messageBufferOutputHolder.get(); + messageBufferOutput = ref != null ? ref.get() : null; if (messageBufferOutput == null) { messageBufferOutput = new OutputStreamBufferOutput(out); - messageBufferOutputHolder.set(messageBufferOutput); + messageBufferOutputHolder.set(new WeakReference<>(messageBufferOutput)); } else { messageBufferOutput.reset(out); @@ -999,11 +1001,15 @@ protected void _releaseBuffers() // ThreadLocal is always set on the calling thread. A null here would indicate // cross-thread misuse; letting it NPE surfaces that bug immediately. if (ownsThreadLocalBuffer) { - try { - messageBufferOutputHolder.get().reset(null); - } - catch (IOException e) { - throw _wrapIOFailure(e); + WeakReference ref = messageBufferOutputHolder.get(); + OutputStreamBufferOutput buf = ref != null ? ref.get() : null; + if (buf != null) { + try { + buf.reset(null); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } } } } diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 52b7771e..9161dd37 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -35,13 +35,14 @@ import org.msgpack.value.ValueType; import java.io.IOException; +import java.lang.ref.WeakReference; import java.math.BigDecimal; import java.math.BigInteger; public class MessagePackParser extends ParserMinimalBase { - private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); + private static final ThreadLocal, MessageUnpacker>> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); @@ -90,7 +91,7 @@ private enum Type return; } - Tuple messageUnpackerTuple = messageUnpackerHolder.get(); + Tuple, MessageUnpacker> messageUnpackerTuple = messageUnpackerHolder.get(); if (messageUnpackerTuple == null) { messageUnpacker = MessagePack.newDefaultUnpacker(input); } @@ -99,12 +100,13 @@ private enum Type // MessagePackParser needs to use the MessageUnpacker that has the same InputStream // since it has buffer which has loaded the InputStream data ahead. // However, it needs to call MessageUnpacker#reset when the source is different from the previous one. - if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || messageUnpackerTuple.first() != src || src instanceof byte[]) { + Object cachedSrc = messageUnpackerTuple.first().get(); + if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || cachedSrc != src || src instanceof byte[]) { messageUnpackerTuple.second().reset(input); } messageUnpacker = messageUnpackerTuple.second(); } - messageUnpackerHolder.set(new Tuple<>(src, messageUnpacker)); + messageUnpackerHolder.set(new Tuple<>(new WeakReference<>(src), messageUnpacker)); ownsThreadLocalUnpacker = true; } @@ -691,15 +693,15 @@ public void close() finally { isClosed = true; if (ownsThreadLocalUnpacker) { - Tuple tuple = messageUnpackerHolder.get(); - if (tuple != null && tuple.first() instanceof byte[]) { + Tuple, MessageUnpacker> tuple = messageUnpackerHolder.get(); + if (tuple != null && tuple.first().get() instanceof byte[]) { try { tuple.second().close(); } catch (IOException e) { throw _wrapIOFailure(e); } - messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); + messageUnpackerHolder.set(new Tuple<>(new WeakReference<>(null), tuple.second())); } } } From e1dadb14110f90b4c69ced719b35814bdc5b3bc3 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 22:24:44 +0900 Subject: [PATCH 38/50] Fix README: use MessagePackMapper.Builder API for Jackson 3 handleBigIntegerAndBigDecimalAsString() is on Builder in Jackson 3, not an instance method on MessagePackMapper. Update both occurrences to use MessagePackMapper.builder().handleBigIntegerAndBigDecimalAsString().build(). --- msgpack-jackson3/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack-jackson3/README.md b/msgpack-jackson3/README.md index ae971687..d50c81ea 100644 --- a/msgpack-jackson3/README.md +++ b/msgpack-jackson3/README.md @@ -63,10 +63,10 @@ Or more easily: ObjectMapper objectMapper = new MessagePackMapper(); ``` -We strongly recommend to call `MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` if you serialize and/or deserialize BigInteger/BigDecimal values. See [Serialize and deserialize BigDecimal as str type internally in MessagePack format](#serialize-and-deserialize-bigdecimal-as-str-type-internally-in-messagepack-format) for details. +We strongly recommend calling `MessagePackMapper.Builder#handleBigIntegerAndBigDecimalAsString()` if you serialize and/or deserialize BigInteger/BigDecimal values. See [Serialize and deserialize BigDecimal as str type internally in MessagePack format](#serialize-and-deserialize-bigdecimal-as-str-type-internally-in-messagepack-format) for details. ```java -ObjectMapper objectMapper = new MessagePackMapper().handleBigIntegerAndBigDecimalAsString(); +ObjectMapper objectMapper = MessagePackMapper.builder().handleBigIntegerAndBigDecimalAsString().build(); ``` ### Serialization/Deserialization of List @@ -217,10 +217,10 @@ System.out.println(deserialized); // => {42=Hello} ### Serialize and deserialize BigDecimal as str type internally in MessagePack format -`jackson-dataformat-msgpack-jackson3` represents BigDecimal values as float type in MessagePack format by default for backward compatibility. But the default behavior could fail when handling too large value for `double` type. So we strongly recommend to call `MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` to internally handle BigDecimal values as String. +`jackson-dataformat-msgpack-jackson3` represents BigDecimal values as float type in MessagePack format by default for backward compatibility. But the default behavior could fail when handling too large value for `double` type. So we strongly recommend calling `MessagePackMapper.Builder#handleBigIntegerAndBigDecimalAsString()` to internally handle BigDecimal values as String. ```java -ObjectMapper objectMapper = new MessagePackMapper().handleBigIntegerAndBigDecimalAsString(); +ObjectMapper objectMapper = MessagePackMapper.builder().handleBigIntegerAndBigDecimalAsString().build(); Pojo obj = new Pojo(); // This value is too large to be serialized as double From 393ec702a06912810083d9222df07959eaf1ef54 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 22:27:43 +0900 Subject: [PATCH 39/50] Update preexisting-issues.md: issue #2 WeakReference fix, add issue #20 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #2: update description to reflect WeakReference approach replacing the explicit null-out for byte[] sources. Issue #20 (new): numeric accessors NPE on structural tokens — fixed in msgpack-jackson3 via if (type == null) guards; needs same fix in v2. --- plans/preexisting-issues.md | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index fdd23428..f4ffbe97 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -174,10 +174,11 @@ both bugs compound). retains the entire last parsed payload for each thread in a pool indefinitely, which can cause unbounded memory retention after large messages. -Fixed in msgpack-jackson3: on `close()`, if the cached source is a `byte[]`, it is -replaced with `null` in the ThreadLocal (keeping the unpacker alive for reuse but -releasing the byte-array reference). InputStream sources are left unchanged because -they are needed to detect same-stream reuse. Needs the same fix in msgpack-jackson. +Fixed in msgpack-jackson3: sources are now wrapped in `WeakReference` so GC can reclaim +them once the caller drops its reference. On `close()`, byte-array sources are replaced +with `WeakReference(null)` for prompt release; InputStream sources are left alive in the +WeakReference as long as the caller holds them, preserving same-stream reuse detection. +Needs the same fix in msgpack-jackson. ## 3. `MessagePackGenerator`: `close()` does not set `isClosed()` to true @@ -411,3 +412,24 @@ count to be wrong. **Practical impact:** Low. Nil keys in MessagePack maps are unusual, and most real-world data uses string or integer keys. Affects both modules identically. +## 20. `MessagePackParser`: numeric accessors NPE on structural tokens + +**Files:** +- `msgpack-jackson/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` +- `msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java` — FIXED + +When `type` is Java `null` (i.e. the current token is a structural token such as +`START_OBJECT` or `END_ARRAY`), all numeric accessor methods (`getIntValue()`, +`getLongValue()`, etc.) perform `switch (type)` which throws `NullPointerException` +instead of a descriptive `StreamReadException`. `ParserBase`-based parsers (e.g. +CBORParser) avoid this via `_checkNumericValue()` / `_numTypesValid`. `getNumberType()` +should return `null` for non-numeric tokens (matching CBORParser); all other numeric +accessors should throw a descriptive error. + +Fixed in msgpack-jackson3 (commit ccee0c33): `if (type == null)` guards added to all +eight numeric accessors; `testNumericAccessorsOnStructuralTokenThrow` covers the path. +Needs the same fix in msgpack-jackson. + +**Practical impact:** Low. Jackson databind always checks `currentToken()` before calling +numeric accessors, so this NPE is only reachable via direct streaming API misuse. + From faa2173973be418478e9fddcc0f32a776ed22dd8 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 30 May 2026 23:00:42 +0900 Subject: [PATCH 40/50] Fix type==null NPE in getBinaryValue/getEmbeddedObject; fix writeNumber Infinity overflow Add if (type == null) guards to getBinaryValue() and getEmbeddedObject(), matching the pattern already applied to numeric accessors in ccee0c33. Fix writeNumber(String) for values like "1e309" that overflow double to Infinity: add !Double.isInfinite(d) guard before the BigDecimal/double exactness comparison so out-of-range values are kept as BigDecimal rather than silently serialized as Infinity. --- .../msgpack/jackson/dataformat/MessagePackGenerator.java | 5 +++-- .../org/msgpack/jackson/dataformat/MessagePackParser.java | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 04ce03b0..cd8c4fbd 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -824,12 +824,13 @@ public JsonGenerator writeNumber(String encodedValue) throws JacksonException double d = bd.doubleValue(); // Check if the double can perfectly represent the exact decimal value. - if (bd.compareTo(new BigDecimal(String.valueOf(d))) == 0) { + // isInfinite guard: values like "1e309" overflow double to Infinity; keep as BigDecimal. + if (!Double.isInfinite(d) && bd.compareTo(new BigDecimal(String.valueOf(d))) == 0) { // It's a safe ordinary floating-point number. addValueNode(d); } else { - // It has more precision than a double can handle. + // It has more precision than a double can handle, or overflows double range. addValueNode(bd); } return this; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 9161dd37..2efa747e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -360,6 +360,9 @@ public int getStringOffset() @Override public byte[] getBinaryValue(Base64Variant b64variant) { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not of binary type"); + } switch (type) { case BYTES: return bytesValue; @@ -611,6 +614,9 @@ private Object deserializedExtensionTypeValue() @Override public Object getEmbeddedObject() { + if (type == null) { + return _reportError("Current token (" + _currToken + ") not of embeddable type"); + } switch (type) { case BYTES: return bytesValue; From 1f65e27f617ca9913c6070dcd4a58fea6c71aebc Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 00:06:09 +0900 Subject: [PATCH 41/50] Add comments explaining internal buffer management in parser and generator Parser constructor reset() path: note that MessageUnpacker.reset() replaces the internal MessageBufferInput and clears the read buffer to EMPTY_BUFFER, making the old ArrayBufferInput (and its byte[]) GC-eligible. Parser close() path: note that MessageUnpacker.close() calls ArrayBufferInput.close() which sets buffer = null, releasing the byte[] payload while keeping the unpacker alive for reuse. Generator _releaseBuffers(): note that reset(null) clears only the OutputStream reference; the internal MessageBuffer is intentionally retained for reuse and is reclaimed when the WeakReference is collected after GC. --- .../msgpack/jackson/dataformat/MessagePackGenerator.java | 4 ++++ .../org/msgpack/jackson/dataformat/MessagePackParser.java | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index cd8c4fbd..7e852997 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -1006,6 +1006,10 @@ protected void _releaseBuffers() OutputStreamBufferOutput buf = ref != null ? ref.get() : null; if (buf != null) { try { + // reset(null) clears the OutputStream reference inside OutputStreamBufferOutput + // but intentionally retains its internal MessageBuffer for reuse on the next + // generator created on this thread. The MessageBuffer is reclaimed when the + // WeakReference is collected after this generator instance is GC'd. buf.reset(null); } catch (IOException e) { diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 2efa747e..649a412d 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -102,6 +102,9 @@ private enum Type // However, it needs to call MessageUnpacker#reset when the source is different from the previous one. Object cachedSrc = messageUnpackerTuple.first().get(); if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || cachedSrc != src || src instanceof byte[]) { + // reset() replaces the internal MessageBufferInput and clears the unpacker's + // internal read buffer to EMPTY_BUFFER. The old ArrayBufferInput becomes + // unreachable here (we discard the return value), so its byte[] is GC-eligible. messageUnpackerTuple.second().reset(input); } messageUnpacker = messageUnpackerTuple.second(); @@ -702,6 +705,9 @@ public void close() Tuple, MessageUnpacker> tuple = messageUnpackerHolder.get(); if (tuple != null && tuple.first().get() instanceof byte[]) { try { + // close() calls ArrayBufferInput.close() which sets buffer = null, + // releasing the byte[] payload reference held by the unpacker's input. + // The unpacker itself is kept alive for reuse on the next parse. tuple.second().close(); } catch (IOException e) { From 1c942d04a8861c8c5dfc9984a98ec02f0fb7dd9f Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 00:22:13 +0900 Subject: [PATCH 42/50] Simplify parser ThreadLocal: wrap Tuple in WeakReference instead of source Replace ThreadLocal, MessageUnpacker>> with ThreadLocal>>. Under GC pressure the entire tuple (source + unpacker) is reclaimed together; under normal conditions reuse is unaffected. Removes the complexity of individually weak-referencing the source inside the tuple. --- .../jackson/dataformat/MessagePackParser.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 649a412d..3f61f417 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -42,7 +42,7 @@ public class MessagePackParser extends ParserMinimalBase { - private static final ThreadLocal, MessageUnpacker>> messageUnpackerHolder = new ThreadLocal<>(); + private static final ThreadLocal>> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); @@ -91,7 +91,8 @@ private enum Type return; } - Tuple, MessageUnpacker> messageUnpackerTuple = messageUnpackerHolder.get(); + WeakReference> ref = messageUnpackerHolder.get(); + Tuple messageUnpackerTuple = ref != null ? ref.get() : null; if (messageUnpackerTuple == null) { messageUnpacker = MessagePack.newDefaultUnpacker(input); } @@ -100,7 +101,7 @@ private enum Type // MessagePackParser needs to use the MessageUnpacker that has the same InputStream // since it has buffer which has loaded the InputStream data ahead. // However, it needs to call MessageUnpacker#reset when the source is different from the previous one. - Object cachedSrc = messageUnpackerTuple.first().get(); + Object cachedSrc = messageUnpackerTuple.first(); if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(streamReadFeatures) || cachedSrc != src || src instanceof byte[]) { // reset() replaces the internal MessageBufferInput and clears the unpacker's // internal read buffer to EMPTY_BUFFER. The old ArrayBufferInput becomes @@ -109,7 +110,7 @@ private enum Type } messageUnpacker = messageUnpackerTuple.second(); } - messageUnpackerHolder.set(new Tuple<>(new WeakReference<>(src), messageUnpacker)); + messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(src, messageUnpacker))); ownsThreadLocalUnpacker = true; } @@ -702,8 +703,9 @@ public void close() finally { isClosed = true; if (ownsThreadLocalUnpacker) { - Tuple, MessageUnpacker> tuple = messageUnpackerHolder.get(); - if (tuple != null && tuple.first().get() instanceof byte[]) { + WeakReference> ref = messageUnpackerHolder.get(); + Tuple tuple = ref != null ? ref.get() : null; + if (tuple != null && tuple.first() instanceof byte[]) { try { // close() calls ArrayBufferInput.close() which sets buffer = null, // releasing the byte[] payload reference held by the unpacker's input. @@ -713,7 +715,7 @@ public void close() catch (IOException e) { throw _wrapIOFailure(e); } - messageUnpackerHolder.set(new Tuple<>(new WeakReference<>(null), tuple.second())); + messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); } } } From 27b87569c92753f3f4781fc0104e99c5f44e6bfe Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 00:37:05 +0900 Subject: [PATCH 43/50] Extract ThreadLocal cleanup out of finally block in MessagePackParser.close() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isClosed = true belongs in finally (must run even if _closeInput throws); the ThreadLocal cleanup does not — move it after the try/finally block. --- .../jackson/dataformat/MessagePackParser.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 3f61f417..b52b35a0 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -702,21 +702,22 @@ public void close() } finally { isClosed = true; - if (ownsThreadLocalUnpacker) { - WeakReference> ref = messageUnpackerHolder.get(); - Tuple tuple = ref != null ? ref.get() : null; - if (tuple != null && tuple.first() instanceof byte[]) { - try { - // close() calls ArrayBufferInput.close() which sets buffer = null, - // releasing the byte[] payload reference held by the unpacker's input. - // The unpacker itself is kept alive for reuse on the next parse. - tuple.second().close(); - } - catch (IOException e) { - throw _wrapIOFailure(e); - } - messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); + } + + if (ownsThreadLocalUnpacker) { + WeakReference> ref = messageUnpackerHolder.get(); + Tuple tuple = ref != null ? ref.get() : null; + if (tuple != null && tuple.first() instanceof byte[]) { + try { + // close() calls ArrayBufferInput.close() which sets buffer = null, + // releasing the byte[] payload reference held by the unpacker's input. + // The unpacker itself is kept alive for reuse on the next parse. + tuple.second().close(); + } + catch (IOException e) { + throw _wrapIOFailure(e); } + messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); } } } From f2b16237714b435911a0ee9999006d89e126b326 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 00:47:18 +0900 Subject: [PATCH 44/50] Move ThreadLocal byte[] cleanup into _closeInput() All input-related cleanup now lives in _closeInput(); close() only sets isClosed = true in its finally block. --- .../jackson/dataformat/MessagePackParser.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index b52b35a0..651ecb87 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -676,6 +676,17 @@ protected void _closeInput() throws IOException if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { messageUnpacker.close(); } + if (ownsThreadLocalUnpacker) { + WeakReference> ref = messageUnpackerHolder.get(); + Tuple tuple = ref != null ? ref.get() : null; + if (tuple != null && tuple.first() instanceof byte[]) { + // close() calls ArrayBufferInput.close() which sets buffer = null, + // releasing the byte[] payload reference held by the unpacker's input. + // The unpacker itself is kept alive for reuse on the next parse. + tuple.second().close(); + messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); + } + } } @Override @@ -703,23 +714,6 @@ public void close() finally { isClosed = true; } - - if (ownsThreadLocalUnpacker) { - WeakReference> ref = messageUnpackerHolder.get(); - Tuple tuple = ref != null ? ref.get() : null; - if (tuple != null && tuple.first() instanceof byte[]) { - try { - // close() calls ArrayBufferInput.close() which sets buffer = null, - // releasing the byte[] payload reference held by the unpacker's input. - // The unpacker itself is kept alive for reuse on the next parse. - tuple.second().close(); - } - catch (IOException e) { - throw _wrapIOFailure(e); - } - messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); - } - } } @Override From de60fea4830aa3dd65ea6914c75261b8f0a4135b Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 01:49:08 +0900 Subject: [PATCH 45/50] Remove explicit ThreadLocal cleanup; rely on WeakReference for reclamation Generator: remove ownsThreadLocalBuffer field and buf.reset(null) from _releaseBuffers(). The WeakReference on the ThreadLocal allows GC to reclaim OutputStreamBufferOutput when the generator is collected; AUTO_CLOSE_TARGET handles OutputStream closing via _closeInput(). Parser: remove ownsThreadLocalUnpacker field and the byte[] null-out block from _closeInput(). The WeakReference allows GC to reclaim the unpacker and source together under memory pressure. --- .../dataformat/MessagePackGenerator.java | 22 ------------------- .../jackson/dataformat/MessagePackParser.java | 14 ------------ 2 files changed, 36 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 7e852997..74fce079 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -60,7 +60,6 @@ public class MessagePackGenerator private final List nodes; private boolean isElementsClosed = false; private MessagePackWriteContext writeContext; - private final boolean ownsThreadLocalBuffer; private static final class RawUtf8String { @@ -211,7 +210,6 @@ private MessagePackGenerator( this.writeContext = MessagePackWriteContext.createRootContext( StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) ? DupDetector.rootDetector(this) : null); - this.ownsThreadLocalBuffer = false; } public MessagePackGenerator( @@ -233,7 +231,6 @@ public MessagePackGenerator( this.writeContext = MessagePackWriteContext.createRootContext( StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) ? DupDetector.rootDetector(this) : null); - this.ownsThreadLocalBuffer = reuseResourceInGenerator; } private MessageBufferOutput getMessageBufferOutputForOutputStream( @@ -998,25 +995,6 @@ protected void _closeInput() throws IOException @Override protected void _releaseBuffers() { - // No null check on get(): generators are single-threaded by contract so this - // ThreadLocal is always set on the calling thread. A null here would indicate - // cross-thread misuse; letting it NPE surfaces that bug immediately. - if (ownsThreadLocalBuffer) { - WeakReference ref = messageBufferOutputHolder.get(); - OutputStreamBufferOutput buf = ref != null ? ref.get() : null; - if (buf != null) { - try { - // reset(null) clears the OutputStream reference inside OutputStreamBufferOutput - // but intentionally retains its internal MessageBuffer for reuse on the next - // generator created on this thread. The MessageBuffer is reclaimed when the - // WeakReference is collected after this generator instance is GC'd. - buf.reset(null); - } - catch (IOException e) { - throw _wrapIOFailure(e); - } - } - } } @Override diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 651ecb87..e26b950c 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -55,7 +55,6 @@ public class MessagePackParser private long currentPosition; private final IOContext ioContext; private ExtensionTypeCustomDeserializers extTypeCustomDesers; - private final boolean ownsThreadLocalUnpacker; private enum Type { @@ -87,7 +86,6 @@ private enum Type streamReadContext = MessagePackReadContext.createRootContext(dups); if (!reuseResourceInParser) { messageUnpacker = MessagePack.newDefaultUnpacker(input); - ownsThreadLocalUnpacker = false; return; } @@ -111,7 +109,6 @@ private enum Type messageUnpacker = messageUnpackerTuple.second(); } messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(src, messageUnpacker))); - ownsThreadLocalUnpacker = true; } public void setExtensionTypeCustomDeserializers(ExtensionTypeCustomDeserializers extTypeCustomDesers) @@ -676,17 +673,6 @@ protected void _closeInput() throws IOException if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { messageUnpacker.close(); } - if (ownsThreadLocalUnpacker) { - WeakReference> ref = messageUnpackerHolder.get(); - Tuple tuple = ref != null ? ref.get() : null; - if (tuple != null && tuple.first() instanceof byte[]) { - // close() calls ArrayBufferInput.close() which sets buffer = null, - // releasing the byte[] payload reference held by the unpacker's input. - // The unpacker itself is kept alive for reuse on the next parse. - tuple.second().close(); - messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); - } - } } @Override From cbe877a6a875a31bfa0cb01e8b97806b5017a3a0 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 09:18:43 +0900 Subject: [PATCH 46/50] Revert "Remove explicit ThreadLocal cleanup; rely on WeakReference for reclamation" This reverts commit de60fea4830aa3dd65ea6914c75261b8f0a4135b. --- .../dataformat/MessagePackGenerator.java | 22 +++++++++++++++++++ .../jackson/dataformat/MessagePackParser.java | 14 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 74fce079..7e852997 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -60,6 +60,7 @@ public class MessagePackGenerator private final List nodes; private boolean isElementsClosed = false; private MessagePackWriteContext writeContext; + private final boolean ownsThreadLocalBuffer; private static final class RawUtf8String { @@ -210,6 +211,7 @@ private MessagePackGenerator( this.writeContext = MessagePackWriteContext.createRootContext( StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) ? DupDetector.rootDetector(this) : null); + this.ownsThreadLocalBuffer = false; } public MessagePackGenerator( @@ -231,6 +233,7 @@ public MessagePackGenerator( this.writeContext = MessagePackWriteContext.createRootContext( StreamWriteFeature.STRICT_DUPLICATE_DETECTION.enabledIn(streamWriteFeatures) ? DupDetector.rootDetector(this) : null); + this.ownsThreadLocalBuffer = reuseResourceInGenerator; } private MessageBufferOutput getMessageBufferOutputForOutputStream( @@ -995,6 +998,25 @@ protected void _closeInput() throws IOException @Override protected void _releaseBuffers() { + // No null check on get(): generators are single-threaded by contract so this + // ThreadLocal is always set on the calling thread. A null here would indicate + // cross-thread misuse; letting it NPE surfaces that bug immediately. + if (ownsThreadLocalBuffer) { + WeakReference ref = messageBufferOutputHolder.get(); + OutputStreamBufferOutput buf = ref != null ? ref.get() : null; + if (buf != null) { + try { + // reset(null) clears the OutputStream reference inside OutputStreamBufferOutput + // but intentionally retains its internal MessageBuffer for reuse on the next + // generator created on this thread. The MessageBuffer is reclaimed when the + // WeakReference is collected after this generator instance is GC'd. + buf.reset(null); + } + catch (IOException e) { + throw _wrapIOFailure(e); + } + } + } } @Override diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index e26b950c..651ecb87 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -55,6 +55,7 @@ public class MessagePackParser private long currentPosition; private final IOContext ioContext; private ExtensionTypeCustomDeserializers extTypeCustomDesers; + private final boolean ownsThreadLocalUnpacker; private enum Type { @@ -86,6 +87,7 @@ private enum Type streamReadContext = MessagePackReadContext.createRootContext(dups); if (!reuseResourceInParser) { messageUnpacker = MessagePack.newDefaultUnpacker(input); + ownsThreadLocalUnpacker = false; return; } @@ -109,6 +111,7 @@ private enum Type messageUnpacker = messageUnpackerTuple.second(); } messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(src, messageUnpacker))); + ownsThreadLocalUnpacker = true; } public void setExtensionTypeCustomDeserializers(ExtensionTypeCustomDeserializers extTypeCustomDesers) @@ -673,6 +676,17 @@ protected void _closeInput() throws IOException if (StreamReadFeature.AUTO_CLOSE_SOURCE.enabledIn(_streamReadFeatures)) { messageUnpacker.close(); } + if (ownsThreadLocalUnpacker) { + WeakReference> ref = messageUnpackerHolder.get(); + Tuple tuple = ref != null ? ref.get() : null; + if (tuple != null && tuple.first() instanceof byte[]) { + // close() calls ArrayBufferInput.close() which sets buffer = null, + // releasing the byte[] payload reference held by the unpacker's input. + // The unpacker itself is kept alive for reuse on the next parse. + tuple.second().close(); + messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); + } + } } @Override From 47570de84d7d90a5b3313c7621f1702b802da667 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 14:21:07 +0900 Subject: [PATCH 47/50] Replace WeakReference with strong ref in generator/parser ThreadLocals Benchmarking shows our ThreadLocal buffers retain ~8 KB/thread (generator) and ~0.2 KB/thread (parser), which is negligible compared to Jackson's own BufferRecycler (~130 KB/thread for large payloads). WeakReference added no meaningful benefit in practice. --- .../benchmark/ThreadLocalMemoryBenchmark.java | 163 ++++++++++++++++++ .../dataformat/MessagePackGenerator.java | 18 +- .../jackson/dataformat/MessagePackParser.java | 16 +- plans/threadlocal-memory-benchmark-results.md | 57 ++++++ 4 files changed, 235 insertions(+), 19 deletions(-) create mode 100644 msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/ThreadLocalMemoryBenchmark.java create mode 100644 plans/threadlocal-memory-benchmark-results.md diff --git a/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/ThreadLocalMemoryBenchmark.java b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/ThreadLocalMemoryBenchmark.java new file mode 100644 index 00000000..64580f27 --- /dev/null +++ b/msgpack-jackson3/src/jmh/java/org/msgpack/jackson/dataformat/benchmark/ThreadLocalMemoryBenchmark.java @@ -0,0 +1,163 @@ +// +// MessagePack for Java +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.jackson.dataformat.benchmark; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.core.json.JsonFactory; +import tools.jackson.databind.json.JsonMapper; +import tools.jackson.core.util.JsonRecyclerPools; +import org.msgpack.jackson.dataformat.MessagePackMapper; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * Measures heap retained by generator and parser ThreadLocals across N idle threads. + * + * Usage: + * msgpack-jackson3/jmh:runMain ...ThreadLocalMemoryBenchmark [numThreads] [payloadKB] [mode] + * + * mode: generator | parser | both (default: both) + * Append ":noreuse" to mode to disable the generator ThreadLocal and isolate + * Jackson's own BufferRecycler contribution (e.g. "generator:noreuse"). + * Defaults: 1024 threads, 1024 KB payload. + */ +public class ThreadLocalMemoryBenchmark +{ + public static void main(String[] args) throws Exception + { + int numThreads = args.length > 0 ? Integer.parseInt(args[0]) : 1024; + int payloadKB = args.length > 1 ? Integer.parseInt(args[1]) : 1024; + String mode = args.length > 2 ? args[2] : "both"; + + System.out.printf("Threads: %d Payload: %d KB Mode: %s%n", numThreads, payloadKB, mode); + + boolean reuseGen = !mode.endsWith(":noreuse"); + if (!reuseGen) { mode = mode.replace(":noreuse", ""); } + + ObjectMapper mapper; + switch (mode) { + case "vanilla": + mapper = new JsonMapper(); + break; + case "vanilla-shared-pool": + mapper = new JsonMapper(JsonFactory.builder() + .recyclerPool(JsonRecyclerPools.sharedBoundedPool()) + .build()); + break; + default: + mapper = new MessagePackMapper( + new org.msgpack.jackson.dataformat.MessagePackFactory() + .setReuseResourceInGenerator(reuseGen)); + break; + } + byte[] payload = new byte[payloadKB * 1024]; + + Runnable work; + switch (mode) { + case "generator": + work = () -> { + try { mapper.writeValueAsBytes(payload); } + catch (Exception e) { throw new RuntimeException(e); } + }; + break; + case "parser": { + byte[] pre; + try { pre = mapper.writeValueAsBytes(payload); } + catch (Exception e) { throw new RuntimeException(e); } + final byte[] preSerializedPayload = pre; + work = () -> { + try { mapper.readValue(preSerializedPayload, byte[].class); } + catch (Exception e) { throw new RuntimeException(e); } + }; + break; + } + default: + work = () -> { + try { + byte[] serialized = mapper.writeValueAsBytes(payload); + mapper.readValue(serialized, byte[].class); + } + catch (Exception e) { throw new RuntimeException(e); } + }; + } + + measure(numThreads, work); + } + + private static void measure(int numThreads, Runnable work) throws Exception + { + forceGc(); + long beforePool = usedHeap(); + + ExecutorService pool = Executors.newFixedThreadPool(numThreads); + try { + // Phase 1: start all pool threads with a no-op to establish the per-thread + // baseline without any Jackson involvement. + runAll(pool, numThreads, () -> {}); + forceGc(); + long afterThreads = usedHeap(); + + // Phase 2: run the actual Jackson work on the same threads. + runAll(pool, numThreads, work); + forceGc(); + long afterWork = usedHeap(); + + long threadDelta = afterThreads - beforePool; + long jacksonDelta = afterWork - afterThreads; + System.out.printf("Before pool (baseline): %.2f MB%n", + toMB(beforePool)); + System.out.printf("After no-op (threads alive, no Jackson): %.2f MB thread overhead: %.2f KB/thread%n", + toMB(afterThreads), toKB(threadDelta) / numThreads); + System.out.printf("After Jackson work (threads alive): %.2f MB Jackson-only: %.2f KB/thread%n", + toMB(afterWork), toKB(jacksonDelta) / numThreads); + } + finally { + pool.shutdownNow(); + pool.awaitTermination(5, TimeUnit.SECONDS); + } + } + + private static void runAll(ExecutorService pool, int numThreads, Runnable work) throws Exception + { + Callable task = () -> { work.run(); return null; }; + List> futures = pool.invokeAll(Collections.nCopies(numThreads, task)); + for (Future f : futures) { + f.get(); + } + } + + private static void forceGc() throws InterruptedException + { + for (int i = 0; i < 5; i++) { System.gc(); Thread.sleep(200); } + } + + private static long usedHeap() + { + MemoryMXBean mem = ManagementFactory.getMemoryMXBean(); + return mem.getHeapMemoryUsage().getUsed(); + } + + private static double toMB(long bytes) { return bytes / (1024.0 * 1024.0); } + private static double toKB(long bytes) { return bytes / 1024.0; } +} diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index 7e852997..b870b4b8 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Reader; -import java.lang.ref.WeakReference; + import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -50,7 +50,9 @@ public class MessagePackGenerator private static final int IN_OBJECT = 1; private static final int IN_ARRAY = 2; private final MessagePacker messagePacker; - private static final ThreadLocal> messageBufferOutputHolder = new ThreadLocal<>(); + // Retained heap per idle thread: ~8 KB (OutputStreamBufferOutput + internal MessageBuffer). + // Negligible compared to Jackson's own BufferRecycler (~130 KB/thread for large payloads). + private static final ThreadLocal messageBufferOutputHolder = new ThreadLocal<>(); private final OutputStream output; private final MessagePack.PackerConfig packerConfig; private final boolean supportIntegerKeys; @@ -243,11 +245,10 @@ private MessageBufferOutput getMessageBufferOutputForOutputStream( { OutputStreamBufferOutput messageBufferOutput; if (reuseResourceInGenerator) { - WeakReference ref = messageBufferOutputHolder.get(); - messageBufferOutput = ref != null ? ref.get() : null; + messageBufferOutput = messageBufferOutputHolder.get(); if (messageBufferOutput == null) { messageBufferOutput = new OutputStreamBufferOutput(out); - messageBufferOutputHolder.set(new WeakReference<>(messageBufferOutput)); + messageBufferOutputHolder.set(messageBufferOutput); } else { messageBufferOutput.reset(out); @@ -1002,14 +1003,9 @@ protected void _releaseBuffers() // ThreadLocal is always set on the calling thread. A null here would indicate // cross-thread misuse; letting it NPE surfaces that bug immediately. if (ownsThreadLocalBuffer) { - WeakReference ref = messageBufferOutputHolder.get(); - OutputStreamBufferOutput buf = ref != null ? ref.get() : null; + OutputStreamBufferOutput buf = messageBufferOutputHolder.get(); if (buf != null) { try { - // reset(null) clears the OutputStream reference inside OutputStreamBufferOutput - // but intentionally retains its internal MessageBuffer for reuse on the next - // generator created on this thread. The MessageBuffer is reclaimed when the - // WeakReference is collected after this generator instance is GC'd. buf.reset(null); } catch (IOException e) { diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 651ecb87..48bd438e 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -35,14 +35,16 @@ import org.msgpack.value.ValueType; import java.io.IOException; -import java.lang.ref.WeakReference; + import java.math.BigDecimal; import java.math.BigInteger; public class MessagePackParser extends ParserMinimalBase { - private static final ThreadLocal>> messageUnpackerHolder = new ThreadLocal<>(); + // Retained heap per idle thread: ~0.2 KB (MessageUnpacker with cleared input buffer). + // Negligible compared to Jackson's own BufferRecycler (~130 KB/thread for large payloads). + private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); @@ -91,8 +93,7 @@ private enum Type return; } - WeakReference> ref = messageUnpackerHolder.get(); - Tuple messageUnpackerTuple = ref != null ? ref.get() : null; + Tuple messageUnpackerTuple = messageUnpackerHolder.get(); if (messageUnpackerTuple == null) { messageUnpacker = MessagePack.newDefaultUnpacker(input); } @@ -110,7 +111,7 @@ private enum Type } messageUnpacker = messageUnpackerTuple.second(); } - messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(src, messageUnpacker))); + messageUnpackerHolder.set(new Tuple<>(src, messageUnpacker)); ownsThreadLocalUnpacker = true; } @@ -677,14 +678,13 @@ protected void _closeInput() throws IOException messageUnpacker.close(); } if (ownsThreadLocalUnpacker) { - WeakReference> ref = messageUnpackerHolder.get(); - Tuple tuple = ref != null ? ref.get() : null; + Tuple tuple = messageUnpackerHolder.get(); if (tuple != null && tuple.first() instanceof byte[]) { // close() calls ArrayBufferInput.close() which sets buffer = null, // releasing the byte[] payload reference held by the unpacker's input. // The unpacker itself is kept alive for reuse on the next parse. tuple.second().close(); - messageUnpackerHolder.set(new WeakReference<>(new Tuple<>(null, tuple.second()))); + messageUnpackerHolder.set(new Tuple<>(null, tuple.second())); } } } diff --git a/plans/threadlocal-memory-benchmark-results.md b/plans/threadlocal-memory-benchmark-results.md new file mode 100644 index 00000000..45fc7be1 --- /dev/null +++ b/plans/threadlocal-memory-benchmark-results.md @@ -0,0 +1,57 @@ +# ThreadLocal Memory Benchmark Results + +Generated: 2026-05-31, branch `support-jackson3` + +## Setup + +- Threads: 1024 +- Payload: 1024 KB (1 MB byte array) +- JVM: OpenJDK 21 +- Tool: `ThreadLocalMemoryBenchmark` (msgpack-jackson3/src/jmh/...) +- ThreadLocal implementation: **strong references** (no WeakReference) + +## Measurement method + +Each run has two phases on the **same** fixed thread pool: + +1. **No-op phase** — all threads run a no-op; GC × 5; measure heap → thread-pool-only baseline +2. **Work phase** — same threads run Jackson work; GC × 5; measure heap + +Reported delta = `(afterWork − afterNoOp) / numThreads`. +Each mode row is an **independent run**, not cumulative. + +## Results + +| Mode | Heap after no-op | Heap after work | Delta/thread | +|------|-----------------|-----------------|--------------| +| thread pool only (no Jackson) | 4.24 MB | — | 0.46 KB/thread | +| vanilla (JsonMapper, write+read, default pool) | 4.24 MB | 149.01 MB | 145 KB/thread | +| vanilla-shared-pool (JsonMapper, write+read, sharedBoundedPool) | 4.24 MB | 18.37 MB | 14 KB/thread | +| generator (MessagePackMapper, write only) | 4.24 MB | 141.83 MB | 138 KB/thread | +| generator:noreuse (our ThreadLocal disabled) | 4.23 MB | 134.00 MB | 130 KB/thread | +| parser (MessagePackMapper, read only) | 5.48 MB | 5.71 MB | 0.22 KB/thread | +| both (MessagePackMapper, write+read) | 4.23 MB | 12.80 MB | 8.57 KB/thread | +| both:noreuse (our ThreadLocal disabled) | 4.24 MB | 4.70 MB | 0.47 KB/thread | + +## Our ThreadLocal cost (strong reference) + +| ThreadLocal | Retained/thread | How measured | +|---|---|---| +| `messageBufferOutputHolder` (generator) | **~8 KB** | `generator` − `generator:noreuse` = 138 − 130 | +| `messageUnpackerHolder` (parser) | **~0.2 KB** | `parser` result | + +Both are negligible compared to Jackson's own `BufferRecycler` (~130 KB/thread for large payloads). + +## Notes + +- **`vanilla` vs `vanilla-shared-pool`**: the default `JsonMapper` uses a ThreadLocal-based + `BufferRecycler` (145 KB/thread); switching to `sharedBoundedPool` drops this to 14 KB/thread. + This is a Jackson configuration choice, unrelated to our code. + +- **`both` (8.57 KB) << `generator` (138 KB)**: in `both` mode, the read runs after the write + on the same thread. Jackson's `ByteArrayBuilder` (backed by `BufferRecycler`) retains a large + write buffer in `generator`-only mode; in `both`, the subsequent read evicts it with a smaller + buffer. Our `OutputStreamBufferOutput` (~8 KB) accounts for nearly all of the `both` delta. + Note: Jackson's `BufferRecycler` is not involved in msgpack's internal write path; the large + retention in `generator` mode comes from `writeValueAsBytes`'s `ByteArrayBuilder` wrapper, + not from msgpack's own output buffer. From 23d825d8e9dc259d01937186b024f48fc63daa03 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 14:23:07 +0900 Subject: [PATCH 48/50] Remove concrete Jackson memory figures from ThreadLocal comments --- .../org/msgpack/jackson/dataformat/MessagePackGenerator.java | 2 +- .../java/org/msgpack/jackson/dataformat/MessagePackParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java index b870b4b8..97da3356 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackGenerator.java @@ -51,7 +51,7 @@ public class MessagePackGenerator private static final int IN_ARRAY = 2; private final MessagePacker messagePacker; // Retained heap per idle thread: ~8 KB (OutputStreamBufferOutput + internal MessageBuffer). - // Negligible compared to Jackson's own BufferRecycler (~130 KB/thread for large payloads). + // Negligible compared to Jackson's own per-thread buffer retention. private static final ThreadLocal messageBufferOutputHolder = new ThreadLocal<>(); private final OutputStream output; private final MessagePack.PackerConfig packerConfig; diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 48bd438e..0ff32dc2 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -43,7 +43,7 @@ public class MessagePackParser extends ParserMinimalBase { // Retained heap per idle thread: ~0.2 KB (MessageUnpacker with cleared input buffer). - // Negligible compared to Jackson's own BufferRecycler (~130 KB/thread for large payloads). + // Negligible compared to Jackson's own per-thread buffer retention. private static final ThreadLocal> messageUnpackerHolder = new ThreadLocal<>(); private final MessageUnpacker messageUnpacker; From 2d6e9bd8ba3672f68732599c766f297e44c4d1ea Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 14:32:13 +0900 Subject: [PATCH 49/50] Fix NIL map key emitting VALUE_NULL instead of PROPERTY_NAME A nil key in a msgpack map was always emitting VALUE_NULL, ignoring the isObjectValueSet flag checked by all other key types (STRING, INTEGER, BOOLEAN). This desynchronized the read context, causing the following value to be misinterpreted as the next key. Add testNilKey to cover the corrected token stream. --- .../jackson/dataformat/MessagePackParser.java | 8 +++++++- .../dataformat/MessagePackParserTest.java | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java index 0ff32dc2..e2da95a4 100644 --- a/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java +++ b/msgpack-jackson3/src/main/java/org/msgpack/jackson/dataformat/MessagePackParser.java @@ -227,7 +227,13 @@ else if (streamReadContext.inArray()) { case NIL: type = Type.NULL; messageUnpacker.unpackNil(); - nextToken = JsonToken.VALUE_NULL; + if (isObjectValueSet) { + streamReadContext.setCurrentName(null); + nextToken = JsonToken.PROPERTY_NAME; + } + else { + nextToken = JsonToken.VALUE_NULL; + } break; case BOOLEAN: boolean b = messageUnpacker.unpackBoolean(); diff --git a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java index babbc866..970daac5 100644 --- a/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java +++ b/msgpack-jackson3/src/test/java/org/msgpack/jackson/dataformat/MessagePackParserTest.java @@ -705,6 +705,26 @@ public Object deserializeKey(String key, DeserializationContext ctxt) assertEquals((Integer) 11, map.get(false)); } + @Test + public void testNilKey() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(out).packMapHeader(1); + packer.packNil(); + packer.packInt(42); + packer.close(); + + JsonParser parser = new MessagePackMapper().createParser(out.toByteArray()); + assertEquals(JsonToken.START_OBJECT, parser.nextToken()); + assertEquals(JsonToken.PROPERTY_NAME, parser.nextToken()); + assertNull(parser.currentName()); + assertEquals(JsonToken.VALUE_NUMBER_INT, parser.nextToken()); + assertEquals(42, parser.getIntValue()); + assertEquals(JsonToken.END_OBJECT, parser.nextToken()); + parser.close(); + } + @Test public void extensionTypeCustomDeserializers() throws IOException From f77b2d9db3338492adc3c9899a64800845fb9305 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sun, 31 May 2026 15:01:17 +0900 Subject: [PATCH 50/50] Fix stale WeakReference docs and README method reference - preexisting-issues.md: update parser ThreadLocal description to reflect strong references and actual close() behavior - README.md: MessagePackMapper.Builder#handleBigIntegerAndBigDecimalAsString(), not MessagePackMapper# --- msgpack-jackson3/README.md | 2 +- plans/preexisting-issues.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack-jackson3/README.md b/msgpack-jackson3/README.md index d50c81ea..26262b70 100644 --- a/msgpack-jackson3/README.md +++ b/msgpack-jackson3/README.md @@ -231,7 +231,7 @@ byte[] converted = objectMapper.writeValueAsBytes(obj); System.out.println(objectMapper.readValue(converted, Pojo.class)); // => Pojo{value=1234567890.98765432100} ``` -`MessagePackMapper#handleBigIntegerAndBigDecimalAsString()` is equivalent to the following configuration. +`MessagePackMapper.Builder#handleBigIntegerAndBigDecimalAsString()` is equivalent to the following configuration. ```java ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); diff --git a/plans/preexisting-issues.md b/plans/preexisting-issues.md index f4ffbe97..1f9aca26 100644 --- a/plans/preexisting-issues.md +++ b/plans/preexisting-issues.md @@ -174,10 +174,10 @@ both bugs compound). retains the entire last parsed payload for each thread in a pool indefinitely, which can cause unbounded memory retention after large messages. -Fixed in msgpack-jackson3: sources are now wrapped in `WeakReference` so GC can reclaim -them once the caller drops its reference. On `close()`, byte-array sources are replaced -with `WeakReference(null)` for prompt release; InputStream sources are left alive in the -WeakReference as long as the caller holds them, preserving same-stream reuse detection. +Fixed in msgpack-jackson3: on `close()`, byte-array sources are released by calling +`unpacker.close()` (nulls out `ArrayBufferInput`'s internal buffer) and replacing the +Tuple with `(null, unpacker)`. The ThreadLocal uses strong references; benchmarking showed +retention is ~0.2 KB/thread, which is negligible. Needs the same fix in msgpack-jackson. ## 3. `MessagePackGenerator`: `close()` does not set `isClosed()` to true