From 895586cc92c3420595bd3210ae2a4992c7b076fd Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 12:48:48 +0200 Subject: [PATCH 01/10] Remove legacy DirectByteBuffer-based OTEP context storage (Phase 3) dd-trace-java PR #11899 has migrated onto the all-native context API (setTraceContext/clearTraceContext/setContextValue/clearContextValue), so java-profiler was the last consumer of the DirectByteBuffer-based ThreadContext/OtelContextStorage path, which carried a virtual-thread use-after-free risk. Deletes the DBB conduit and its Java/native/test/ benchmark/chaos surface, replacing coverage with equivalents against the all-native path. Co-Authored-By: Claude Sonnet 5 --- ddprof-lib/src/main/cpp/javaApi.cpp | 177 +++-- ddprof-lib/src/main/cpp/otel_context.h | 3 +- ddprof-lib/src/main/cpp/threadLocalData.h | 7 +- ddprof-lib/src/main/cpp/wallClock.cpp | 4 +- .../com/datadoghq/profiler/ContextSetter.java | 115 ---- .../profiler/ContextStorageMode.java | 28 - .../datadoghq/profiler/ContextValueCache.java | 10 +- .../com/datadoghq/profiler/JavaProfiler.java | 246 ++----- .../com/datadoghq/profiler/OTelContext.java | 2 +- .../profiler/OtelContextStorage.java | 165 ----- .../com/datadoghq/profiler/ScopeStack.java | 96 --- .../com/datadoghq/profiler/ThreadContext.java | 644 ------------------ ddprof-stresstest/README.md | 14 +- ddprof-stresstest/build.gradle.kts | 2 +- ddprof-stresstest/src/chaos/README.md | 4 +- .../com/datadoghq/profiler/chaos/Main.java | 6 +- .../chaos/ReapplyContextAntagonist.java | 124 ---- .../chaos/ReapplyContextValueAntagonist.java | 114 ++++ ...VirtualThreadContextCascadeAntagonist.java | 560 --------------- .../counters/TracedParallelWork.java | 22 +- .../throughput/ContextCombinedBenchmark.java | 46 +- .../throughput/ThreadContextBenchmark.java | 209 ------ .../profiler/context/ContextExecutor.java | 5 - .../datadoghq/profiler/context/Tracing.java | 2 +- .../profiler/AbstractProfilerTest.java | 1 - .../profiler/ContextValueCacheTest.java | 115 ++++ .../profiler/MaxContextSlotsTest.java | 3 - .../profiler/OtelContextStorageTest.java | 113 --- .../datadoghq/profiler/ScopeStackTest.java | 164 ----- .../context/AllNativeContextSamplingTest.java | 15 +- .../context/AllNativeContextTest.java | 189 ++--- .../context/CarrierContextStorageTest.java | 133 ---- .../context/OtelContextStorageModeTest.java | 296 -------- .../profiler/context/TagContextTest.java | 641 ----------------- .../filter/ThreadFilterSmokeTest.java | 3 +- .../WriteStackTracesAfterClassUnloadTest.java | 6 +- .../profiler/queue/QueueTimeTest.java | 5 +- .../profiler/wallclock/PrecheckTest.java | 15 +- .../WallclockMitigationsCombinedTest.java | 4 +- doc/architecture/TLSContext.md | 319 +++------ ...07-02-all-native-context-storage-design.md | 316 --------- ...26-07-09-phase2-dd-trace-java-migration.md | 89 --- 42 files changed, 604 insertions(+), 4428 deletions(-) delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java delete mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java create mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java delete mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java delete mode 100644 ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java delete mode 100644 doc/plans/2026-07-02-all-native-context-storage-design.md delete mode 100644 doc/plans/2026-07-09-phase2-dd-trace-java-migration.md diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 83f6045c1a..3f71a1b74c 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -710,51 +710,6 @@ Java_com_datadoghq_profiler_OTelContext_readProcessCtx0(JNIEnv *env, jclass unus #endif } -extern "C" DLLEXPORT jobject JNICALL -Java_com_datadoghq_profiler_JavaProfiler_initializeContextTLS0(JNIEnv* env, jclass unused, jlongArray metadata) { - // Initialize thread TLS if it has not yet done - ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); - assert(thrd != nullptr); - - if (!thrd->isContextInitialized()) { - ContextApi::initializeContextTLS(thrd); - } - - OtelThreadContextRecord* record = thrd->getOtelContextRecord(); - - // Contiguity of record + tag_encodings + LRS is enforced by alignas(8) on _otel_ctx_record - // plus sizeof(OtelThreadContextRecord) being a multiple of 8 (see thread.h). - // Compile-time alignment check always runs; runtime pointer-layout check is debug-only. - static_assert(DD_TAGS_CAPACITY * sizeof(u32) % alignof(u64) == 0, - "tag encodings array size must be aligned to u64 for contiguous sidecar layout"); -#ifdef DEBUG - uint8_t* record_start = reinterpret_cast(record); - uint8_t* sidecar_start = reinterpret_cast(thrd->getOtelTagEncodingsPtr()); - assert(sidecar_start == record_start + OTEL_MAX_RECORD_SIZE - && "_otel_ctx_record and _otel_tag_encodings must be contiguous"); -#endif - - // Fill metadata[6]: [VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - // ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET]. - // All offsets are absolute within the unified buffer returned below. - if (metadata != nullptr && env->GetArrayLength(metadata) >= 6) { - jlong meta[6]; - meta[0] = (jlong)offsetof(OtelThreadContextRecord, valid); - meta[1] = (jlong)offsetof(OtelThreadContextRecord, trace_id); - meta[2] = (jlong)offsetof(OtelThreadContextRecord, span_id); - meta[3] = (jlong)offsetof(OtelThreadContextRecord, attrs_data_size); - meta[4] = (jlong)offsetof(OtelThreadContextRecord, attrs_data); - meta[5] = (jlong)(OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32)); - env->SetLongArrayRegion(metadata, 0, 6, meta); - } - - // Single contiguous view over [record | tag_encodings | LRS] — used for per-field - // access and for bulk snapshot/restore. All three regions are in one ProfiledThread - // memory block. - size_t totalSize = OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32) + sizeof(u64); - return env->NewDirectByteBuffer((void*)record, (jlong)totalSize); -} - // --------------------------------------------------------------------------- // All-native context write API (OTEP #4947). // @@ -762,25 +717,21 @@ Java_com_datadoghq_profiler_JavaProfiler_initializeContextTLS0(JNIEnv* env, jcla // ProfiledThread::current(). Because a JNI native frame pins a mounted virtual thread to its // carrier for the duration of the call (the continuation cannot freeze across a native frame), // the record resolved here is guaranteed live and cannot migrate mid-write — there is no cached -// per-thread buffer to dangle. This replaces the DirectByteBuffer path (see ThreadContext), -// eliminating the virtual-thread use-after-free. +// per-thread buffer to dangle, and hence no virtual-thread use-after-free. // // Signal-safety / reader coherence: the sampler (ContextApi::get, wallClock.cpp) reads the same -// record on the same carrier, gated on record->valid. Each write follows the detach -> mutate -> -// attach protocol used by ThreadContext: store valid=0, release fence, mutate, release fence, -// store valid=1. On x86 the release fence is a compiler barrier; on aarch64 it is a real barrier -// pairing with the sampler's acquire load of valid in ContextApi::get. +// record on the same carrier, gated on record->valid. Each write follows a detach -> mutate -> +// attach protocol: store valid=0, release fence, mutate, release fence, store valid=1. On x86 the +// release fence is a compiler barrier; on aarch64 it is a real barrier pairing with the sampler's +// acquire load of valid in ContextApi::get. // --------------------------------------------------------------------------- -// Byte layout constants shared with ThreadContext (see otel_context.h / ThreadContext.java). static const int OTEL_LRS_ENTRY_SIZE = 18; // fixed attrs_data[0] entry: key(1)+len(1)+16 hex bytes // Writes the full fixed LRS attrs_data entry: header (key_index=0, length=16) at attrs_data[0..2) // plus the 16 hex value bytes at attrs_data[2..18). The combined write/clear entry points -// (setTraceContext0 / clearTraceContext0) call this so they establish the LRS entry themselves -// rather than relying on the ThreadContext ctor — i.e. they work on a record that only saw the -// ProfiledThread zero-init, the phase-2 pure-native case where no DirectByteBuffer / ThreadContext -// was ever created. Mirrors ThreadContext's LRS entry layout. +// (setTraceContext0 / clearTraceContext0) establish this entry themselves on a record that only +// ever saw the ProfiledThread zero-init. // // Note: the single-attribute path (setContextValue0) does NOT write this entry; it assumes a // preceding setTraceContext0 has already established it (the production order — app tags are set @@ -800,7 +751,7 @@ static inline void otelWriteLrsEntry(OtelThreadContextRecord* record, u64 v) { } // Compacts out the attrs_data entry with the given OTEP key index; returns the new size. -// Mirrors ThreadContext.compactOtepAttribute. Record must be detached. +// Record must be detached. static int otelCompactAttr(OtelThreadContextRecord* record, int otepKeyIndex) { int currentSize = record->attrs_data_size; uint8_t* d = record->attrs_data; @@ -825,7 +776,7 @@ static int otelCompactAttr(OtelThreadContextRecord* record, int otepKeyIndex) { } // Replaces/inserts an attribute value in attrs_data (record must be detached). Returns false on -// attrs_data overflow (nothing appended). Mirrors ThreadContext.replaceOtepAttribute. +// attrs_data overflow (nothing appended). static bool otelReplaceAttr(OtelThreadContextRecord* record, int otepKeyIndex, const uint8_t* utf8, int valueLen) { int currentSize = otelCompactAttr(record, otepKeyIndex); @@ -886,10 +837,10 @@ Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass un // clearTraceContext0. The public setTraceContext wrapper enforces this by throwing // IllegalArgumentException, so a zero span reaching here is a direct-JNI/contract violation. assert(spanId != 0 && "setTraceContext0 requires a non-zero span; use clearTraceContext0 to clear"); - // Publish the OTEP TLS pointer and mark the thread initialized on first native write, exactly - // as the DirectByteBuffer path does in initializeContextTLS0. Without this a thread that only - // ever uses the all-native API writes a record that ContextApi::get / the wallclock sampler - // ignore (both gate on isContextInitialized) and that external OTEP readers can't discover. + // Publish the OTEP TLS pointer and mark the thread initialized on first native write. Without + // this a thread that never explicitly initializes context writes a record that ContextApi::get + // / the wallclock sampler ignore (both gate on isContextInitialized) and that external OTEP + // readers can't discover. if (!thrd->isContextInitialized()) { ContextApi::initializeContextTLS(thrd); } @@ -931,7 +882,7 @@ Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass un } // Combined per-deactivation clear: zeros scalar context + custom slots and leaves the record -// detached (valid=0), mirroring the DBB clear path (setContext(0,0,0,0) + clearContextValue*). +// detached (valid=0). extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass unused) { ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); @@ -951,7 +902,7 @@ Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass *lrs = 0; record->attrs_data_size = (uint16_t)OTEL_LRS_ENTRY_SIZE; otelWriteLrsEntry(record, 0); - // clear path leaves valid=0 (no attach), mirroring ThreadContext.clearContextDirect. + // clear path leaves valid=0 (no attach) — a cleared context is not "active". } // Single pre-resolved attribute write (sidecar encoding + attrs_data value) in one detach/attach @@ -977,10 +928,9 @@ Java_com_datadoghq_profiler_JavaProfiler_setContextValue0(JNIEnv* env, jclass un // Publish the record (valid=1) after the write: setting a value means "make it visible", so an // attribute is observable to the sampler even with no active span (zero trace/span). This is the // app-context-independent-of-span model dd-trace-java relies on — app-owned attributes (e.g. - // http.route) and the reapply-after-deactivation path stay visible between spans. Mirrors the DBB - // single-attribute setter ThreadContext.setContextAttributeDirect, which likewise always - // re-attaches. (clearContextValue0, by contrast, preserves the prior valid: removing a value must - // not resurrect a deactivated record.) + // http.route) and the reapply-after-deactivation path stay visible between spans. + // (clearContextValue0, by contrast, preserves the prior valid: removing a value must not + // resurrect a deactivated record.) __atomic_store_n(&record->valid, (uint8_t)0, __ATOMIC_RELAXED); __atomic_thread_fence(__ATOMIC_RELEASE); @@ -1022,10 +972,7 @@ Java_com_datadoghq_profiler_JavaProfiler_clearContextValue0(JNIEnv* env, jclass } // Copies the current thread's custom-attribute sidecar tag encodings (enc[]) into out, reading the -// record directly via ProfiledThread::current() — no ThreadContext / DirectByteBuffer, so it does -// NOT reset the record the way the deprecated DBB copyTags does. This lets a caller observe -// encodings written through the all-native setContextValue path (introspection / tests); the DBB -// read went through ThreadContext, whose ctor resets the record and would clobber native writes. +// record directly via ProfiledThread::current() without mutating it. Introspection / test use. extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass unused, jintArray out) { if (out == nullptr) { @@ -1050,7 +997,7 @@ Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass un } extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_ThreadContext_registerConstant0(JNIEnv* env, jclass unused, jstring value) { +Java_com_datadoghq_profiler_ContextValueCache_registerConstant0(JNIEnv* env, jclass unused, jstring value) { JniString value_str(env, value); u32 encoding = Profiler::instance()->contextValueMap()->bounded_lookup( value_str.c_str(), value_str.length(), 1 << 16); @@ -1088,3 +1035,87 @@ Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) ContextApi::get(spanId, rootSpanId); TEST_LOG("===> Context: tid:%lu, spanId=%lu, rootSpanId=%lu", OS::threadId(), spanId, rootSpanId); } + +// ---- Test-only reads of the current thread's OTEP record ----------------------------------- +// Each reads the current carrier's record directly via ProfiledThread::current(), with no +// detach/attach (diagnostic-only, not on any signal-handler or hot write path). + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testGetSpanId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return 0; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + uint64_t beSpan; + memcpy(&beSpan, record->span_id, 8); + return (jlong)__builtin_bswap64(beSpan); +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testGetRootSpanId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return 0; + } + u32* enc = thrd->getOtelTagEncodingsPtr(); + u64* lrs = reinterpret_cast(enc + DD_TAGS_CAPACITY); + return (jlong)*lrs; +} + +extern "C" DLLEXPORT jstring JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testReadTraceId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return nullptr; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + static const char HEXD[16] = + {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; + char hex[33]; + for (int i = 0; i < 16; i++) { + uint8_t b = record->trace_id[i]; + hex[i * 2] = HEXD[b >> 4]; + hex[i * 2 + 1] = HEXD[b & 0xF]; + } + hex[32] = '\0'; + return env->NewStringUTF(hex); +} + +extern "C" DLLEXPORT jstring JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testReadContextAttribute0(JNIEnv* env, jclass unused, jint slot) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr || slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { + return nullptr; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + int targetKey = slot + 1; + int size = record->attrs_data_size; + uint8_t* d = record->attrs_data; + int pos = 0; + while (pos + 2 <= size) { + int k = d[pos]; + int len = d[pos + 1]; + if (pos + 2 + len > size) { + break; + } + if (k == targetKey) { + char buf[256]; + memcpy(buf, d + pos + 2, len); + buf[len] = '\0'; + return env->NewStringUTF(buf); + } + pos += 2 + len; + } + return nullptr; +} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testIsContextValid0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return JNI_FALSE; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + return __atomic_load_n(&record->valid, __ATOMIC_ACQUIRE) ? JNI_TRUE : JNI_FALSE; +} diff --git a/ddprof-lib/src/main/cpp/otel_context.h b/ddprof-lib/src/main/cpp/otel_context.h index 82a6701b2d..cee65b747b 100644 --- a/ddprof-lib/src/main/cpp/otel_context.h +++ b/ddprof-lib/src/main/cpp/otel_context.h @@ -57,8 +57,7 @@ struct __attribute__((packed)) OtelThreadContextRecord { uint8_t attrs_data[OTEL_MAX_ATTRS_DATA_SIZE]; }; static_assert(sizeof(OtelThreadContextRecord) == OTEL_MAX_RECORD_SIZE, - "OtelThreadContextRecord size must match OTEL_MAX_RECORD_SIZE (640 bytes); " - "update the Java constant ThreadContext.OTEL_MAX_RECORD_SIZE if the struct changes"); + "OtelThreadContextRecord size must match OTEL_MAX_RECORD_SIZE (640 bytes)"); // OTEP #4947 TLS pointer — MUST appear in dynsym for external profiler discovery DLLEXPORT extern thread_local OtelThreadContextRecord* otel_thread_ctx_v1; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 293a8c15a8..ff8f68a026 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -90,13 +90,10 @@ class ProfiledThread : public ThreadLocalData { #endif // alignas(8) + sizeof(OtelThreadContextRecord)==640 (multiple of 8) guarantee // _otel_tag_encodings sits at +640 with no padding, so the three fields form one - // 688-byte contiguous region exposed as a combined DirectByteBuffer. + // 688-byte contiguous region. alignas(8) OtelThreadContextRecord _otel_ctx_record; - // These two fields MUST be contiguous and 8-byte aligned — the JNI layer - // exposes them as a single DirectByteBuffer (sidecar), and VarHandle long - // views require 8-byte alignment for the buffer base address. + // 8-byte aligned so VarHandle long views over this region require no unaligned access. // Read invariant: sidecar readers must gate on record->valid (see ContextApi::get). - // ThreadContext.restore() relies on this to perform a bulk memcpy under valid=0. alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY]; u64 _otel_local_root_span_id; diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index eac9f3fd37..798da6c3d3 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -51,9 +51,9 @@ static inline bool hasKnownActiveTraceContext(ProfiledThread* thread) { } OtelThreadContextRecord* record = thread->getOtelContextRecord(); - // record->valid is not a context-presence bit. ThreadContext leaves cleared + // record->valid is not a context-presence bit. clearTraceContext0 leaves cleared // records invalid indefinitely, so gating on valid=1 disables wallprecheck for - // the common no-context state after a Java ThreadLocal reset. + // the common no-context state after deactivation. return loadSpanId(record) != 0; } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java deleted file mode 100644 index 09ed77f638..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * DirectByteBuffer context wrapper. - * - * @deprecated Superseded by {@link JavaProfiler#setTraceContext} / - * {@link JavaProfiler#setContextValue} (all-native). Removed in phase 3. - */ -@Deprecated -public class ContextSetter { - - private final List attributes; - private final JavaProfiler profiler; - - public ContextSetter(JavaProfiler profiler, List attributes) { - this.profiler = profiler; - Set unique = new HashSet<>(attributes); - this.attributes = new ArrayList<>(unique.size()); - for (int i = 0; i < Math.min(attributes.size(), ThreadContext.MAX_CUSTOM_SLOTS); i++) { - String attribute = attributes.get(i); - if (unique.remove(attribute)) { - this.attributes.add(attribute); - } - } - } - - public int[] snapshotTags() { - int[] snapshot = new int[attributes.size()]; - profiler.copyTags(snapshot); - return snapshot; - } - - /** - * Copies current sidecar encodings into {@code snapshot}. The array must have at least - * {@code attributes.size()} elements; arrays shorter than {@code attributes.size()} are - * silently ignored. Indices {@code [attributes.size(), snapshot.length)} are zeroed after - * copying to prevent stale data from leaking to the caller. - * Use the no-arg {@link #snapshotTags()} overload to obtain a correctly sized array. - */ - public void snapshotTags(int[] snapshot) { - if (snapshot.length >= attributes.size()) { - profiler.copyTags(snapshot); - Arrays.fill(snapshot, attributes.size(), snapshot.length, 0); - } - } - - public int offsetOf(String attribute) { - return attributes.indexOf(attribute); - } - - /** Number of (deduplicated, truncated) context attribute slots. Pure Java; no native/DBB read. */ - public int size() { - return attributes.size(); - } - - public boolean setContextValue(String attribute, String value) { - return setContextValue(offsetOf(attribute), value); - } - - public boolean setContextValue(int offset, String value) { - if (offset >= 0) { - return profiler.setContextAttribute(offset, value); - } - return false; - } - - /** - * Re-applies attribute values from precomputed constant IDs and UTF-8 bytes, indexed by slot - * (as produced by {@link #snapshotTags(int[])}). Restores both the DD sidecar encoding and the - * OTEP attrs_data value for every slot whose constantId is {@code > 0}, in a single atomic - * publish — no String allocation, hashing, or cache lookup. Intended for re-applying - * application-managed context after a {@code setContext} span activation wipes the slots. - * - *

Partial-write on overflow. A {@code false} return does not mean the record is - * unchanged: slots that were written before an attrs_data overflow remain published. Overflowed - * slots are zeroed in both the sidecar and attrs_data views. Callers must not assume the record - * is unmodified when {@code false} is returned. - */ - public boolean setContextValuesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return profiler.setContextAttributesByIdAndBytes(constantIds, utf8); - } - - public boolean clearContextValue(String attribute) { - return clearContextValue(offsetOf(attribute)); - } - - public boolean clearContextValue(int offset) { - if (offset >= 0) { - profiler.clearContextAttribute(offset); - return true; - } - return false; - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java deleted file mode 100644 index 6aa18260d7..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -/** - * Scope of the OTEL context {@link ThreadContext} storage actually in effect, as reported by - * {@link JavaProfiler#contextStorageMode()}. See {@link OtelContextStorage} for how it is - * selected. - */ -public enum ContextStorageMode { - /** Carrier-scoped via {@code jdk.internal.misc.CarrierThreadLocal} (JDK 21+). */ - CARRIER, - /** Legacy virtual-thread-scoped plain {@link ThreadLocal}. */ - THREAD -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java index 7a0eaafb9d..8e8171959f 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java @@ -40,9 +40,8 @@ * session become stale. {@link JavaProfiler} therefore calls {@link #clear()} on every {@code start} * command; a subsequent {@link #resolve} re-registers the value and re-caches its new encoding. * - *

Replaces the per-{@link ThreadContext} value cache: in the all-native model there is no - * per-thread {@code ThreadContext} instance to host it, and a global cache avoids duplicating the - * table across every carrier / virtual thread. + *

In the all-native model there is no per-thread instance to host this cache, so it is a + * single process-wide table shared across every carrier / virtual thread. */ final class ContextValueCache { @@ -89,7 +88,7 @@ Entry resolve(String value) { if (utf8.length > MAX_VALUE_BYTES) { return null; } - int encoding = ThreadContext.registerConstant0(value); + int encoding = registerConstant0(value); if (encoding < 0) { return null; // Dictionary full } @@ -109,4 +108,7 @@ void clear() { table.set(i, null); } } + + /** Registers {@code value} in the native Dictionary, returning its encoding, or a negative value if full. */ + private static native int registerConstant0(String value); } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index eab2de7580..83a9f6d73a 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -64,43 +64,16 @@ private static boolean isVirtualThread(Thread thread) { } } - // Storage for profiling context. Scoped to the carrier thread when available so a - // mounted virtual thread resolves to its current carrier's OTEP record (the record the - // sampler reads); falls back to plain thread-local storage otherwise. See - // OtelContextStorage for the mode selection and the rationale. - private final ThreadLocal tlsContextStorage = OtelContextStorage.create(); - // Process-wide value->(encoding, utf8) cache for the all-native context write path // (setTraceContext / setContextValue). See ContextValueCache. One instance on the singleton. private final ContextValueCache contextValueCache = new ContextValueCache(); // Number of custom attribute slots on the all-native path. Must equal the native // DD_TAGS_CAPACITY (context.h); kept as a literal (not derived via JNI) because it bounds - // array-slot checks that can run before the native library is loaded, and kept independent of - // ThreadContext so the phase-3 removal of ThreadContext does not strand this constant. Drift - // from the native value is caught at test time by MaxContextSlotsTest via maxContextSlots0(). + // array-slot checks that can run before the native library is loaded. Drift from the native + // value is caught at test time by MaxContextSlotsTest via maxContextSlots0(). static final int MAX_CONTEXT_SLOTS = 10; - /** - * Returns the calling thread's (or, in carrier mode, its current carrier's) - * {@link ThreadContext}, creating and caching it on first use. Replaces the previous - * {@code ThreadLocal.withInitial(...)} supplier: a carrier-scoped storage instance is - * built reflectively and cannot carry a supplier, so lazy initialization is done here. - * - *

Race-free without synchronization: a carrier runs at most one mounted virtual - * thread at a time and this method has no blocking point, so no unmount can occur - * mid-call. A redundant re-init could at worst produce a second {@link ThreadContext} - * over the same carrier record, which is harmless. - */ - private ThreadContext currentContext() { - ThreadContext ctx = tlsContextStorage.get(); - if (ctx == null) { - ctx = initializeThreadContext(); - tlsContextStorage.set(ctx); - } - return ctx; - } - private JavaProfiler() { } @@ -250,54 +223,10 @@ public void removeThread() { filterThreadRemove0(); } - /** - * Passing context identifier to a profiler. This ID is thread-local and is dumped in - * the JFR output only. 0 is a reserved value for "no-context". - * - *

Note: {@code rootSpanId} maps to {@code localRootSpanId} internally. A synthetic - * trace_id of {@code [0, spanId]} is written to the OTEP record. For correct W3C - * trace ID interop use {@link #setContext(long, long, long, long)}. - * - * @param spanId Span identifier that should be stored for current thread - * @param rootSpanId Local root span identifier (used for endpoint correlation) - * @deprecated Use {@link #setContext(long, long, long, long)} for full OTEP interop. - */ - @Deprecated - public void setContext(long spanId, long rootSpanId) { - currentContext().put(spanId, rootSpanId); - } - - /** - * Sets trace context with full 128-bit W3C trace ID, span ID, and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId Span identifier - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - * @deprecated DirectByteBuffer path; use {@link #setTraceContext} (all-native). Removed in phase 3. - */ - @Deprecated - public void setContext(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - currentContext().put(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Resets the current thread's context to zero (traceId=0, spanId=0, localRootSpanId=0). - * Custom context attributes are also cleared. - * - * @deprecated DirectByteBuffer path; use {@link #clearTraceContext} (all-native). Removed in phase 3. - */ - @Deprecated - public void clearContext() { - currentContext().put(0, 0, 0, 0); - } - // ---- All-native context write API (OTEP #4947) -------------------------------------------- - // Provisional names (subject to dd-trace-java coordination). These resolve the current carrier's - // OTEP record inside a single JNI call per operation — no cached DirectByteBuffer, so they are - // race-free under virtual-thread migration (see the design note and setTraceContext0 et al.). - // They coexist with the deprecated DirectByteBuffer path below; both write the same native - // record, and no thread uses both at once. + // Each of these resolves the current carrier's OTEP record inside a single JNI call per + // operation — no cached DirectByteBuffer, so they are race-free under virtual-thread migration + // (see the design note and setTraceContext0 et al.). /** * Combined per-scope-activation write: full trace/span context plus up to two span-derived @@ -375,10 +304,9 @@ public void clearContextValue(int slot) { /** * Copies the current thread's custom-attribute sidecar tag encodings into {@code out} (index = - * slot), reading the native record directly — no {@link ThreadContext} / DirectByteBuffer, so it - * does not reset the record. Unlike the deprecated DBB read path, this observes encodings written - * through the all-native {@link #setContextValue} path. Introspection / test use; entries beyond - * {@code MAX_CONTEXT_SLOTS} are left untouched. + * slot), reading the native record directly. Observes encodings written through the all-native + * {@link #setContextValue} path. Introspection / test use; entries beyond {@code + * MAX_CONTEXT_SLOTS} are left untouched. */ public void copyContextTags(int[] out) { copyContextTags0(out); @@ -414,69 +342,6 @@ private ContextValueCache.Entry resolveContextValue(int slot, CharSequence value return contextValueCache.resolve(value.toString()); } - /** - * Sets a custom context attribute at the given slot offset for the current thread. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values return {@code false} - * @param value the string value to record; {@code null} returns {@code false} without - * writing; an empty string is written as a zero-length entry (not a clear — - * use {@link #clearContextAttribute(int)} to remove a value) - * @return true if the value was recorded; false if {@code offset} is out of range, - * {@code value} is null, the Dictionary is full, or {@code attrs_data} overflows - * for this slot - * @deprecated DirectByteBuffer path; use {@link #setContextValue} (all-native). Removed in phase 3. - */ - @Deprecated - public boolean setContextAttribute(int offset, String value) { - return currentContext().setContextAttribute(offset, value); - } - - /** - * Clears the custom context attribute at the given slot offset for the current thread. - * Zeros the sidecar encoding and removes it from OTEP {@code attrs_data}. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values are silently ignored - * @deprecated DirectByteBuffer path; use {@link #clearContextValue} (all-native). Removed in phase 3. - */ - @Deprecated - public void clearContextAttribute(int offset) { - currentContext().clearContextAttribute(offset); - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes for - * the current thread in a single detach/attach window. - * - *

    - *
  • Slots with {@code constantIds[i] <= 0} are skipped.
    • - *
  • Returns {@code false} without writing if the thread's record is not currently valid - * (span-less), to avoid resurrecting a cleared record.
    • - *
  • On {@code attrs_data} overflow, the overflowed slot's sidecar is zeroed and - * {@code false} is returned; slots written before the overflow are retained.
    • - *
- * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most 255 bytes - * (the OTEP attrs_data entry length field is one byte) for every slot - * whose {@code constantId > 0} - * @return true if every slot with {@code constantId > 0} was written; false on a cleared - * (span-less) record, or {@code attrs_data} overflow for any slot - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} is null - * @throws IllegalArgumentException if the arrays have different lengths, exceed the slot limit, - * or any active {@code utf8[i]} exceeds 255 bytes - * @deprecated DirectByteBuffer path; unused by dd-trace-java. Removed in phase 3. - */ - @Deprecated - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return currentContext().setContextAttributesByIdAndBytes(constantIds, utf8); - } - - @Deprecated - void copyTags(int[] snapshot) { - currentContext().copyCustoms(snapshot); - } - /** * Dumps the JFR recording at the provided path * @param recording the path to the recording @@ -589,15 +454,6 @@ public Map getDebugCounters() { return counters; } - private static ThreadContext initializeThreadContext() { - long[] metadata = new long[6]; - ByteBuffer buffer = initializeContextTLS0(metadata); - if (buffer == null) { - throw new IllegalStateException("Failed to initialize OTEL TLS — ProfiledThread not available"); - } - return new ThreadContext(buffer, metadata); - } - private static native boolean init0(); private native void stop0() throws IllegalStateException; private native String execute0(String command) throws IllegalArgumentException, IllegalStateException, IOException; @@ -635,22 +491,6 @@ private static ThreadContext initializeThreadContext() { private static native String getStatus0(); - /** - * Initializes context TLS for the current thread and returns a single DirectByteBuffer - * spanning the OTEP record + tag-encoding sidecar + LRS (688 bytes, contiguous in - * ProfiledThread). Sets otel_thread_ctx_v1 permanently to the thread's - * OtelThreadContextRecord. - * - * @param metadata output array filled with absolute offsets into the returned buffer: - * [0] VALID_OFFSET — offset of 'valid' field - * [1] TRACE_ID_OFFSET — offset of 'trace_id' field - * [2] SPAN_ID_OFFSET — offset of 'span_id' field - * [3] ATTRS_DATA_SIZE_OFFSET — offset of 'attrs_data_size' field - * [4] ATTRS_DATA_OFFSET — offset of 'attrs_data' field - * [5] LRS_OFFSET — offset of local_root_span_id - */ - private static native ByteBuffer initializeContextTLS0(long[] metadata); - // All-native context write primitives (OTEP #4947). Each resolves the current carrier's record // inside the JNI call (which pins a mounted virtual thread to its carrier), so there is no // cached per-thread buffer to dangle. See the native implementations in javaApi.cpp and the @@ -672,35 +512,6 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ private static native boolean consumeContextDictionaryReset0(); - /** - * Returns the {@link ThreadContext} for the current storage slot (the calling thread, or in - * {@link ContextStorageMode#CARRIER} its current carrier). - * - *

Do not cache the returned instance across a point where the calling thread may be - * unmounted and remounted on a different carrier (any blocking operation on a virtual - * thread). In carrier mode the returned context's buffer targets the carrier that was mounted - * at call time; after migration it no longer corresponds to the current carrier's record — the - * sampler reads the new carrier, and once the old carrier's OS thread exits the buffer dangles. - * Callers that write context (span/attributes) should re-fetch per use — the {@code setContext*} - * methods already do this internally via {@code currentContext()}. - * - * @deprecated DirectByteBuffer path (test/diagnostic only); the all-native API is stateless and - * exposes no per-thread handle. Removed in phase 3. - */ - @Deprecated - public ThreadContext getThreadContext() { - return currentContext(); - } - - /** - * Diagnostics/tests: the resolved OTEL context storage mode, as selected by - * {@code -D}{@value OtelContextStorage#MODE_PROPERTY} and the availability of - * {@code jdk.internal.misc.CarrierThreadLocal}. - */ - public ContextStorageMode contextStorageMode() { - return OtelContextStorage.modeOf(tlsContextStorage); - } - // --- test and debug utility methods /** @@ -711,13 +522,38 @@ public ContextStorageMode contextStorageMode() { public static native void dumpContext(); - /** - * Resets the cached ThreadContext for the current storage slot — the calling thread in - * {@link ContextStorageMode#THREAD}, or its current carrier in - * {@link ContextStorageMode#CARRIER}. The next call to {@link #getThreadContext()} - * or any {@code setContext} overload will re-create it with fresh OTEL TLS buffers. - */ - public void resetThreadContext() { - tlsContextStorage.remove(); + // ---- Test-only reads of the current thread's OTEP record ---------------------------------- + // Each resolves the current carrier's record directly (like the write primitives above) with + // no cached buffer and no per-thread Java object; introspection/test use only. + + /** Test-only: the current thread's span ID from the OTEP record. */ + long testGetSpanId() { + return testGetSpanId0(); } + + /** Test-only: the current thread's local root span ID from the OTEP record. */ + long testGetRootSpanId() { + return testGetRootSpanId0(); + } + + /** Test-only: the current thread's trace ID as a 32-char lowercase hex string. */ + String testReadTraceId() { + return testReadTraceId0(); + } + + /** Test-only: the current thread's custom attribute value at {@code slot}, or null if unset. */ + String testReadContextAttribute(int slot) { + return testReadContextAttribute0(slot); + } + + /** Test-only: whether the current thread's OTEP record is currently valid (published). */ + boolean testIsContextValid() { + return testIsContextValid0(); + } + + private static native long testGetSpanId0(); + private static native long testGetRootSpanId0(); + private static native String testReadTraceId0(); + private static native String testReadContextAttribute0(int slot); + private static native boolean testIsContextValid0(); } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java index 2110ac65c7..3ed5dbc191 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java @@ -235,7 +235,7 @@ public ProcessContext readProcessContext() { * element is null, the entire process context publish is * skipped - no context is published (a warning is logged) and * no exception is thrown. Order must match the indices used - * with {@link ThreadContext#setContextAttribute(int, String)}. + * with {@link JavaProfiler#setContextValue(int, CharSequence)}. * * @throws NullPointerException if {@code attributeKeys} is null * diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java deleted file mode 100644 index 1bc399f9c6..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.util.Locale; - -/** - * Factory for the thread-local that backs {@link ThreadContext} storage, and the - * home of the context-storage mode selection. - * - *

Why this exists: the OTEP record a {@link ThreadContext} writes to is embedded in - * the carrier's native {@code ProfiledThread}, and that carrier's record is what - * the sampler (an async-signal handler bound to the carrier OS thread) reads. A plain - * {@link ThreadLocal} keys the {@code ThreadContext} — and therefore its - * {@code DirectByteBuffer} conduit — by the virtual thread, pinning it to - * whichever carrier was mounted at first use. That is wrong once the virtual thread - * migrates (writes land on the old carrier, so a sampler on the new carrier sees stale - * or empty context) and unsafe once the old carrier's OS thread exits (the record is - * freed while the buffer keeps being written — a use-after-free that can corrupt - * JVM-owned native memory). See {@code ThreadContext} and the design note. - * - *

{@link ContextStorageMode#CARRIER} scopes storage to the carrier via - * {@code jdk.internal.misc.CarrierThreadLocal} (JDK 21+), whose {@code get()/set()/remove()} - * operate on the current carrier's map even when called from a mounted virtual thread. A - * mounted virtual thread then always resolves to its current carrier's record, which is - * exactly what the sampler reads; storage lifetime matches the native record's lifetime, - * so the dangling-buffer window is eliminated. - * - *

{@code CarrierThreadLocal} lives in a non-exported package, so {@link ContextStorageMode#CARRIER} - * needs {@code --add-exports java.base/jdk.internal.misc=ALL-UNNAMED} at runtime (a - * {@code -javaagent} can grant this via {@code Instrumentation.redefineModule}). When the - * type is missing (older JDK) or inaccessible (export not granted), we degrade to - * {@link ContextStorageMode#THREAD} — today's plain {@code ThreadLocal} behavior — never failing hard. - * - *

The instance is held as a {@link ThreadLocal} (the supertype, available on the Java 8 - * baseline) and constructed reflectively, so calls dispatch virtually to the carrier-scoped - * overrides with no per-call reflection and no compile-time dependency on the internal type. - */ -final class OtelContextStorage { - - /** - * Selector system property: {@code auto} (default) | {@code carrier} | {@code thread}. - * Named under {@code ddprof.debug.*} because it is an internal knob, not part of the - * supported configuration surface. - *

    - *
  • {@code auto} — use {@link ContextStorageMode#CARRIER} when available, else - * {@link ContextStorageMode#THREAD} (logged loudly on JDK 21+, where the fallback is - * unsafe under virtual threads).
    • - *
  • {@code carrier} — require carrier scoping; {@link #create()} throws if unavailable.
    • - *
  • {@code thread} — force legacy behavior (disables carrier scoping entirely).
    • - *
- */ - static final String MODE_PROPERTY = "ddprof.debug.context.storage.mode"; - - private static final String INTERNAL_CTL = "jdk.internal.misc.CarrierThreadLocal"; - - private OtelContextStorage() {} - - /** - * The scope of a storage instance returned by {@link #create()} — a property of the - * instance itself (its concrete type), so there is no shared mutable state to leak - * between callers. - */ - static ContextStorageMode modeOf(ThreadLocal storage) { - return storage != null && INTERNAL_CTL.equals(storage.getClass().getName()) - ? ContextStorageMode.CARRIER : ContextStorageMode.THREAD; - } - - /** - * Build the backing thread-local according to {@link #MODE_PROPERTY}. - * - *
    - *
  • {@code thread} — always plain thread-scoped storage (legacy). Never throws.
    • - *
  • {@code carrier} — carrier scoping is required: throws if - * {@code CarrierThreadLocal} is not accessible, rather than silently reintroducing - * the virtual-thread-pinned storage this class exists to remove.
    • - *
  • {@code auto} (default) — carrier scoping when available; otherwise falls back to - * thread-scoped storage. The fallback is expected on JDK < 21 (silent); on a - * Loom-capable JVM it is logged loudly because it is the pre-fix behavior and is - * unsafe if virtual threads carry context.
    • - *
- * - *

Rationale for not failing hard by default: at profiler init we only know the JVM is - * Loom-capable, not whether virtual threads will actually route context here, and the - * runtime export that carrier scoping needs is granted by the agent — which may land after - * this library. Failing hard by default would break profiler startup for every JDK 21+ - * deployment (including non-Loom apps) until that grant is in place. Callers that can - * guarantee the export (and know they use Loom) should opt into {@code carrier} to get - * fail-fast behavior. - */ - static ThreadLocal create() { - // Resolved once per JavaProfiler instance — this is the tlsContextStorage field - // initializer, not a per-thread path. The per-thread get-or-init in JavaProfiler - // (currentContext) never reads this property, so parsing it here is not a hot path. - // Locale.ROOT: the values are ASCII keywords, so lower-casing must be locale-independent - // (a default-locale toLowerCase() maps "CARRIER" to "carrıer" under tr_TR, breaking the match). - String requested = System.getProperty(MODE_PROPERTY, "auto").trim().toLowerCase(Locale.ROOT); - boolean forceThread = "thread".equals(requested); - boolean requireCarrier = "carrier".equals(requested); - - if (forceThread) { - return new ThreadLocal<>(); - } - - // "auto" or "carrier": prefer carrier scoping when the internal type is reachable. - ThreadLocal carrier = tryCreateCarrierLocal(); - if (carrier != null) { - return carrier; - } - - // Carrier scoping unavailable (JDK < 21, or the jdk.internal.misc export not granted). - if (requireCarrier) { - throw new IllegalStateException("ddprof: -D" + MODE_PROPERTY + "=carrier requires " - + INTERNAL_CTL + ", which is not accessible. On JDK 21+ add " - + "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED (a -javaagent can grant " - + "this via Instrumentation.redefineModule). Falling back to thread-scoped storage " - + "would re-expose the virtual-thread context use-after-free; set -D" - + MODE_PROPERTY + "=thread to explicitly accept legacy thread-scoped storage."); - } - - // auto: degrade so profiling still loads, but be loud on Loom-capable JVMs where the - // fallback can misattribute or corrupt context under virtual threads. - if (Platform.isJavaVersionAtLeast(21)) { - System.out.println("[WARN] ddprof: carrier-scoped OTEL context storage is unavailable on a " - + "JDK 21+ JVM (" + INTERNAL_CTL + " not accessible); falling back to thread-scoped " - + "storage. Under virtual threads this can misattribute context and, if a carrier " - + "thread exits, corrupt native memory. Add " - + "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED, set -D" + MODE_PROPERTY - + "=carrier to fail fast instead, or -D" + MODE_PROPERTY + "=thread to silence this."); - } - return new ThreadLocal<>(); - } - - /** - * Returns a {@code CarrierThreadLocal} (as its {@link ThreadLocal} supertype) or - * {@code null} if the type is absent (JDK < 21) or not accessible (export not granted). - */ - @SuppressWarnings("unchecked") - private static ThreadLocal tryCreateCarrierLocal() { - try { - Class ctl = Class.forName(INTERNAL_CTL); - // Public no-arg constructor of a public type; the newInstance access check - // is what fails (IllegalAccessException) when the package is not exported. - Object instance = ctl.getConstructor().newInstance(); - return (ThreadLocal) instance; - } catch (Throwable t) { - // ClassNotFoundException (JDK < 21), IllegalAccessException (no --add-exports), - // or any other reflective failure — degrade silently to thread scoping. - return null; - } - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java deleted file mode 100644 index cf02c454a6..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler; - -import java.util.Arrays; - -/** - * Per-thread stack of {@link ThreadContext} snapshots for nested scopes. - * - *

Provides bulk save/restore of the full OTEP record + sidecar state via one memcpy per - * transition. Not thread-safe: a single stack instance must be accessed only from its - * owning thread. - * - *

Storage is tiered to keep shallow nesting allocation-free: - *

    - *
  • Depths 0 .. {@value #FAST_DEPTH}-1: one contiguous byte[] allocated eagerly.
    • - *
  • Depths {@value #FAST_DEPTH} and beyond: lazily allocated {@value #CHUNK_DEPTH}-slot - * chunks, each a single byte[]. Chunks are allocated once per depth band and reused.
    • - *
- * - * @deprecated DirectByteBuffer snapshot/restore path; unused by dd-trace-java (re-derives - * context). Removed in phase 3. - */ -@Deprecated -public final class ScopeStack { - private static final int FAST_DEPTH = 6; - private static final int CHUNK_DEPTH = 12; - private static final int SLOT_SIZE = ThreadContext.SNAPSHOT_SIZE; - - private final byte[] fast = new byte[FAST_DEPTH * SLOT_SIZE]; - // chunks[i] covers depths [FAST_DEPTH + i*CHUNK_DEPTH .. FAST_DEPTH + (i+1)*CHUNK_DEPTH). - private byte[][] chunks; - private int depth; - - public void enter(ThreadContext ctx) { - int d = depth; - ctx.snapshot(bufferFor(d), offsetFor(d)); - depth = d + 1; - } - - public void exit(ThreadContext ctx) { - int d = depth - 1; - if (d < 0) { - throw new IllegalStateException("ScopeStack underflow"); - } - ctx.restore(bufferFor(d), offsetFor(d)); - depth = d; - } - - /** Current nesting depth (number of outstanding {@link #enter} calls). */ - public int depth() { - return depth; - } - - private byte[] bufferFor(int d) { - if (d < FAST_DEPTH) { - return fast; - } - // chunkFor is idempotent: if this depth was previously populated (via a matching enter), - // it returns the existing chunk without allocating. - return chunkFor((d - FAST_DEPTH) / CHUNK_DEPTH); - } - - private static int offsetFor(int d) { - int slot = d < FAST_DEPTH ? d : (d - FAST_DEPTH) % CHUNK_DEPTH; - return slot * SLOT_SIZE; - } - - private byte[] chunkFor(int idx) { - byte[][] cs = chunks; - if (cs == null) { - cs = new byte[4][]; - chunks = cs; - } else if (idx >= cs.length) { - int newLen = cs.length; - while (newLen <= idx) { - newLen <<= 1; - } - cs = Arrays.copyOf(cs, newLen); - chunks = cs; - } - byte[] c = cs[idx]; - if (c == null) { - c = new byte[CHUNK_DEPTH * SLOT_SIZE]; - cs[idx] = c; - } - return c; - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java deleted file mode 100644 index 7c8af9bdff..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java +++ /dev/null @@ -1,644 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.Objects; - -/** - * Thread-local context for trace/span identification. - * - *

Uses OTEP #4947 TLS record for all context storage. - * Context is written directly to the OTEP record via DirectByteBuffer - * for minimal overhead. Only little-endian platforms are supported. - * - * @deprecated DirectByteBuffer context path; superseded by - * {@link JavaProfiler#setTraceContext} (all-native). Removed in phase 3. - */ -@Deprecated -public final class ThreadContext { - static final int MAX_CUSTOM_SLOTS = 10; - // Max UTF-8 byte length for a custom attribute value. Matches the 1-byte length - // field in the OTEP attrs_data entry header. Enforced up front in setContextAttribute - // so replaceOtepAttribute can assume the input always fits. - private static final int MAX_VALUE_BYTES = 255; - private static final int OTEL_MAX_RECORD_SIZE = 640; - private static final int SIDECAR_SIZE = MAX_CUSTOM_SLOTS * Integer.BYTES + Long.BYTES; // 48 - // Package-private so ScopeStack can size its byte[] scratch. - static final int SNAPSHOT_SIZE = OTEL_MAX_RECORD_SIZE + SIDECAR_SIZE; // 688 - private static final int LRS_OTEP_KEY_INDEX = 0; - // LRS is always a fixed 16-hex-char value in attrs_data (zero-padded u64). - // The entry header is 2 bytes (key_index + length), giving 18 bytes total. - private static final int LRS_FIXED_VALUE_LEN = 16; - private static final int LRS_ENTRY_SIZE = 2 + LRS_FIXED_VALUE_LEN; - private static final byte[] HEX_DIGITS = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; - - private static final BufferWriter BUFFER_WRITER = new BufferWriter(); - - // ---- Per-thread bounded direct-mapped cache for attribute values ---- - // Stores {int encoding, byte[] utf8Bytes} per entry. - // - encoding: Dictionary constant pool ID for DD JFR sidecar - // - utf8Bytes: UTF-8 value for OTEP attrs_data (external profilers) - // - // Instance (not static) so that only the owning thread ever reads or writes - // the cache arrays — no cross-thread races, no memory barriers needed. - // Collision evicts the old entry; a miss triggers one JNI registerConstant0() call. - // The Dictionary (contextValueMap) is never cleared, so encodings remain valid - // for the JVM lifetime. On profiler restart the ThreadContext instance is recreated, - // clearing the cache; the first miss per value pays one JNI call to re-populate. - private static final int CACHE_SIZE = 256; - private static final int CACHE_MASK = CACHE_SIZE - 1; - - // Attribute value cache: String value → {int encoding, byte[] utf8} - // Keyed by value string (not by keyIndex) — same string always maps to - // same encoding regardless of key slot. - private final String[] attrCacheKeys = new String[CACHE_SIZE]; - private final int[] attrCacheEncodings = new int[CACHE_SIZE]; - private final byte[][] attrCacheBytes = new byte[CACHE_SIZE][]; - - // OTEP record field offsets (from packed struct) - private final int validOffset; - private final int traceIdOffset; - private final int spanIdOffset; - private final int attrsDataSizeOffset; - private final int attrsDataOffset; - private final int maxAttrsDataSize; - private final int lrsOffset; // localRootSpanId offset in the unified buffer - // Base offset of the tag-encoding sidecar within the unified buffer. Every tag slot i - // lives at ctxBuffer[tagEncodingsOffset + i * Integer.BYTES]. Equal to OTEL_MAX_RECORD_SIZE. - private static final int TAG_ENCODINGS_OFFSET = OTEL_MAX_RECORD_SIZE; - - // Single buffer spanning [OTEP record | tag_encodings | LRS] — 688 bytes contiguous. - // Used for per-field access AND for bulk snapshot/restore memcpy. Position state is - // thread-confined to snapshot/restore, which reset it before each bulk op. - private final ByteBuffer ctxBuffer; - - /** - * Creates a ThreadContext from the single DirectByteBuffer returned by native initializeContextTLS0. - * - * @param ctxBuffer 688-byte unified buffer spanning record + tag_encodings + LRS - * @param metadata array with absolute offsets [VALID, TRACE_ID, SPAN_ID, - * ATTRS_DATA_SIZE, ATTRS_DATA, LRS] - */ - public ThreadContext(ByteBuffer ctxBuffer, long[] metadata) { - // Uses native order for uint16_t attrs_data_size (read by C as native uint16_t). - // trace_id/span_id are uint8_t[] arrays requiring big-endian — handled via Long.reverseBytes() - // in setContextDirect(). Only little-endian platforms are supported. - this.ctxBuffer = ctxBuffer.order(ByteOrder.nativeOrder()); - this.validOffset = (int) metadata[0]; - this.traceIdOffset = (int) metadata[1]; - this.spanIdOffset = (int) metadata[2]; - this.attrsDataSizeOffset = (int) metadata[3]; - this.attrsDataOffset = (int) metadata[4]; - this.maxAttrsDataSize = OTEL_MAX_RECORD_SIZE - this.attrsDataOffset; - this.lrsOffset = (int) metadata[5]; - if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { - throw new UnsupportedOperationException( - "ByteBuffer context path requires little-endian platform"); - } - // Zero sidecar + record to prevent stale encodings from a previous profiler session. - // The native ProfiledThread survives across sessions, so the buffer may hold - // old tag encodings and the record may hold old attrs_data. - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - this.ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - this.ctxBuffer.putLong(this.lrsOffset, 0); - this.ctxBuffer.put(this.validOffset, (byte) 0); - // Pre-initialize the fixed-size LRS entry at attrs_data[0..LRS_ENTRY_SIZE-1]: - // key_index=0, length=16, value=16 zero hex bytes. - // The entry is always present; updates overwrite only the 16 value bytes. - this.ctxBuffer.put(this.attrsDataOffset, (byte) LRS_OTEP_KEY_INDEX); - this.ctxBuffer.put(this.attrsDataOffset + 1, (byte) LRS_FIXED_VALUE_LEN); - for (int i = 0; i < LRS_FIXED_VALUE_LEN; i++) { - this.ctxBuffer.put(this.attrsDataOffset + 2 + i, (byte) '0'); - } - this.ctxBuffer.putShort(this.attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - } - - /** - * Returns the current span ID. - * Reads directly from the OTEP record buffer (big-endian bytes → native long). - */ - public long getSpanId() { - return Long.reverseBytes(ctxBuffer.getLong(spanIdOffset)); - } - - /** - * Returns the current local root span ID. - * Reads directly from the LRS region of ctxBuffer (native long). - */ - public long getRootSpanId() { - return ctxBuffer.getLong(lrsOffset); - } - - /** - * Sets trace context with 2-arg legacy API. - * Maps rootSpanId to localRootSpanId. Uses spanId as traceIdLow so that - * the OTEL clear check (all-zero) only fires when spanId is actually 0. - * Note: this produces a synthetic trace_id of [0, spanId] in the OTEP record. - * External OTEP readers will see this as a real W3C trace ID. Callers needing - * correct OTEP interop must use the 4-arg {@link #put(long, long, long, long)}. - * - *

Parameter mapping to the 4-arg API: {@code spanId} → {@code spanId}, - * {@code rootSpanId} → {@code localRootSpanId} (first argument). - * - * @deprecated Use {@link #put(long, long, long, long)} instead. - */ - @Deprecated - public long put(long spanId, long rootSpanId) { - put(rootSpanId, spanId, 0, spanId); - return 0; // Return type kept for ABI compatibility; value carries no meaning. - } - - /** - * Sets trace context with full 128-bit W3C trace ID and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId The span ID - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - */ - public void put(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - setContextDirect(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Clears a custom attribute: zeros the sidecar encoding and removes it from OTEP attrs_data. - */ - public void clearContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return; - } - int otepKeyIndex = keyIndex + 1; - detach(); - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - removeOtepAttribute(otepKeyIndex); - attach(); - } - - public void copyCustoms(int[] value) { - int len = Math.min(value.length, MAX_CUSTOM_SLOTS); - for (int i = 0; i < len; i++) { - value[i] = ctxBuffer.getInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES); - } - } - - /** - * Captures the full record + sidecar state into {@code scratch[offset..offset+SNAPSHOT_SIZE)}. - * Pair with {@link #restore} for nested-scope propagation. - * - *

The detach/memcpy/re-publish pair hides the bulk read from any signal handler going - * through {@code ContextApi::get} — while {@code valid=0}, sidecar reads are gated off. The - * pre-snapshot {@code valid} state is preserved in {@code scratch[offset + validOffset]} so - * {@link #restore} can replay it. If the record was already invalid (e.g. the all-zero clear - * path in {@link #setContextDirect} leaves {@code valid=0} with a stale {@code attrs_data_size} - * / {@code attrs_data}), the live buffer is left invalid after snapshot — re-publishing would - * expose a cleared-but-stale record. - */ - public void snapshot(byte[] scratch, int offset) { - byte priorValid = ctxBuffer.get(validOffset); - detach(); - // Cast to Buffer: ByteBuffer.position(int) only returns ByteBuffer since JDK 9 (covariant - // return). This source is compiled for Java 8 runtimes where the method lives on Buffer. - ((Buffer) ctxBuffer).position(0); - ctxBuffer.get(scratch, offset, SNAPSHOT_SIZE); - // Overwrite the valid byte in scratch (memcpy captured the post-detach 0) with the - // pre-snapshot value. restore() consults this to decide whether to re-attach. - scratch[offset + validOffset] = priorValid; - if (priorValid != 0) { - attach(); - } - } - - /** - * Restores a previously captured state. The detach/memcpy/conditional-attach pair hides the - * memcpy from readers going through {@link #ctxBuffer}'s valid flag ({@code ContextApi::get} - * in native code), which is the sole gate for sidecar reads (see {@code thread.h}). - * - *

The valid byte inside scratch is cleared to 0 for the duration of the memcpy so that - * even if the captured state had {@code valid=1}, the live buffer cannot transiently observe - * {@code valid=1} alongside partially-written fields. The captured value is restored into - * scratch after the memcpy so subsequent snapshot/restore cycles keep working, and - * {@link #attach} re-publishes only when the saved state was itself valid — matching the - * semantics of {@link #snapshot}. - */ - public void restore(byte[] scratch, int offset) { - int validIdx = offset + validOffset; - byte wasValid = scratch[validIdx]; - scratch[validIdx] = 0; - detach(); - ((Buffer) ctxBuffer).position(0); - ctxBuffer.put(scratch, offset, SNAPSHOT_SIZE); - if (wasValid != 0) { - scratch[validIdx] = wasValid; - attach(); - } - } - - /** - * Sets a custom attribute on the current thread's context by string value. - * Uses a per-thread encoding cache to avoid JNI for repeated values - * (the common case). On cache miss, a single JNI call registers the value - * in the native Dictionary; subsequent calls with the same value are - * zero-JNI ByteBuffer writes. - * - *

High-cardinality values are not supported. Each unique value - * permanently occupies one slot in the native Dictionary, which is bounded - * at 65536 entries across all threads for the JVM lifetime. Once exhausted, - * this method returns {@code false} and clears the attribute. Use only - * low-cardinality values (e.g. endpoint names, DB system names). UUIDs, - * request IDs, and other per-request-unique strings will exhaust the - * Dictionary and cause attributes to be silently dropped. - * - *

Value size limit. The UTF-8 encoding of {@code value} must fit in - * {@value #MAX_VALUE_BYTES} bytes (the OTEP attrs_data entry length field is one byte). - * Oversized values are rejected up front — they never reach the Dictionary or attrs_data. - * - * @param keyIndex Index into the registered attribute key map (0-based) - * @param value The string value for this attribute - * @return true if the attribute was set successfully, false if the value is too long, - * the Dictionary is full, attrs_data overflows, or keyIndex is out of range - */ - public boolean setContextAttribute(int keyIndex, String value) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS || value == null) { - return false; - } - return setContextAttributeDirect(keyIndex, value); - } - - /** - * Writes both the sidecar encoding (DD signal handler) and OTEP attrs_data - * UTF-8 value (external profilers) via ByteBuffer. - */ - private boolean setContextAttributeDirect(int keyIndex, String value) { - - // Resolve encoding + UTF-8 bytes from per-thread cache - int slot = value.hashCode() & CACHE_MASK; - int encoding; - byte[] utf8; - if (value.equals(attrCacheKeys[slot])) { - // Cache hit — the value was previously validated and cached; no re-check needed. - encoding = attrCacheEncodings[slot]; - utf8 = attrCacheBytes[slot]; - } else { - // Cache miss: encode UTF-8 and validate size BEFORE touching the Dictionary. - // Rejecting here avoids an orphan Dictionary entry (the native Dictionary is - // write-only for the JVM lifetime and cannot be undone). - utf8 = value.getBytes(StandardCharsets.UTF_8); - if (utf8.length > MAX_VALUE_BYTES) { - return false; - } - encoding = registerConstant0(value); - if (encoding < 0) { - // Dictionary full: clear sidecar AND remove the OTEP attrs_data entry - // so both views stay consistent (both report no value for this key). - clearContextAttribute(keyIndex); - return false; - } - attrCacheEncodings[slot] = encoding; - attrCacheBytes[slot] = utf8; - attrCacheKeys[slot] = value; - } - - // Write both sidecar and OTEP attrs_data inside the detach/attach window - // so a signal handler never sees a new sidecar encoding alongside old attrs_data. - detach(); - boolean written = writeSlot(keyIndex, encoding, utf8); - attach(); - return written; - } - - /** - * Writes one slot's sidecar encoding and OTEP attrs_data value. The caller must already hold - * the detach/attach window. On attrs_data overflow the old entry was compacted out and the new - * one couldn't fit, so the sidecar is zeroed to keep both views agreeing there is no value. - * - * @return true if the value was written; false on attrs_data overflow (sidecar left zeroed) - */ - private boolean writeSlot(int keyIndex, int encoding, byte[] utf8) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, encoding); - if (!replaceOtepAttribute(keyIndex + 1, utf8)) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - return false; - } - return true; - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes in a - * single detach/attach window, without Dictionary lookups or per-thread cache access. - * - *

Both arrays are indexed by key slot, matching the layout produced by - * {@link #copyCustoms(int[])}. For each slot {@code i} with {@code constantIds[i] > 0}, the - * sidecar encoding (DD signal handler) and the OTEP attrs_data value (external profilers) are - * written together; slots with {@code constantIds[i] <= 0} are left untouched. Performing all - * writes inside one detach/attach window means a signal handler never observes a partially - * re-applied set. - * - *

Intended for the reapply-app-context hot path: the caller already holds the constant IDs - * (from {@link #copyCustoms(int[])}) and the UTF-8 bytes (from the original Strings), so this - * does no String allocation, hashing, or cache lookup. The record is re-published only if it - * was valid before the call, so a cleared (span-less) record is not resurrected. - * - *

Caller contract. {@code constantIds[i]} must be an ID previously returned for the - * same value via {@link #copyCustoms(int[])} within the current profiler session, and - * {@code utf8[i]} must be that value's UTF-8 bytes. The (id, bytes) pairing is not verifiable - * here; a mismatch silently diverges the sidecar and attrs_data views. - * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most - * {@value #MAX_VALUE_BYTES} bytes for every slot whose constantId {@code > 0} - * @return true if every slot with {@code constantId > 0} was written successfully; false if - * the record was not valid before the call (nothing is published), or if any slot - * overflowed {@code attrs_data} (that slot's sidecar is zeroed). Note: a {@code false} - * return due to {@code attrs_data} overflow does not mean the record is - * unmodified — slots processed before the overflowed one are durably written. - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} (where {@code constantIds[i] > 0}) is null - * @throws IllegalArgumentException if the arrays have different lengths, - * {@code constantIds.length > MAX_CUSTOM_SLOTS}, or any - * active {@code utf8[i].length > MAX_VALUE_BYTES} - */ - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - Objects.requireNonNull(constantIds, "constantIds"); - Objects.requireNonNull(utf8, "utf8"); - if (constantIds.length != utf8.length) { - throw new IllegalArgumentException("constantIds and utf8 must have the same length"); - } - if (constantIds.length > MAX_CUSTOM_SLOTS) { - throw new IllegalArgumentException("constantIds.length exceeds MAX_CUSTOM_SLOTS"); - } - int len = constantIds.length; - // Validate active slots before touching the buffer so a bad input never leaves - // the record detached (valid=0) after an exception unwinds past attach(). - for (int i = 0; i < len; i++) { - if (constantIds[i] > 0) { - if (utf8[i] == null) { - throw new NullPointerException("utf8[" + i + "]"); - } - if (utf8[i].length > MAX_VALUE_BYTES) { - throw new IllegalArgumentException("utf8[" + i + "].length exceeds MAX_VALUE_BYTES"); - } - } - } - // Never resurrect a cleared (span-less) record: valid=0 means no reader can observe - // what we write, and re-publishing would expose a record with no trace/span context. - if (ctxBuffer.get(validOffset) == 0) { - return false; - } - detach(); - boolean allWritten = true; - for (int i = 0; i < len; i++) { - int constantId = constantIds[i]; - if (constantId <= 0) { - continue; - } - if (!writeSlot(i, constantId, utf8[i])) { - allWritten = false; - } - } - attach(); - return allWritten; - } - - /** - * Write context directly to the OTEP record via ByteBuffer. - * trace_id and span_id are OTEP big-endian byte arrays — Long.reverseBytes() - * converts from native LE to big-endian. - */ - private void setContextDirect(long localRootSpanId, long spanId, long trHi, long trLo) { - detach(); - - if (trHi == 0 && trLo == 0 && spanId == 0) { - clearContextDirect(); - return; - } - - // Write trace_id (big-endian) + span_id (big-endian) - ctxBuffer.putLong(traceIdOffset, Long.reverseBytes(trHi)); - ctxBuffer.putLong(traceIdOffset + 8, Long.reverseBytes(trLo)); - ctxBuffer.putLong(spanIdOffset, Long.reverseBytes(spanId)); - - // Reset custom attribute state so the previous span's values don't leak - // into this span. Callers set attributes again via setContextAttribute(). - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - // offset into ctxBuffer for tag-encoding slot i - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - // Reset attrs_data_size to contain only the fixed LRS entry, discarding - // any custom attribute entries written during the previous span. - ctxBuffer.putShort(attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - - // Update LRS sidecar and OTEP attrs_data inside the detach/attach window so a - // signal handler never sees the new LRS with old trace/span IDs. - ctxBuffer.putLong(lrsOffset, localRootSpanId); - writeLrsHex(localRootSpanId); - - attach(); - } - - // ---- LRS helpers ---- - - /** - * Zeros trace/span IDs, sidecar encodings, and LRS. Called between detach() and the - * return in the all-zero path; valid stays 0 (no attach) so no reader can see attrs_data. - * attrs_data_size is not reset here; the next non-zero setContext call will reset it - * before attach(). This is safe because valid remains 0 after clear, so no reader will - * observe the stale attrs_data_size. - * - *

External OTEP readers see valid=0 and skip the record — they cannot distinguish - * "cleared" from "being mutated". This is intentional: without also resetting - * attrs_data_size here, publishing valid=1 with a stale attrs_data_size would expose - * a partially-valid record. The cleared state is effectively invisible to external - * readers until the next non-zero setContext call publishes it. - */ - private void clearContextDirect() { - ctxBuffer.putLong(traceIdOffset, 0); - ctxBuffer.putLong(traceIdOffset + 8, 0); - ctxBuffer.putLong(spanIdOffset, 0); - writeLrsHex(0); - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - ctxBuffer.putLong(lrsOffset, 0); - } - - /** - * Overwrite the 16 hex value bytes of the fixed LRS entry in-place. - * The entry header (key_index=0, length=16) is pre-initialized and never touched. - * Called between detach() and attach(); no allocation. - */ - private void writeLrsHex(long val) { - int base = attrsDataOffset + 2; // skip key_index byte + length byte - for (int i = 15; i >= 0; i--) { - ctxBuffer.put(base + i, HEX_DIGITS[(int)(val & 0xF)]); - val >>>= 4; - } - } - - // ---- OTEP record helpers (called between detach/attach) ---- - - /** - * Invalidates the record (sets valid=0). Typically followed by attach(), but the - * clear path intentionally leaves the record invalid without calling attach(). - */ - private void detach() { - ctxBuffer.put(validOffset, (byte) 0); - BUFFER_WRITER.storeFence(); - } - - /** Validate record. */ - private void attach() { - // storeFence ensures all record writes are visible before valid=1. - // The TLS pointer (otel_thread_ctx_v1) is permanent and never - // written here; external profilers rely solely on the valid flag. - BUFFER_WRITER.storeFence(); - // Plain put is sufficient: signal handlers run on the same hardware thread, - // so they observe stores in program order — no volatile needed for same-thread - // visibility. The preceding storeFence() provides the release barrier. - ctxBuffer.put(validOffset, (byte) 1); - } - - /** - * Replace or insert an attribute in attrs_data. Record must be detached. - * Writes the pre-encoded UTF-8 bytes into the record. - * - *

Caller contract: {@code utf8.length <= MAX_VALUE_BYTES}, enforced at the public - * entry point in {@link #setContextAttributeDirect}. - */ - private boolean replaceOtepAttribute(int otepKeyIndex, byte[] utf8) { - int currentSize = compactOtepAttribute(otepKeyIndex); - int valueLen = utf8.length; - int entrySize = 2 + valueLen; - if (currentSize + entrySize <= maxAttrsDataSize) { - int base = attrsDataOffset + currentSize; - ctxBuffer.put(base, (byte) otepKeyIndex); - ctxBuffer.put(base + 1, (byte) valueLen); - for (int i = 0; i < valueLen; i++) { - ctxBuffer.put(base + 2 + i, utf8[i]); - } - currentSize += entrySize; - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return true; - } - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return false; - } - - /** Remove an attribute from attrs_data by compacting. Record must be detached. */ - private void removeOtepAttribute(int otepKeyIndex) { - int currentSize = compactOtepAttribute(otepKeyIndex); - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - } - - /** - * Scan attrs_data and compact out the entry with the given key_index. - * Returns the new attrs_data size after compaction. - * - *

{@code otepKeyIndex} is always {@code keyIndex + 1} for user attributes, - * so it is never 0. Index 0 is reserved for the fixed LRS entry. - */ - private int compactOtepAttribute(int otepKeyIndex) { - int currentSize = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int readPos = 0; - int writePos = 0; - boolean found = false; - while (readPos + 2 <= currentSize) { - int k = ctxBuffer.get(attrsDataOffset + readPos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + readPos + 1) & 0xFF; - if (readPos + 2 + len > currentSize) { currentSize = writePos; break; } - if (k == otepKeyIndex) { - found = true; - readPos += 2 + len; - } else { - if (found && writePos < readPos) { - for (int i = 0; i < 2 + len; i++) { - ctxBuffer.put(attrsDataOffset + writePos + i, - ctxBuffer.get(attrsDataOffset + readPos + i)); - } - } - writePos += 2 + len; - readPos += 2 + len; - } - } - return found ? writePos : currentSize; - } - - /** - * Reads a custom attribute value by key index by scanning {@code attrs_data}. - * - *

Test-only. The only caller is {@code TagContextTest}, which uses it via - * {@link JavaProfiler#getThreadContext()} to verify that writes to the OTEP record are - * observable after set / clear / span-reset cycles. No production path — neither the DD - * signal handler nor the OTEL eBPF reader — ever calls this method: the DD handler reads - * sidecar encoding IDs and the OTEL reader parses {@code attrs_data} directly from native - * memory. The per-call {@code byte[]} / {@code String} allocation is therefore acceptable; - * do not introduce a readback cache unless a real production consumer appears. - * - * @param keyIndex 0-based user key index (same as passed to setContextAttribute) - * @return the attribute value string, or null if not set - */ - public String readContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return null; - } - // valid=0 → record was detached or never published. No attrs_data to trust. - if (ctxBuffer.get(validOffset) == 0) { - return null; - } - int otepKeyIndex = keyIndex + 1; - int size = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int pos = 0; - while (pos + 2 <= size) { - int k = ctxBuffer.get(attrsDataOffset + pos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + pos + 1) & 0xFF; - if (pos + 2 + len > size) { - break; - } - if (k == otepKeyIndex) { - byte[] bytes = new byte[len]; - for (int i = 0; i < len; i++) { - bytes[i] = ctxBuffer.get(attrsDataOffset + pos + 2 + i); - } - return new String(bytes, StandardCharsets.UTF_8); - } - pos += 2 + len; - } - return null; - } - - /** - * Reads the trace ID from the OTEP record as a 32-char lowercase hex string. - * The trace ID is stored big-endian; this method returns it as-is. - * Intended for tests only. - */ - public String readTraceId() { - StringBuilder sb = new StringBuilder(32); - for (int i = 0; i < 16; i++) { - int b = ctxBuffer.get(traceIdOffset + i) & 0xFF; - sb.append((char) HEX_DIGITS[b >> 4]); - sb.append((char) HEX_DIGITS[b & 0xF]); - } - return sb.toString(); - } - - // Package-private (was private) so the all-native path's ContextValueCache can reuse the same - // process-global Dictionary registration. Relocates when ThreadContext is removed (phase 3). - static native int registerConstant0(String value); -} diff --git a/ddprof-stresstest/README.md b/ddprof-stresstest/README.md index f22caff0eb..c8b9db566f 100644 --- a/ddprof-stresstest/README.md +++ b/ddprof-stresstest/README.md @@ -92,18 +92,6 @@ Measure end-to-end profiling engine performance including signal handlers, stack **Documentation**: See `doc/architecture/CallTraceStorage.md` for detailed CallTraceStorage architecture, benchmark results analysis, and optimization recommendations. -### ThreadContext Benchmarks - -Compare performance of JNI-based native vs DirectByteBuffer-based Java implementations for thread context storage. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ThreadContextBenchmark -``` - -Tests various thread counts to measure both single-threaded overhead and multi-threaded contention. - ### ThreadFilter Benchmarks Measure thread filtering performance and overhead. @@ -264,7 +252,7 @@ ddprof-stresstest/ │ └── scenarios/ │ ├── throughput/ # Raw performance benchmarks │ │ ├── ProfilerThroughput* # End-to-end profiling engine suite -│ │ ├── ThreadContext* # ThreadContext benchmarks +│ │ ├── ContextCombinedBenchmark # All-native context write API perf guard │ │ └── ThreadFilter* # ThreadFilter benchmarks │ └── counters/ # Feature-specific benchmarks │ ├── TracedParallelWork # Distributed tracing overhead diff --git a/ddprof-stresstest/build.gradle.kts b/ddprof-stresstest/build.gradle.kts index 8119efb614..550baee43b 100644 --- a/ddprof-stresstest/build.gradle.kts +++ b/ddprof-stresstest/build.gradle.kts @@ -83,7 +83,7 @@ dependencies { // provides the (relocated) runtime classes and intercepts @Trace. "chaosCompileOnly"(libs.dd.trace.api) // ddprof-lib public API: compile-only; the patched dd-java-agent provides the - // classes at runtime for antagonists that call JavaProfiler/ThreadContext directly. + // classes at runtime for antagonists that call JavaProfiler directly. "chaosCompileOnly"(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug"))) } diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md index 2c72a4eba0..74e5f19d4b 100644 --- a/ddprof-stresstest/src/chaos/README.md +++ b/ddprof-stresstest/src/chaos/README.md @@ -14,8 +14,8 @@ the runner script. | `vthread-churn` | virtual thread mount/unmount, carrier-thread context, `ProfiledThread` | | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | | `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | -| `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | -| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free | +| `trace-context` | `setTraceContext`/`clearTraceContext` racing signals, span ID propagation | +| `reapply-context-value` | concurrent per-slot `setContextValue`/`clearContextValue` churn interleaved with span activation, racing the all-native record's write/read window | ## Deferred diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 65b29c7c6e..7e846c02d1 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -74,8 +74,6 @@ private static Antagonist create(String name) { return new AllocStormAntagonist(); case "vthread-churn": return new VirtualThreadChurnAntagonist(); - case "vthread-context-cascade": - return new VirtualThreadContextCascadeAntagonist(); case "classloader-churn": return new ClassLoaderChurnAntagonist(); case "trace-context": @@ -94,8 +92,8 @@ private static Antagonist create(String name) { return new WeakRefWaveAntagonist(); case "dump-storm": return new DumpStormAntagonist(); - case "reapply-context": - return new ReapplyContextAntagonist(); + case "reapply-context-value": + return new ReapplyContextValueAntagonist(); // Deferred: dlopen-churn (needs per-arch dummy .so built in CI prep). default: throw new IllegalArgumentException("unknown antagonist: " + name); diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java deleted file mode 100644 index e853fa3c8b..0000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import com.datadoghq.profiler.ContextSetter; -import com.datadoghq.profiler.JavaProfiler; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Arrays; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -/** - * Drives the reapply-by-id-and-bytes hot path continuously from multiple threads while the - * profiler's wall-clock signal fires, racing the {@code detach}/{@code attach} window. - * - *

Each worker thread loops: span-activate (wiping slots) → reapply snapshot. This mirrors - * dd-trace-java's {@code reapplyAppContext} pattern and exercises the single-window invariant - * (no partial publish visible to a signal handler) under high thread count and signal pressure. - * - *

The only expected failure signal is a JVM crash. An unexpected {@code false} return from - * reapply (which should never happen since the record is always valid right after - * {@code setContext}) throws {@link IllegalStateException} to surface the bug immediately. - */ -public final class ReapplyContextAntagonist implements Antagonist { - - private static final String[] ATTR_NAMES = { - "http.route", "http.method", "http.status", "db.operation", "rpc.service" - }; - - private static final String[] ROUTES = { - "GET /api/users", - "POST /api/orders", - "GET /api/health", - "PUT /api/users/{id}", - "DELETE /api/sessions" - }; - - private final int workerCount; - private final ExecutorService pool; - private volatile boolean running; - - public ReapplyContextAntagonist() { - this(8); - } - - public ReapplyContextAntagonist(int workerCount) { - this.workerCount = workerCount; - this.pool = - Executors.newFixedThreadPool( - workerCount, - r -> { - Thread t = new Thread(r, "chaos-reapply-context"); - t.setDaemon(true); - return t; - }); - } - - @Override - public String name() { - return "reapply-context"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < workerCount; i++) { - pool.submit(this::workerLoop); - } - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - pool.shutdown(); - try { - pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void workerLoop() { - JavaProfiler profiler; - try { - profiler = JavaProfiler.getInstance(); - } catch (Exception e) { - System.err.println("[chaos] reapply-context: failed to get profiler: " + e); - return; - } - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); - - // Prime the per-thread encoding cache and capture a stable snapshot. - long spanId = Thread.currentThread().getId() + 1; - long localRootSpanId = spanId * 31L; - long traceIdLow = spanId * 6364136223846793005L + 1442695040888963407L; - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - contextSetter.setContextValue(i, ROUTES[i]); - } - int[] constantIds = contextSetter.snapshotTags(); - byte[][] utf8 = new byte[ROUTES.length][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - - while (running) { - // Span activation wipes all custom slots. - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - // Reapply restores them — this is the hot path under test. - if (!contextSetter.setContextValuesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException("reapply failed unexpectedly — record should be valid after setContext"); - } - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java new file mode 100644 index 0000000000..50ec02f77f --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java @@ -0,0 +1,114 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import com.datadoghq.profiler.JavaProfiler; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +/** + * Drives repeated per-slot {@code setContextValue}/{@code clearContextValue} calls continuously + * from multiple threads, interleaved with {@code setTraceContext} span activations, while the + * profiler's wall-clock signal fires — racing the all-native record's write/read window. + * + *

Each worker thread loops: activate a span (resetting custom slots), churn a handful of + * attribute slots with set/clear calls, deactivate. Unlike the legacy DirectByteBuffer conduit + * this antagonist's predecessor exercised, there is no cached per-thread buffer to go stale — the + * only thing under test here is that concurrent single-slot native writes never produce a partial + * or torn record visible to a signal handler. + */ +public final class ReapplyContextValueAntagonist implements Antagonist { + + private static final String[] ROUTES = { + "GET /api/users", + "POST /api/orders", + "GET /api/health", + "PUT /api/users/{id}", + "DELETE /api/sessions" + }; + + private static final int SLOT_COUNT = 5; + + private final int workerCount; + private final ExecutorService pool; + private volatile boolean running; + + public ReapplyContextValueAntagonist() { + this(8); + } + + public ReapplyContextValueAntagonist(int workerCount) { + this.workerCount = workerCount; + this.pool = + Executors.newFixedThreadPool( + workerCount, + r -> { + Thread t = new Thread(r, "chaos-reapply-context-value"); + t.setDaemon(true); + return t; + }); + } + + @Override + public String name() { + return "reapply-context-value"; + } + + @Override + public void start() { + running = true; + for (int i = 0; i < workerCount; i++) { + pool.submit(this::workerLoop); + } + } + + @Override + public void stopGracefully(Duration timeout) { + running = false; + pool.shutdown(); + try { + pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void workerLoop() { + JavaProfiler profiler; + try { + profiler = JavaProfiler.getInstance(); + } catch (Exception e) { + System.err.println("[chaos] reapply-context-value: failed to get profiler: " + e); + return; + } + + long spanId = Thread.currentThread().getId() + 1; + long localRootSpanId = spanId * 31L; + long traceIdLow = spanId * 6364136223846793005L + 1442695040888963407L; + + while (running) { + // Span activation resets all custom slots. + profiler.setTraceContext(localRootSpanId, spanId, 0, traceIdLow, -1, null, -1, null); + for (int i = 0; i < SLOT_COUNT; i++) { + profiler.setContextValue(i, ROUTES[i % ROUTES.length]); + } + for (int i = 0; i < SLOT_COUNT; i += 2) { + profiler.clearContextValue(i); + } + for (int i = 0; i < SLOT_COUNT; i += 2) { + profiler.setContextValue(i, ROUTES[(i + ThreadLocalRandom.current().nextInt(ROUTES.length)) % ROUTES.length]); + } + profiler.clearTraceContext(); + } + } +} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java deleted file mode 100644 index ecc3c951e6..0000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ /dev/null @@ -1,560 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import datadog.trace.api.Trace; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.concurrent.Executor; -import java.util.concurrent.Semaphore; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and - * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. - * - *

Unlike a hand-rolled context simulation, every context write here is driven by the real - * Datadog tracer: {@link #runCascadeNode}/{@link #nestedOp}/{@link #staleRacerBefore}/{@link - * #staleRacerAfter} are {@link Trace}-annotated methods. Under a {@code dd-java-agent}- - * instrumented JVM the tracer intercepts each one, activating a real span on entry ({@code - * JavaProfiler.setContext}) and deactivating it on exit ({@code clearContext}) — exactly what a - * genuinely-instrumented application does. {@code dd-trace-api} is a {@code compileOnly} - * dependency (see {@code TraceContextAntagonist}): at runtime the patched {@code dd-java-agent} - * provides the (relocated) classes and intercepts the annotation, so this antagonist never touches - * {@code com.datadoghq.profiler.*} directly. - * - *

Each cascade node ({@link #runCascadeNode}) runs a handful of nested sub-operations, each - * activating a new child span and sleeping briefly ({@link #forceUnmount}) to force the virtual - * thread off its current carrier before the span's exit (and the next nested entry) write context - * again — the high context-change-rate dimension. It then fans out to child virtual threads while - * its own span is still active, handing the tracer's own async/virtual-thread instrumentation the - * job of propagating that span as parent context onto the next hop (the chain continues on a new - * thread/carrier). On exit, the span's close restores whatever was logically active underneath it. - * - *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived - * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) - * across a rotating set of monitors, forcing the JDK's virtual-thread scheduler to spin up extra - * compensating carrier platform threads while the pins are held. - * - *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} - * fallback (the default on JDK 21+ without {@code --add-exports - * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit - * for a virtual thread's OTEL context (which backs both the span id fields written by {@code - * setContext} and any custom attributes) is cached in a plain {@code ThreadLocal} the first time - * the thread writes context, bound to whichever carrier happened to be mounted at that moment; the - * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} - * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment - * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races - * this cheaply: a nested span's entry writes, {@link #forceUnmount} forces a short real unmount, - * then the span's exit (or the next nested entry) writes again — if the virtual thread was - * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. - * - *

Catching genuine memory reuse (not just misattribution) needs the original carrier - * to actually be torn down first. {@link #driverLoop} runs a duty cycle for the cascade/pin-churn - * side only (a short active burst followed by a quiet phase with no new work, long enough for the - * JDK's default {@code ForkJoinPool} scheduler — 30s keep-alive, see {@code - * java.lang.VirtualThread#createDefaultScheduler()} — to actually reap idle carriers). Stale- - * buffer racers ({@link #runStaleBufferRacer}) don't wait on that: every racer runs on a virtual - * thread bound to a purpose-built {@link #fastCarrierScheduler} — a {@link ThreadPoolExecutor} - * with a 200ms keep-alive — via the package-private {@code VirtualThread(Executor, String, int, - * Runnable)} constructor (JDK 21+; see {@code java.lang.VirtualThread}, requires {@code - * --add-opens java.base/java.lang=ALL-UNNAMED}). {@link #staleRacerLoop} keeps up to {@link - * #MAX_STALE_RACERS} of these racers in flight continuously, independent of the cascade/pin duty - * cycle above — sustained concurrent hammering rather than one batch per cycle, since more - * concurrent carrier churn means more chances for a freed {@code ProfiledThread} slot to actually - * get reused before a stale racer touches it. Each racer writes context once ({@link - * #staleRacerBefore}), sleeps a randomized interval straddling the scheduler's keep-alive ({@link - * #FAST_STALE_RACER_SLEEP_MIN_MILLIS}–{@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}), then writes - * again ({@link #staleRacerAfter}) — varying the wait means a run samples both "carrier likely - * still tearing down" and "carrier long gone, memory possibly already reused" timings, instead of - * only ever probing one fixed instant. - * - *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops the whole - * antagonist on older runtimes where it's absent. But on a JDK where it is present, the - * {@code VirtualThread} constructor above is required, not optional: it's this antagonist's only - * way to reach the stale-carrier UAF without a slow duty-cycle wait, so {@link #start} fails fast - * with an {@link IllegalStateException} (crashing the chaos harness process, which the CI runner - * already treats as a failure) rather than silently losing that coverage if a missing {@code - * --add-opens} flag or a JDK build that changed the constructor's signature makes it - * unreachable. Live virtual thread count is bounded by semaphore permit pools so cascades, - * pinning churn, and stale-buffer racers cannot runaway the JVM. - */ -public final class VirtualThreadContextCascadeAntagonist implements Antagonist { - - private static final Method OF_VIRTUAL = resolveOfVirtual(); - private static final Method BUILDER_START = resolveBuilderStart(); - - private static final int FAN_OUT = 3; - private static final int MAX_DEPTH = 4; - private static final int NESTED_OPS_PER_HOP = 3; - private static final int MAX_LIVE_VTHREADS = 512; - private static final int ROOT_BATCH = 16; - - // Carrier-pinning churn: rotating monitors held across a blocking sleep to force the - // scheduler to spin up (and later reap) compensating carrier platform threads. - private static final int PIN_LOCK_COUNT = 32; - private static final int PIN_BATCH = 24; - private static final int MAX_PIN_VTHREADS = 128; - private static final Object[] PIN_LOCKS = new Object[PIN_LOCK_COUNT]; - - static { - for (int i = 0; i < PIN_LOCK_COUNT; i++) { - PIN_LOCKS[i] = new Object(); - } - } - - // Duty cycle driving genuine carrier teardown: an active burst of cascade/pin-churn work, - // then a quiet phase long enough to clear the JDK's default 30s carrier keep-alive so idle - // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. - private static final long ACTIVE_PHASE_MILLIS = 8_000L; - private static final long QUIET_PHASE_MILLIS = 40_000L; - - // Stale-buffer racers run continuously on their own driver, decoupled from the cascade/pin - // duty cycle above — that cycle only exists to let the JDK default scheduler's 30s keep-alive - // reap cascade/pin carriers, which is irrelevant to racers bound to fastCarrierScheduler. - private static final int STALE_RACER_BATCH = 64; - private static final int MAX_STALE_RACERS = 1024; - - // A short keep-alive forces genuine carrier-thread exit (and thus a real ProfiledThread - // free()) in well under a second instead of 30s. The racer's sleep is randomized across a - // range straddling this keep-alive so runs cover both "carrier likely still exiting" and - // "carrier long gone, memory possibly already reused" timings instead of one fixed instant. - private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; - private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; - private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; - - // Caps the fast-carrier pool's platform-thread growth. MAX_STALE_RACERS is - // already the hard ceiling on concurrent virtual threads that could each - // need a carrier, so this doesn't reduce headroom — it just removes the - // unbounded-growth characteristic of Integer.MAX_VALUE. - private static final int MAX_FAST_CARRIER_THREADS = MAX_STALE_RACERS; - - private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); - - private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); - private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); - private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); - private final Executor fastCarrierScheduler = VTHREAD_CTOR != null ? newFastCarrierScheduler() : null; - private volatile boolean running; - private volatile boolean activePhase = true; - private Thread driver; - private Thread pinningDriver; - private Thread staleRacerDriver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "vthread-context-cascade"; - } - - @Override - public void start() { - if (OF_VIRTUAL != null && BUILDER_START != null && VTHREAD_CTOR == null) { - // We're on a VT-capable JDK, so the fast-carrier constructor should be reachable — - // its absence (missing --add-opens, or a JDK build that changed the constructor) - // would otherwise silently drop stale-racer coverage instead of failing the run. - throw new IllegalStateException( - "vthread-context-cascade: java.lang.VirtualThread(Executor, String, int, " - + "Runnable) is unreachable on a VT-capable JDK — pass --add-opens " - + "java.base/java.lang=ALL-UNNAMED, or this antagonist loses its core " - + "stale-carrier UAF coverage"); - } - running = true; - driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); - driver.setDaemon(true); - driver.start(); - pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); - pinningDriver.setDaemon(true); - pinningDriver.start(); - staleRacerDriver = new Thread(this::staleRacerLoop, "chaos-vthread-cascade-racer-driver"); - staleRacerDriver.setDaemon(true); - staleRacerDriver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long deadlineNanos = System.nanoTime() + timeout.toNanos(); - // Driver threads are only non-null once start() has actually spawned them; guard against - // stopGracefully being called when start() never ran or failed partway through. - if (driver != null) { - try { - driver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - if (pinningDriver != null) { - try { - pinningDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - if (staleRacerDriver != null) { - try { - staleRacerDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. - try { - liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is - // best-effort only — outstanding racers are daemon threads and die with the JVM. - try { - liveStaleRacers.tryAcquire(MAX_STALE_RACERS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - /** Milliseconds remaining until {@code deadlineNanos}, clamped to zero (never negative). */ - private static long remainingMillis(long deadlineNanos) { - long remainingNanos = deadlineNanos - System.nanoTime(); - return remainingNanos <= 0L ? 0L : TimeUnit.NANOSECONDS.toMillis(remainingNanos); - } - - /** - * Independently pins short-lived virtual threads to their carrier and releases them again, - * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes - * are racing carrier migration on the other driver. - */ - private void pinningChurnLoop() { - if (OF_VIRTUAL == null || BUILDER_START == null) { - return; - } - while (running) { - if (!activePhase) { - // Quiet phase: stay out of the way so idle carriers can actually be reaped. - try { - Thread.sleep(50L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - continue; - } - for (int i = 0; i < PIN_BATCH && running && activePhase; i++) { - if (!livePinVthreads.tryAcquire()) { - continue; - } - final Object lock = PIN_LOCKS[ThreadLocalRandom.current().nextInt(PIN_LOCK_COUNT)]; - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke( - builder, - (Runnable) - () -> { - try { - synchronized (lock) { - // Sleeping while holding a monitor pins the - // carrier: the scheduler must compensate with an - // extra platform thread for other runnable - // virtual threads, then reap it once idle. - Thread.sleep(2L); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - livePinVthreads.release(); - } - }); - } catch (Throwable t) { - livePinVthreads.release(); - } - } - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private void driverLoop() { - if (OF_VIRTUAL == null || BUILDER_START == null) { - System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); - return; - } - while (running) { - // Active phase: mint cascade work. - activePhase = true; - long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); - while (running && System.nanoTime() < activeDeadlineNanos) { - for (int i = 0; i < ROOT_BATCH && running; i++) { - // Mint one chain-of-operations: a fresh synthetic trace, its root span - // activated by the tracer the moment runCascadeNode is entered. - spawnCascadeNode(0); - } - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - if (!running) { - return; - } - // Quiet phase: mint nothing new so short-lived virtual threads drain and carriers - // can actually sit idle past the scheduler's 30s keep-alive and get reaped. - activePhase = false; - try { - Thread.sleep(QUIET_PHASE_MILLIS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - /** - * Independent driver that keeps the stale-buffer racer population saturated at {@link - * #MAX_STALE_RACERS} for as long as the antagonist runs, regardless of the cascade/pin duty - * cycle — the fast-carrier scheduler needs no quiet phase to reap its carriers, so there's no - * reason to gate racer spawning on one. This is the "many vthreads hammering" dimension: - * sustained concurrent pressure on the fast-carrier scheduler's pool, not periodic bursts. - */ - private void staleRacerLoop() { - while (running) { - boolean spawnedAny = spawnStaleBufferRacers(); - try { - // Once the pool is fully saturated, back off instead of re-polling 1000x/sec for - // permits that free up far slower (racers sleep up to FAST_STALE_RACER_SLEEP_MAX_MILLIS). - Thread.sleep(spawnedAny ? 1L : 20L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - /** Returns whether at least one racer was actually started this pass. */ - private boolean spawnStaleBufferRacers() { - boolean spawnedAny = false; - for (int i = 0; i < STALE_RACER_BATCH && running; i++) { - if (!liveStaleRacers.tryAcquire()) { - continue; - } - if (startFastCarrierVirtualThread(this::runStaleBufferRacer) == null) { - liveStaleRacers.release(); - } else { - spawnedAny = true; - } - } - return spawnedAny; - } - - /** - * Activates a span once, sleeps past the carrier's keep-alive, then activates another. Under - * {@code ContextStorageMode.THREAD} the second activation reuses whatever {@code - * DirectByteBuffer} was cached on the first — if the original carrier has since been reaped - * and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. - * - *

The sleep is randomized per racer across [{@link #FAST_STALE_RACER_SLEEP_MIN_MILLIS}, - * {@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}] rather than fixed, so a sustained run samples a - * spread of timings relative to {@link #FAST_SCHEDULER_KEEPALIVE_MILLIS} instead of only ever - * probing the same instant. - */ - private void runStaleBufferRacer() { - try { - staleRacerBefore(); - long sleepMillis = - ThreadLocalRandom.current() - .nextLong( - FAST_STALE_RACER_SLEEP_MIN_MILLIS, - FAST_STALE_RACER_SLEEP_MAX_MILLIS + 1); - Thread.sleep(sleepMillis); - staleRacerAfter(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - liveStaleRacers.release(); - } - } - - @Trace(operationName = "chaos.stale.before", resourceName = "chaos.stale.before") - private void staleRacerBefore() { - // Entry/exit alone is the "write" — the tracer activates/deactivates a span around this - // no-op body, exactly like TraceContextAntagonist's inner()/outer(). - } - - @Trace(operationName = "chaos.stale.after", resourceName = "chaos.stale.after") - private void staleRacerAfter() {} - - /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ - private void spawnCascadeNode(int depth) { - if (!running || !liveVthreads.tryAcquire()) { - return; - } - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke( - builder, - (Runnable) - () -> { - try { - runCascadeNode(depth); - } finally { - liveVthreads.release(); - } - }); - } catch (Throwable t) { - liveVthreads.release(); - } - } - - /** - * One hop of the cascade, run as its own {@link Trace}-annotated span on a fresh virtual - * thread. The tracer activates this span on entry and deactivates it on exit; while it's - * active, this hop fans out to child hops on new virtual threads, handing the tracer's own - * virtual-thread instrumentation the job of propagating it as parent context onto each child. - */ - @Trace(operationName = "chaos.cascade.hop", resourceName = "chaos.cascade.hop") - private void runCascadeNode(int depth) { - long r = ThreadLocalRandom.current().nextLong() ^ ((long) depth << 48); - for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { - r = nestedOp(r); - } - sink.addAndGet(r); - - // Propagate the chain to the next hop before this thread's own span is torn down. - if (depth < MAX_DEPTH && running) { - for (int i = 0; i < FAN_OUT; i++) { - spawnCascadeNode(depth + 1); - } - } - } - - /** - * A nested sub-operation within a hop: the tracer activates a child span on entry, {@link - * #forceUnmount} sleeps briefly to force the virtual thread off its current carrier, then the - * span's exit (or the next nested entry) writes context again — racing carrier migration - * against the cached context conduit. - */ - @Trace(operationName = "chaos.cascade.op", resourceName = "chaos.cascade.op") - private long nestedOp(long seed) { - return forceUnmount(seed); - } - - /** - * Burns some CPU (a stand-in for real work) then blocks briefly, forcing the virtual thread - * to genuinely unmount. On resumption it may be remounted on a different carrier than the - * one it was on when the context before this call was written. - */ - private static long forceUnmount(long seed) { - long r = seed; - for (int i = 0; i < 2_000; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - } - try { - Thread.sleep(1L + (r & 1)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return r; - } - - /** - * Resolves {@code java.lang.VirtualThread}'s package-private {@code (Executor, String, int, - * Runnable)} constructor, the only way to bind a virtual thread to a custom scheduler on - * mainline OpenJDK 21+ (there is no public {@code Thread.ofVirtual().scheduler(Executor)} - * API). Requires {@code --add-opens java.base/java.lang=ALL-UNNAMED}; returns {@code null} - * (caller falls back to the default-scheduler path) if that flag is absent, the JVM is - * pre-21, or the constructor's signature changes on some future JDK build. - */ - private static Constructor resolveVirtualThreadCtor() { - try { - Class vthreadClass = Class.forName("java.lang.VirtualThread"); - Constructor ctor = - vthreadClass.getDeclaredConstructor( - Executor.class, String.class, int.class, Runnable.class); - ctor.setAccessible(true); - return ctor; - } catch (Throwable t) { - return null; - } - } - - /** - * A carrier pool with a short keep-alive instead of the JDK default scheduler's fixed 30s, so - * an idle carrier's OS thread actually exits (and its {@code ProfiledThread} is freed) in - * well under a second. - */ - private static Executor newFastCarrierScheduler() { - return new ThreadPoolExecutor( - 0, - MAX_FAST_CARRIER_THREADS, - FAST_SCHEDULER_KEEPALIVE_MILLIS, - TimeUnit.MILLISECONDS, - new SynchronousQueue<>(), - r -> { - Thread carrier = new Thread(r, "chaos-vthread-cascade-fast-carrier"); - carrier.setDaemon(true); - return carrier; - }); - } - - /** - * Starts {@code task} on a virtual thread bound to {@link #fastCarrierScheduler}. Returns - * {@code null} (never starts anything) if {@link #VTHREAD_CTOR} wasn't resolved or the - * reflective construction fails; {@link #start} already fails fast when {@link #VTHREAD_CTOR} - * is unreachable on a VT-capable JDK, so callers here only need to handle the rarer - * construction-failure case (e.g. transient reflection errors). - * - *

{@code 0} for the characteristics argument mirrors {@code Thread.ofVirtual()}'s default - * (the only known bit, {@code Thread.NO_INHERIT_THREAD_LOCALS}, is unset). - */ - private Thread startFastCarrierVirtualThread(Runnable task) { - if (VTHREAD_CTOR == null) { - return null; - } - try { - Thread t = (Thread) VTHREAD_CTOR.newInstance(fastCarrierScheduler, null, 0, task); - t.start(); - return t; - } catch (Throwable t) { - return null; - } - } - - private static Method resolveOfVirtual() { - try { - return Thread.class.getMethod("ofVirtual"); - } catch (NoSuchMethodException e) { - return null; - } - } - - private static Method resolveBuilderStart() { - try { - Class builder = Class.forName("java.lang.Thread$Builder"); - return builder.getMethod("start", Runnable.class); - } catch (ClassNotFoundException | NoSuchMethodException e) { - return null; - } - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java index 9b538e3dbc..0b43dc50ce 100644 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java +++ b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java @@ -1,6 +1,5 @@ package com.datadoghq.profiler.stresstest.scenarios.counters; -import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; import com.datadoghq.profiler.context.ContextExecutor; import com.datadoghq.profiler.context.Tracing; @@ -9,7 +8,6 @@ import org.openjdk.jmh.infra.Blackhole; import java.io.IOException; -import java.util.Arrays; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -18,6 +16,9 @@ public class TracedParallelWork { + private static final int SLOT_TAG0 = 0; + private static final int SLOT_TAG1 = 1; + @State(Scope.Benchmark) public static class BenchmarkState extends Configuration { @@ -29,10 +30,6 @@ public static class BenchmarkState extends Configuration { int tagCardinality; ContextExecutor executor; JavaProfiler profiler; - ContextSetter contextSetter; - - int tag0; - int tag1; String[] tagValues; @@ -49,9 +46,6 @@ public String getTag(long id) { public void setup() throws IOException { profiler = JavaProfiler.getInstance(); executor = new ContextExecutor(200, profiler); - contextSetter = new ContextSetter(profiler, Arrays.asList("tag0", "tag1")); - tag0 = contextSetter.offsetOf("tag0"); - tag1 = contextSetter.offsetOf("tag1"); tagValues = IntStream.range(0, tagCardinality).mapToObj(i -> UUID.randomUUID().toString()) .toArray(String[]::new); } @@ -59,12 +53,10 @@ public void setup() throws IOException { @Benchmark @Threads(8) - @SuppressWarnings("deprecation") public Object work(BenchmarkState state, Blackhole bh) throws ExecutionException, InterruptedException { try (Tracing.Context context = Tracing.newContext(state::newTraceId, state.profiler)) { - state.profiler.setContext(context.getSpanId(), context.getRootSpanId()); - state.contextSetter.setContextValue(state.tag0, state.getTag(context.getSpanId())); - state.contextSetter.setContextValue(state.tag1, state.getTag(context.getSpanId() + 1)); + state.profiler.setContextValue(SLOT_TAG0, state.getTag(context.getSpanId())); + state.profiler.setContextValue(SLOT_TAG1, state.getTag(context.getSpanId() + 1)); Future f = state.executor.submit(() -> compute(state)); bh.consume(compute(state)); return f.get(); @@ -74,9 +66,9 @@ public Object work(BenchmarkState state, Blackhole bh) throws ExecutionException public long compute(BenchmarkState state) { long x = ThreadLocalRandom.current().nextLong(); for (int i = 0; i < 10_000; i++) { - state.contextSetter.setContextValue(state.tag0, state.getTag(x)); + state.profiler.setContextValue(SLOT_TAG0, state.getTag(x)); x ^= ThreadLocalRandom.current().nextLong(); - state.contextSetter.setContextValue(state.tag1, state.getTag(x)); + state.profiler.setContextValue(SLOT_TAG1, state.getTag(x)); } return x; } diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java index 2f3277faab..d2c606a80b 100644 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java +++ b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java @@ -36,16 +36,11 @@ /** * Perf-guard for the all-native context write API: the full per-scope activate+deactivate cycle - * as dd-trace-java drives it, comparing the production combined native calls - * ({@code setTraceContext}/{@code clearTraceContext}) against the deprecated DirectByteBuffer - * sequence ({@code setContext} + 2x {@code setContextAttribute}, then {@code clearContext} + 2x - * {@code clearContextAttribute}). + * as dd-trace-java drives it, using the combined native calls ({@code setTraceContext}/ + * {@code clearTraceContext}). * *

One measured op = one full activate+deactivate cycle (@OperationsPerInvocation({@value #BATCH})). - * The all-native cycle is expected to be at parity or a modest win over the fine-grained DBB cycle, - * on both platform and mounted virtual threads and independent of storage mode (see the design - * note, doc/plans/2026-07-02-all-native-context-storage-design.md). Guards against a regression in - * the shipping API. + * Guards against a regression in the shipping API, on both platform and mounted virtual threads. * *

Run: {@code ./gradlew :ddprof-stresstest:jmh -PjmhInclude="ContextCombinedBenchmark"} */ @@ -55,7 +50,6 @@ jvmArgsAppend = {"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED"}) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 2) -@SuppressWarnings("deprecation") // intentionally exercises the deprecated DirectByteBuffer path as the baseline public class ContextCombinedBenchmark { static final int BATCH = 20_000; @@ -89,16 +83,11 @@ private static void runOnVirtualThread(Runnable r) throws InterruptedException { public static class ProfilerState { JavaProfiler profiler; - @Param({"carrier", "thread"}) - String mode; - @Setup(Level.Trial) public void setup() throws Exception { - System.setProperty("ddprof.debug.context.storage.mode", mode); profiler = JavaProfiler.getInstance(); Path jfr = Files.createTempFile("ctx-combined", ".jfr"); profiler.execute("start,cpu=10ms,attributes=op;res,jfr,file=" + jfr.toAbsolutePath()); - System.out.println("[bench] requested mode=" + mode + " actual=" + profiler.contextStorageMode()); } } @@ -121,17 +110,6 @@ private static void nativeCycle(JavaProfiler p, CtxState s) { p.clearTraceContext(); } - // ---- deprecated DirectByteBuffer cycle (6 calls; cache-hit attribute writes) ---- - - private static void dbbCycle(JavaProfiler p, CtxState s) { - p.setContext(s.lrs, s.span, 0, s.trLo); - p.setContextAttribute(SLOT_OP, OP_NAME); - p.setContextAttribute(SLOT_RES, RES_NAME); - p.setContext(0, 0, 0, 0); - p.clearContextAttribute(SLOT_OP); - p.clearContextAttribute(SLOT_RES); - } - @Benchmark @OperationsPerInvocation(BATCH) public void platform_native_cycle(ProfilerState ps, CtxState s) { @@ -140,14 +118,6 @@ public void platform_native_cycle(ProfilerState ps, CtxState s) { } } - @Benchmark - @OperationsPerInvocation(BATCH) - public void platform_dbb_cycle(ProfilerState ps, CtxState s) { - for (int i = 0; i < BATCH; i++) { - dbbCycle(ps.profiler, s); - } - } - @Benchmark @OperationsPerInvocation(BATCH) public void vthread_native_cycle(ProfilerState ps, CtxState s) throws InterruptedException { @@ -157,14 +127,4 @@ public void vthread_native_cycle(ProfilerState ps, CtxState s) throws Interrupte } }); } - - @Benchmark - @OperationsPerInvocation(BATCH) - public void vthread_dbb_cycle(ProfilerState ps, CtxState s) throws InterruptedException { - runOnVirtualThread(() -> { - for (int i = 0; i < BATCH; i++) { - dbbCycle(ps.profiler, s); - } - }); - } } diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java deleted file mode 100644 index dbd4eec4cc..0000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Benchmarks for ThreadContext operations — measures the per-call cost of - * context set/get/attribute operations on the span start/end hot path. - * - *

Run: ./gradlew :ddprof-stresstest:jmh -PjmhInclude="ThreadContextBenchmark" - * - *

Run with different JAVA_HOME to compare JNI (17+) vs ByteBuffer (<17) paths. - */ -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 2, warmups = 0) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 2) -public class ThreadContextBenchmark { - - private static final String[] ROUTES = { - "GET /api/users", "POST /api/orders", "GET /api/health", - "PUT /api/users/{id}", "DELETE /api/sessions" - }; - - @State(Scope.Benchmark) - public static class ProfilerState { - JavaProfiler profiler; - - @Setup(Level.Trial) - public void setup() throws Exception { - profiler = JavaProfiler.getInstance(); - Path jfr = Files.createTempFile("bench", ".jfr"); - profiler.execute("start,cpu=10ms,attributes=http.route,jfr,file=" + jfr.toAbsolutePath()); - } - } - - @State(Scope.Thread) - public static class ThreadState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int counter; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - } - } - - @State(Scope.Thread) - public static class ReapplyState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int[] constantIds; - byte[][] utf8; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - - // Prime the normal path to obtain constant IDs, then snapshot for reapply. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - ctx.setContextAttribute(i, ROUTES[i]); - } - constantIds = new int[10]; - ctx.copyCustoms(constantIds); - utf8 = new byte[10][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - } - - @Setup(Level.Iteration) - public void resetToSteadyState() { - // Re-establish a live span (valid=1) and pre-populate attrs_data with all slots - // before each measurement window. Without this, reapplyByIdAndBytes sees a - // different attrs_data state on the first invocation of each iteration (empty - // after put() in the previous iteration or after the trial setup), causing a - // bimodal distribution across forks due to JIT profile divergence. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - if (!ctx.setContextAttributesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException( - "resetToSteadyState: setContextAttributesByIdAndBytes failed; benchmark state invalid"); - } - } - } - - @Benchmark - public void setContextFull(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - public boolean setAttrCacheHit(ThreadState ts) { - return ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public void spanLifecycle(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(2) - public void setContextFull_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(4) - public void setContextFull_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(2) - public void spanLifecycle_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(4) - public void spanLifecycle_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public long getSpanId(ThreadState ts) { - return ts.ctx.getSpanId(); - } - - @Benchmark - public void clearContext(ThreadState ts) { - ts.ctx.put(0, 0, 0, 0); - } - - /** Bare reapply cost with constant IDs and bytes already in hand — no Dictionary lookup. */ - @Benchmark - public boolean reapplyByIdAndBytes(ReapplyState rs) { - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - /** Full reapply cycle: span activation wipes slots, then reapply restores them. */ - @Benchmark - public boolean reapplyCycle(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(2) - public boolean reapplyCycle_2t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(4) - public boolean reapplyCycle_4t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java index edded2ea90..22e3864200 100644 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java +++ b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java @@ -25,11 +25,6 @@ protected RunnableFuture newTaskFor(Runnable runnable, T value) { protected void beforeExecute(Thread t, Runnable r) { super.beforeExecute(t, r); profiler.addThread(); - // Prime OTEL context TLS to avoid race condition with wall clock signals. - // TLS is lazily initialized on first setContext() call, which happens in - // ContextTask.run() after this method returns. If a wall clock signal - // arrives between now and then, the context would be uninitialized. - profiler.setContext(0, 0); } @Override diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java index 18a15a4719..17b2089feb 100644 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java +++ b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java @@ -82,7 +82,7 @@ public MigratingContext snapshot() { } private void notifyProfiler() { - profiler.setContext(spanId, rootSpanId); + profiler.setTraceContext(rootSpanId, spanId, 0, spanId, -1, null, -1, null); } public long getRootSpanId() { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index de75c2f068..1da7b59bf8 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -330,7 +330,6 @@ protected void runTests(Runnable... runnables) throws InterruptedException { public final void stopProfiler() { if (!stopped) { profiler.stop(); - profiler.resetThreadContext(); stopped = true; checkConfig(); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java new file mode 100644 index 0000000000..f025023976 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Unit tests for {@link ContextValueCache}, the process-wide value cache backing the all-native + * context write path. Package-private, so this test lives in {@code com.datadoghq.profiler}. + */ +public class ContextValueCacheTest { + + private final ContextValueCache cache = new ContextValueCache(); + + private static void loadLibrary() throws IOException { + JavaProfiler.getInstance(); // registerConstant0 is native + } + + @Test + public void nullResolvesToNull() throws IOException { + loadLibrary(); + assertNull(cache.resolve(null)); + } + + @Test + public void resolveHitsCacheOnSecondLookup() throws IOException { + loadLibrary(); + ContextValueCache.Entry first = cache.resolve("hello"); + assertNotNull(first); + ContextValueCache.Entry second = cache.resolve("hello"); + assertSame(first, second, "an unchanged value must hit the cache, not re-register"); + } + + @Test + public void oversizedValueResolvesToNull() throws IOException { + loadLibrary(); + char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES + 1]; + java.util.Arrays.fill(chars, 'x'); + assertNull(cache.resolve(new String(chars))); + } + + @Test + public void maxSizeValueResolves() throws IOException { + loadLibrary(); + char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES]; + java.util.Arrays.fill(chars, 'x'); + String value = new String(chars); + ContextValueCache.Entry e = cache.resolve(value); + assertNotNull(e); + assertEquals(ContextValueCache.MAX_VALUE_BYTES, e.utf8.length); + } + + @Test + public void hashCollisionEvictsPreviousEntryButBothRemainResolvable() throws IOException { + loadLibrary(); + // ContextValueCache is direct-mapped by value.hashCode() & 0xFF (SIZE=256). Find two + // distinct strings that collide in the same slot to exercise the eviction path. + String a = "collision-a"; + int slotA = a.hashCode() & 0xFF; + String b = null; + for (int i = 0; i < 100_000; i++) { + String candidate = "collision-b-" + i; + if ((candidate.hashCode() & 0xFF) == slotA && !candidate.equals(a)) { + b = candidate; + break; + } + } + assertNotNull(b, "failed to find a colliding string for the test slot"); + + ContextValueCache.Entry entryA = cache.resolve(a); + assertNotNull(entryA); + ContextValueCache.Entry entryB = cache.resolve(b); + assertNotNull(entryB); + assertNotEquals(entryA.encoding, entryB.encoding); + + // Re-resolving "a" after it was evicted from its slot by "b" must still work — it's a + // cache miss, so it re-registers and gets a fresh (but valid) entry. + ContextValueCache.Entry entryAAgain = cache.resolve(a); + assertNotNull(entryAAgain); + assertEquals(a, entryAAgain.key); + } + + @Test + public void clearDropsCachedEntries() throws IOException { + loadLibrary(); + ContextValueCache.Entry before = cache.resolve("to-clear"); + assertNotNull(before); + cache.clear(); + ContextValueCache.Entry after = cache.resolve("to-clear"); + assertNotNull(after); + assertNotSame(before, after, "resolve after clear() must not return the pre-clear entry"); + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java index 002e4fba93..787275dcdf 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java @@ -34,8 +34,5 @@ public void javaBoundMatchesNativeCapacity() throws IOException { JavaProfiler.getInstance(); // ensure the native library is loaded assertEquals(JavaProfiler.maxContextSlots0(), JavaProfiler.MAX_CONTEXT_SLOTS, "JavaProfiler.MAX_CONTEXT_SLOTS drifted from native DD_TAGS_CAPACITY"); - // ThreadContext mirrors the same native bound on the deprecated path; keep them aligned too. - assertEquals(JavaProfiler.MAX_CONTEXT_SLOTS, ThreadContext.MAX_CUSTOM_SLOTS, - "MAX_CONTEXT_SLOTS drifted from ThreadContext.MAX_CUSTOM_SLOTS"); } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java deleted file mode 100644 index ec45ce7cda..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Unit tests for {@link OtelContextStorage} mode selection and the {@code thread} kill-switch. - * - *

{@link OtelContextStorage#create()} reads {@link OtelContextStorage#MODE_PROPERTY} on each - * call and returns an instance whose concrete type determines its {@link ContextStorageMode} - * — there is no shared mutable state — so these tests can drive it directly via the system - * property without disturbing the profiler's already-constructed storage. - */ -public class OtelContextStorageTest { - - private String saved; - - @AfterEach - public void restore() { - if (saved == null) { - System.clearProperty(OtelContextStorage.MODE_PROPERTY); - } else { - System.setProperty(OtelContextStorage.MODE_PROPERTY, saved); - } - } - - private void setMode(String mode) { - saved = System.getProperty(OtelContextStorage.MODE_PROPERTY); - System.setProperty(OtelContextStorage.MODE_PROPERTY, mode); - } - - /** The kill-switch: {@code mode=thread} always yields plain thread-scoped storage. */ - @Test - public void threadModeForcesPlainThreadLocal() { - setMode("thread"); - ThreadLocal storage = OtelContextStorage.create(); - assertNotNull(storage); - assertEquals(ContextStorageMode.THREAD, OtelContextStorage.modeOf(storage)); - // A forced-thread instance must be a plain ThreadLocal, never CarrierThreadLocal. - assertEquals(ThreadLocal.class, storage.getClass(), - "thread mode must not use jdk.internal.misc.CarrierThreadLocal"); - } - - /** - * On JDK 21+ with {@code jdk.internal.misc} exported (the build adds the flag), {@code auto} - * resolves to carrier scoping. Skipped otherwise, since the fallback is environment-driven. - */ - @Test - public void autoModeUsesCarrierWhenAvailable() { - assumeTrue(Platform.isJavaVersionAtLeast(21), "carrier scoping needs JDK 21+"); - // Confirm CarrierThreadLocal is actually reachable in this run; if not (export not - // granted), auto legitimately falls back to THREAD and there is nothing to assert. - assumeTrue(carrierThreadLocalAccessible(), - "jdk.internal.misc.CarrierThreadLocal not accessible; needs --add-exports"); - - setMode("auto"); - ThreadLocal storage = OtelContextStorage.create(); - assertEquals(ContextStorageMode.CARRIER, OtelContextStorage.modeOf(storage), - "auto must select carrier scoping when CarrierThreadLocal is accessible"); - } - - /** {@code carrier} succeeds and yields carrier-scoped storage when CarrierThreadLocal is accessible. */ - @Test - public void carrierModeUsesCarrierWhenAvailable() { - assumeTrue(carrierThreadLocalAccessible(), - "CarrierThreadLocal not accessible; the fail-fast path is covered by the throw test"); - setMode("carrier"); - ThreadLocal storage = OtelContextStorage.create(); - assertEquals(ContextStorageMode.CARRIER, OtelContextStorage.modeOf(storage)); - } - - /** - * {@code carrier} fails hard when CarrierThreadLocal is unavailable, rather than silently - * falling back to the virtual-thread-pinned storage the fix removes. - */ - @Test - public void carrierModeThrowsWhenUnavailable() { - assumeTrue(!carrierThreadLocalAccessible(), - "CarrierThreadLocal IS accessible here; the fail-fast path only applies when it is not (older JDKs / no export)"); - setMode("carrier"); - assertThrows(IllegalStateException.class, OtelContextStorage::create); - } - - private static boolean carrierThreadLocalAccessible() { - try { - Class c = Class.forName("jdk.internal.misc.CarrierThreadLocal"); - c.getConstructor().newInstance(); - return true; - } catch (Throwable t) { - return false; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java deleted file mode 100644 index 5078181df2..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java +++ /dev/null @@ -1,164 +0,0 @@ -package com.datadoghq.profiler; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; - -/** - * Pure-Java unit test for {@link ScopeStack}. Uses heap-backed {@link ByteBuffer}s so - * no native library is required. Exercises depth accounting, underflow, and round-trip - * preservation of trace/span IDs across fast-path and chunked-path depths. - */ -public class ScopeStackTest { - - // Offsets mirror OtelThreadContextRecord in otel_context.h and the sidecar layout - // built by initializeContextTLS0 in javaApi.cpp. These are spec-fixed; guarded by - // static_asserts in native code. All are absolute within the unified buffer. - private static final int TRACE_ID_OFFSET = 0; - private static final int SPAN_ID_OFFSET = 16; - private static final int VALID_OFFSET = 24; - private static final int ATTRS_DATA_SIZE_OFFSET = 26; - private static final int ATTRS_DATA_OFFSET = 28; - private static final int LRS_OFFSET = 640 + 40; // after 640-byte record + 10 * sizeof(u32) - - private static ThreadContext newContext() { - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - return new ThreadContext(buf, metadata); - } - - private static void assumeLittleEndian() { - Assumptions.assumeTrue( - ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN, - "ThreadContext only supports little-endian platforms"); - } - - @Test - public void depthBalance() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertEquals(0, stack.depth()); - stack.enter(ctx); - assertEquals(1, stack.depth()); - stack.enter(ctx); - assertEquals(2, stack.depth()); - stack.exit(ctx); - assertEquals(1, stack.depth()); - stack.exit(ctx); - assertEquals(0, stack.depth()); - } - - @Test - public void exitUnderflowThrows() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertThrows(IllegalStateException.class, () -> stack.exit(ctx)); - } - - @Test - public void fastPathRoundTrip() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(/*lrs*/ 100L, /*span*/ 200L, /*trHi*/ 0L, /*trLo*/ 300L); - assertEquals(200L, ctx.getSpanId()); - assertEquals(100L, ctx.getRootSpanId()); - - stack.enter(ctx); - ctx.put(500L, 600L, 0L, 700L); - assertEquals(600L, ctx.getSpanId()); - assertEquals(500L, ctx.getRootSpanId()); - - stack.exit(ctx); - assertEquals(200L, ctx.getSpanId(), "span must be restored"); - assertEquals(100L, ctx.getRootSpanId(), "root span must be restored"); - } - - @Test - public void chunkedPathRoundTrip() { - // Push past FAST_DEPTH (6) to exercise the lazy-chunk path and Arrays.copyOf growth. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - final int depth = 20; // FAST_DEPTH + one full 12-slot chunk + 2 into the next - for (int i = 0; i < depth; i++) { - ctx.put(1000L + i, 2000L + i, 0L, 3000L + i); - stack.enter(ctx); - } - assertEquals(depth, stack.depth()); - - // Scramble state so restore has something to correct. - ctx.put(99L, 99L, 0L, 99L); - - for (int i = depth - 1; i >= 0; i--) { - stack.exit(ctx); - assertEquals(2000L + i, ctx.getSpanId(), "span mismatch at depth " + i); - assertEquals(1000L + i, ctx.getRootSpanId(), "root mismatch at depth " + i); - } - assertEquals(0, stack.depth()); - } - - @Test - public void reusesStackAfterFullUnwind() { - // After the stack returns to depth 0, re-entering must not leak state from the prior run. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - stack.enter(ctx); - ctx.put(10L, 20L, 0L, 30L); - stack.exit(ctx); - assertEquals(2L, ctx.getSpanId()); - - ctx.put(4L, 5L, 0L, 6L); - stack.enter(ctx); - ctx.put(40L, 50L, 0L, 60L); - stack.exit(ctx); - assertEquals(5L, ctx.getSpanId()); - } - - @Test - public void snapshotOverClearedContextDoesNotRepublish() { - // Regression: snapshot() used to unconditionally re-attach, flipping valid back to 1 - // after a zero-put clear. The clear path leaves attrs_data_size / attrs_data stale and - // relies on valid=0 to keep external readers from seeing the stale bytes. Here we verify - // the valid byte directly since setContextAttribute is a native path unavailable to - // pure-Java tests. - assumeLittleEndian(); - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - ThreadContext ctx = new ThreadContext(buf, metadata); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - assertEquals(1, buf.get(VALID_OFFSET), "record must be published after non-zero put"); - - // Zero-put clear: leaves valid=0 (the all-zero early-return in setContextDirect). - ctx.put(0L, 0L, 0L, 0L); - assertEquals(0, buf.get(VALID_OFFSET), "record must be invalid after zero-put clear"); - - stack.enter(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "snapshot must preserve valid=0 — not republish a cleared record"); - - stack.exit(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "restore must replay valid=0 — not republish a cleared record"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java index 9f87801c2c..80b81c1b7b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java @@ -32,12 +32,11 @@ * *

The wall-clock sampler reads context through {@code ContextApi::get}, which returns nothing * until the thread is context-initialized (and only then is the {@code otel_thread_ctx_v1} discovery - * pointer published). Before the fix, that flag was set only by the deprecated DirectByteBuffer path - * ({@code getThreadContext()} / {@code initializeContextTLS0}); a thread that used only the - * all-native API ({@code setTraceContext}) wrote a record the sampler silently ignored. + * pointer published). {@code setTraceContext} must set that flag itself on first write — a thread + * that only ever calls the all-native API must still be visible to the sampler. * - *

This test deliberately never calls {@code getThreadContext()} on the sampled thread. - * If the native write does not initialize TLS, every sample carries spanId 0 and the assertion fails. + *

If the native write does not initialize TLS, every sample carries spanId 0 and the assertion + * fails. */ public class AllNativeContextSamplingTest extends AbstractProfilerTest { @@ -46,7 +45,7 @@ public class AllNativeContextSamplingTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - // filter=0 samples every thread, so no getThreadContext()/registration is needed. + // filter=0 samples every thread, so no extra registration is needed. return "wall=1ms,filter=0,loglevel=warn"; } @@ -54,8 +53,8 @@ protected String getProfilerCommand() { public void nativeOnlyContextIsVisibleToSampler() throws Exception { Assumptions.assumeTrue(!Platform.isJ9() && !Platform.isZing()); - // Register for wall-clock profiling (addThread(); does NOT touch getThreadContext, so the - // all-native-only nature of the test is preserved). + // Register for wall-clock profiling (addThread()); the sampled thread never touches + // anything but the all-native context API. registerCurrentThreadForWallClockProfiling(); // Keep the context live for the whole sampling window; only the all-native path is used. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java index 60f7680e16..70007fe03e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java @@ -17,7 +17,6 @@ import com.datadoghq.profiler.JavaProfiler; import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ThreadContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -38,13 +37,11 @@ * {@code clearTraceContext}, {@code setContextValue}, {@code clearContextValue}. * *

These write the current carrier's native OtelThreadContextRecord directly (no cached - * DirectByteBuffer). The DirectByteBuffer read methods on {@link ThreadContext} - * ({@code getSpanId}/{@code getRootSpanId}/{@code readTraceId}/{@code readContextAttribute}) are a - * view over that same record, so they serve as the read-back oracle here. - * - *

Ordering note: the first {@link JavaProfiler#getThreadContext()} on a thread runs the - * {@code ThreadContext} constructor, which resets the record. So every test resolves the read - * handle before writing, then reads through that handle after the native write. + * per-thread buffer). The read-back oracle is {@link JavaProfiler}'s package-private + * {@code test*} accessors ({@code testGetSpanId}/{@code testGetRootSpanId}/ + * {@code testReadTraceId}/{@code testReadContextAttribute}/{@code testIsContextValid}), which + * read the same record directly and are invoked here via reflection (this test lives in a + * different package than {@link JavaProfiler}). */ public class AllNativeContextTest { @@ -64,7 +61,6 @@ public static void setup() throws IOException { public void cleanup() { if (profilerStarted) { profiler.stop(); - profiler.resetThreadContext(); profilerStarted = false; } } @@ -88,15 +84,34 @@ private static boolean setContextValue0(int slot, int encoding, byte[] utf8) thr return (boolean) m.invoke(null, slot, encoding, utf8); } - /** Reads the record's {@code valid} byte through the ThreadContext buffer (test-only introspection). */ - private static int readValidByte(ThreadContext ctx) throws Exception { - java.lang.reflect.Field bufField = ThreadContext.class.getDeclaredField("ctxBuffer"); - bufField.setAccessible(true); - java.nio.ByteBuffer buf = (java.nio.ByteBuffer) bufField.get(ctx); - java.lang.reflect.Field offField = ThreadContext.class.getDeclaredField("validOffset"); - offField.setAccessible(true); - int off = (int) offField.get(ctx); - return buf.get(off) & 0xFF; + private static long testGetSpanId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testGetSpanId"); + m.setAccessible(true); + return (long) m.invoke(profiler); + } + + private static long testGetRootSpanId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testGetRootSpanId"); + m.setAccessible(true); + return (long) m.invoke(profiler); + } + + private static String testReadTraceId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testReadTraceId"); + m.setAccessible(true); + return (String) m.invoke(profiler); + } + + private static String testReadContextAttribute(int slot) throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testReadContextAttribute", int.class); + m.setAccessible(true); + return (String) m.invoke(profiler, slot); + } + + private static boolean testIsContextValid() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testIsContextValid"); + m.setAccessible(true); + return (boolean) m.invoke(profiler); } /** {@code Thread.ofVirtual().start(task)} via reflection so this compiles with --release 8. */ @@ -111,7 +126,6 @@ private static Thread startVirtualThread(Runnable task) throws Exception { @Test public void setTraceContextRoundTrips() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); long lrs = 0x1111222233334444L; long span = 0xAAAABBBBCCCCDDDDL; @@ -119,95 +133,57 @@ public void setTraceContextRoundTrips() throws Exception { long trLo = 0x9999AAAABBBBCCCCL; profiler.setTraceContext(lrs, span, trHi, trLo, SLOT_OP, "servlet.request", SLOT_RES, "GET /users"); - assertEquals(span, ctx.getSpanId()); - assertEquals(lrs, ctx.getRootSpanId()); - assertEquals("55556666777788889999aaaabbbbcccc", ctx.readTraceId()); - assertEquals("servlet.request", ctx.readContextAttribute(SLOT_OP)); - assertEquals("GET /users", ctx.readContextAttribute(SLOT_RES)); - } - - /** The native write must produce the same observable record state as the DirectByteBuffer path. */ - @Test - public void nativeMatchesDirectByteBufferWrite() throws Exception { - start(); - ThreadContext ctx = profiler.getThreadContext(); - - long lrs = 0x0102030405060708L; - long span = 0x1122334455667788L; - long trHi = 0xAABBCCDDEEFF0011L; - long trLo = 0x2233445566778899L; - - // DirectByteBuffer path (deprecated). - profiler.setContext(lrs, span, trHi, trLo); - ctx.setContextAttribute(SLOT_OP, "servlet.request"); - ctx.setContextAttribute(SLOT_RES, "GET /users/{id}"); - long dbbSpan = ctx.getSpanId(); - long dbbRoot = ctx.getRootSpanId(); - String dbbTrace = ctx.readTraceId(); - String dbbOp = ctx.readContextAttribute(SLOT_OP); - String dbbRes = ctx.readContextAttribute(SLOT_RES); - - profiler.clearTraceContext(); - assertEquals(0, ctx.getSpanId(), "precondition: cleared before native write"); - - // All-native path — identical inputs. - profiler.setTraceContext(lrs, span, trHi, trLo, SLOT_OP, "servlet.request", SLOT_RES, "GET /users/{id}"); - - assertEquals(dbbSpan, ctx.getSpanId(), "spanId"); - assertEquals(dbbRoot, ctx.getRootSpanId(), "rootSpanId"); - assertEquals(dbbTrace, ctx.readTraceId(), "traceId"); - assertEquals(dbbOp, ctx.readContextAttribute(SLOT_OP), "op attribute"); - assertEquals(dbbRes, ctx.readContextAttribute(SLOT_RES), "resource attribute"); + assertEquals(span, testGetSpanId()); + assertEquals(lrs, testGetRootSpanId()); + assertEquals("55556666777788889999aaaabbbbcccc", testReadTraceId()); + assertEquals("servlet.request", testReadContextAttribute(SLOT_OP)); + assertEquals("GET /users", testReadContextAttribute(SLOT_RES)); } @Test public void clearTraceContextResetsRecord() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, SLOT_OP, "op", SLOT_RES, "res"); - assertEquals(0x1L, ctx.getSpanId()); + assertEquals(0x1L, testGetSpanId()); profiler.clearTraceContext(); - assertEquals(0, ctx.getSpanId(), "spanId cleared"); - assertEquals(0, ctx.getRootSpanId(), "rootSpanId cleared"); + assertEquals(0, testGetSpanId(), "spanId cleared"); + assertEquals(0, testGetRootSpanId(), "rootSpanId cleared"); // valid=0 after clear, so attributes are not observable. - assertNull(ctx.readContextAttribute(SLOT_OP), "attributes cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attributes cleared"); } /** setTraceContext resets custom slots, so a span-to-span transition must not leak attributes. */ @Test public void spanTransitionClearsAttributes() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x1L, 0x1L, 0L, 0x1L, SLOT_OP, "/api/spanA", -1, null); - assertEquals("/api/spanA", ctx.readContextAttribute(SLOT_OP)); + assertEquals("/api/spanA", testReadContextAttribute(SLOT_OP)); // Next span carries no activation attributes. profiler.setTraceContext(0x2L, 0x2L, 0L, 0x2L, -1, null, -1, null); - assertEquals(0x2L, ctx.getSpanId()); - assertNull(ctx.readContextAttribute(SLOT_OP), "previous span's attribute must be cleared"); + assertEquals(0x2L, testGetSpanId()); + assertNull(testReadContextAttribute(SLOT_OP), "previous span's attribute must be cleared"); } @Test public void singleAttributeSetAndClear() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span, no attrs assertTrue(profiler.setContextValue(SLOT_OP, "GET /api/users")); - assertEquals("GET /api/users", ctx.readContextAttribute(SLOT_OP)); + assertEquals("GET /api/users", testReadContextAttribute(SLOT_OP)); profiler.clearContextValue(SLOT_OP); - assertNull(ctx.readContextAttribute(SLOT_OP), "attribute cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attribute cleared"); } /** attrs_data overflow is reported (false), not crashed; unrelated slots keep working. */ @Test public void attributeOverflowReturnsFalse() throws Exception { start(); - profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x3L, -1, null, -1, null); StringBuilder sb = new StringBuilder(255); @@ -225,7 +201,6 @@ public void attributeOverflowReturnsFalse() throws Exception { @Test public void rejectedValuesReturnFalse() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); assertFalse(profiler.setContextValue(SLOT_OP, null), "null value rejected"); @@ -234,27 +209,7 @@ public void rejectedValuesReturnFalse() throws Exception { assertFalse(profiler.setContextValue(SLOT_OP, sb.toString()), "oversized (256B) value rejected"); // A subsequent valid write still works. assertTrue(profiler.setContextValue(SLOT_OP, "ok")); - assertEquals("ok", ctx.readContextAttribute(SLOT_OP)); - } - - /** The deprecated DBB path and the all-native path write the same record and interleave cleanly. */ - @Test - public void coexistenceOfDbbAndNativePaths() throws Exception { - start(); - ThreadContext ctx = profiler.getThreadContext(); - - profiler.setContext(0xA0L, 0xA1L, 0L, 0xA1L); // DBB - assertEquals(0xA1L, ctx.getSpanId()); - - profiler.setTraceContext(0xB0L, 0xB1L, 0L, 0xB1L, -1, null, -1, null); // native - assertEquals(0xB1L, ctx.getSpanId()); - - ctx.setContextAttribute(SLOT_OP, "dbb-value"); // DBB attribute - assertEquals("dbb-value", ctx.readContextAttribute(SLOT_OP)); - - assertTrue(profiler.setContextValue(SLOT_RES, "native-value")); // native attribute - assertEquals("native-value", ctx.readContextAttribute(SLOT_RES)); - assertEquals("dbb-value", ctx.readContextAttribute(SLOT_OP), "DBB-written attr still intact"); + assertEquals("ok", testReadContextAttribute(SLOT_OP)); } /** @@ -265,18 +220,17 @@ public void coexistenceOfDbbAndNativePaths() throws Exception { @Test public void setTraceContextRejectsZeroSpanId() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x9L, 0x7L, 0L, 0x7L, -1, null, -1, null); // active span - assertEquals(0x7L, ctx.getSpanId()); + assertEquals(0x7L, testGetSpanId()); assertThrows(IllegalArgumentException.class, () -> profiler.setTraceContext(0x2L, 0L, 0L, 0L, -1, null, -1, null), "spanId=0 must be rejected"); // The rejected call is a no-op: it neither clears nor corrupts the still-active span. - assertEquals(1, readValidByte(ctx), "record still valid after rejected call"); - assertEquals(0x7L, ctx.getSpanId(), "previously active span left intact"); + assertTrue(testIsContextValid(), "record still valid after rejected call"); + assertEquals(0x7L, testGetSpanId(), "previously active span left intact"); } /** @@ -287,17 +241,16 @@ public void setTraceContextRejectsZeroSpanId() throws Exception { @Test public void clearContextValuePreservesInvalidState() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, SLOT_OP, "op", -1, null); // active span - assertEquals(1, readValidByte(ctx), "precondition: active record is valid"); + assertTrue(testIsContextValid(), "precondition: active record is valid"); profiler.clearTraceContext(); - assertEquals(0, readValidByte(ctx), "clearTraceContext leaves the record deactivated"); + assertFalse(testIsContextValid(), "clearTraceContext leaves the record deactivated"); // Clearing an attribute on a deactivated record must not flip valid back to 1. profiler.clearContextValue(SLOT_OP); - assertEquals(0, readValidByte(ctx), + assertFalse(testIsContextValid(), "clearContextValue must preserve valid=0; a deactivated record must stay deactivated"); } @@ -311,27 +264,25 @@ public void clearContextValuePreservesInvalidState() throws Exception { @Test public void setContextValuePublishesAppContextWithoutSpan() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // active span - assertEquals(1, readValidByte(ctx), "precondition: active record is valid"); + assertTrue(testIsContextValid(), "precondition: active record is valid"); profiler.clearTraceContext(); - assertEquals(0, readValidByte(ctx), "clearTraceContext leaves the record deactivated"); + assertFalse(testIsContextValid(), "clearTraceContext leaves the record deactivated"); // Setting an attribute on a deactivated record republishes it so the attribute is visible // with no active span (span/trace stay zero, but the record is valid and carries the value). profiler.setContextValue(SLOT_OP, "late"); - assertEquals(1, readValidByte(ctx), + assertTrue(testIsContextValid(), "setContextValue must publish (valid=1) so app context is visible without a span"); - assertEquals(0, ctx.getSpanId(), "span stays zero — this is app context, not a span"); - assertEquals("late", ctx.readContextAttribute(SLOT_OP), "attribute observable without a span"); + assertEquals(0, testGetSpanId(), "span stays zero — this is app context, not a span"); + assertEquals("late", testReadContextAttribute(SLOT_OP), "attribute observable without a span"); } /** * copyContextTags reads the sidecar tag encodings written through the all-native path directly - * from the record — without going through ThreadContext (which would reset the record). The test - * never calls getThreadContext, proving the native read observes native writes. + * from the record. */ @Test public void copyContextTagsReadsNativeEncodings() throws Exception { @@ -360,13 +311,12 @@ public void copyContextTagsReadsNativeEncodings() throws Exception { @Test public void nativeNullValueByteArrayIsTreatedAsEmpty() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span // encoding 0 is benign; the point is that a null byte[] does not crash and yields an empty // attribute rather than a dereference. assertTrue(setContextValue0(SLOT_OP, 0, null), "null value byte[] accepted as empty"); - assertEquals("", ctx.readContextAttribute(SLOT_OP), "null value stored as empty attribute"); + assertEquals("", testReadContextAttribute(SLOT_OP), "null value stored as empty attribute"); } /** @@ -377,19 +327,18 @@ public void nativeNullValueByteArrayIsTreatedAsEmpty() throws Exception { @Test public void zeroLengthAttributeCompactsCleanly() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span assertTrue(profiler.setContextValue(SLOT_OP, ""), "empty value written"); - assertEquals("", ctx.readContextAttribute(SLOT_OP), "empty attribute observable"); + assertEquals("", testReadContextAttribute(SLOT_OP), "empty attribute observable"); // Overwrite the zero-length entry: compaction must walk over the 2-byte entry correctly. assertTrue(profiler.setContextValue(SLOT_OP, "now-non-empty")); - assertEquals("now-non-empty", ctx.readContextAttribute(SLOT_OP)); + assertEquals("now-non-empty", testReadContextAttribute(SLOT_OP)); // Clearing it back out also compacts across the (now larger) entry without corruption. profiler.clearContextValue(SLOT_OP); - assertNull(ctx.readContextAttribute(SLOT_OP), "attribute cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attribute cleared"); } /** @@ -401,7 +350,6 @@ public void zeroLengthAttributeCompactsCleanly() throws Exception { @Test public void slotBoundaryIsRejected() throws Exception { start(); - profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span final int capacity = 10; // native DD_TAGS_CAPACITY; drift caught by MaxContextSlotsTest @@ -448,15 +396,14 @@ public void nativeWritesFromVirtualThreadsAreCoherent() throws Exception { final long root = 0x900000L + i; startVirtualThread(() -> { try { - // Resolve the read handle first (may init/reset the carrier record), then write - // and read back with no yield point in between. - ThreadContext ctx = profiler.getThreadContext(); + // Write and read back with no yield point in between, so this vthread stays + // mounted on one carrier for the whole write+read. profiler.setTraceContext(root, span, 0L, span, SLOT_OP, "vt", -1, null); - long got = ctx.getSpanId(); + long got = testGetSpanId(); if (got != span) { failures.add("expected span " + span + " but read " + got); } - if (!"vt".equals(ctx.readContextAttribute(SLOT_OP))) { + if (!"vt".equals(testReadContextAttribute(SLOT_OP))) { failures.add("attribute mismatch for span " + span); } } catch (Throwable t) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java deleted file mode 100644 index f0ac047d8f..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.ContextStorageMode; -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ThreadContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Verifies that {@link ThreadContext} storage is scoped to the carrier thread when - * carrier scoping is active ({@link com.datadoghq.profiler.ContextStorageMode#CARRIER}). - * - *

The OTEP record a {@code ThreadContext} writes to is embedded in the carrier's native - * {@code ProfiledThread} and is what the (carrier-bound) sampler reads. Under carrier scoping, - * every virtual thread mounted on a given carrier must resolve to the same - * {@code ThreadContext} — the one whose buffer targets that carrier's live record — regardless - * of how many virtual threads time-share the carrier. Under the legacy plain-{@code ThreadLocal} - * behavior each virtual thread gets its own, pinned to whatever carrier it first ran on. - * - *

Requires JDK 21+ (virtual threads) and {@code jdk.internal.misc.CarrierThreadLocal} being - * accessible (the build adds {@code --add-exports java.base/jdk.internal.misc=ALL-UNNAMED} on - * 21+). When carrier scoping is not active the test is skipped rather than failing, so it never - * gives false confidence on JDKs/configs where the fix cannot engage. - */ -public class CarrierContextStorageTest { - - private static JavaProfiler profiler; - - @BeforeAll - public static void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - /** {@code Thread.ofVirtual().start(task)} via reflection so this compiles with --release 8. */ - private static Thread startVirtualThread(Runnable task) throws Exception { - Method ofVirtual = Thread.class.getMethod("ofVirtual"); - Object builder = ofVirtual.invoke(null); - Class builderInterface = Class.forName("java.lang.Thread$Builder"); - Method start = builderInterface.getMethod("start", Runnable.class); - return (Thread) start.invoke(builder, task); - } - - /** Extracts the carrier name from a mounted VirtualThread's toString, e.g. {@code ...@ForkJoinPool-1-worker-2}. */ - private static String carrierOf(Thread current) { - String s = current.toString(); - int at = s.lastIndexOf('@'); - return at >= 0 ? s.substring(at + 1) : ""; - } - - @Test - public void contextIsSharedPerCarrierAcrossVirtualThreads() throws Exception { - assumeTrue(Platform.isJavaVersionAtLeast(21), "virtual threads require JDK 21+"); - assumeTrue(ContextStorageMode.CARRIER == profiler.contextStorageMode(), - "carrier-scoped storage not active (mode=" + profiler.contextStorageMode() - + "); needs JDK 21+ and --add-exports java.base/jdk.internal.misc=ALL-UNNAMED"); - - // Map each observed carrier to the set of distinct ThreadContext identities seen on it. - final Map> carrierToContexts = new ConcurrentHashMap<>(); - final int nThreads = 2000; - final CountDownLatch done = new CountDownLatch(nThreads); - - for (int i = 0; i < nThreads; i++) { - startVirtualThread(() -> { - try { - // Resolve the context on this vthread and record its identity against the - // carrier it is currently mounted on. With far more vthreads (2000) than - // carriers, many vthreads time-share each carrier, so keying by the virtual - // thread would produce ~one context per vthread while carrier scoping produces - // ~one per carrier. - ThreadContext c1 = profiler.getThreadContext(); - String carrier = carrierOf(Thread.currentThread()); - carrierToContexts - .computeIfAbsent(carrier, k -> ConcurrentHashMap.newKeySet()) - .add(System.identityHashCode(c1)); - } finally { - done.countDown(); - } - }); - } - - assertTrue(done.await(60, TimeUnit.SECONDS), "virtual threads did not finish in time"); - - int carriers = carrierToContexts.size(); - Set allContexts = ConcurrentHashMap.newKeySet(); - for (Map.Entry> e : carrierToContexts.entrySet()) { - // Normally every vthread that ran on this carrier saw the SAME ThreadContext, so the - // expected count is 1. We assert >= 1 rather than == 1 because the key is the carrier - // *name* (e.g. ForkJoinPool-1-worker-3): if the pool retires a worker and creates a - // replacement for the same slot mid-run, a second (equally valid) context can appear - // under one name. The did-NOT-key-by-vthread guarantee is enforced by the aggregate - // bound below, which does not depend on name stability. - assertTrue(e.getValue().size() >= 1, - "carrier " + e.getKey() + " must expose at least one ThreadContext"); - allContexts.addAll(e.getValue()); - } - - // The crux: far fewer distinct contexts than virtual threads — i.e. storage did NOT key by - // the virtual thread. Roughly one context per carrier (allowing for occasional carrier-name - // reuse, so allContexts may slightly exceed the carrier count). - assertTrue(allContexts.size() >= carriers, - "expected at least one ThreadContext per carrier"); - assertTrue(carriers > 0 && carriers < nThreads && allContexts.size() < nThreads, - "expected carrier count (" + carriers + ") and context count (" + allContexts.size() - + ") to be well below vthread count (" + nThreads + ")"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java deleted file mode 100644 index 8e5f4f6ac8..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Tests for OTEL-compatible context storage (OTEP #4947). - */ -public class OtelContextStorageModeTest { - - private static JavaProfiler profiler; - private boolean profilerStarted = false; - - @BeforeAll - public static void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - @AfterEach - public void cleanup() { - if (profilerStarted) { - profiler.stop(); - profiler.resetThreadContext(); - profilerStarted = false; - } - } - - /** - * Tests that context round-trips correctly. - */ - @Test - public void testOtelStorageModeContext() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-otel", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=tag1;tag2;tag3,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - long traceIdHigh = 0x5555666677778888L; - long traceIdLow = 0x9999AAAABBBBCCCCL; - profiler.setContext(localRootSpanId, spanId, traceIdHigh, traceIdLow); - - ThreadContext ctx = profiler.getThreadContext(); - assertEquals(spanId, ctx.getSpanId(), "SpanId should match"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match"); - // Verify the 128-bit trace ID round-trips through the OTEP record (big-endian) - assertEquals("55556666777788889999aaaabbbbcccc", ctx.readTraceId(), "TraceId should match"); - } - - - /** - * Tests that custom attributes are correctly written to and read back from - * the OTEP record's attrs_data (via DirectByteBuffer). Verifies the - * sidecar encoding is set and the UTF-8 value appears in attrs_data. - */ - @Test - public void testOtelModeCustomAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-attrs", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=http.route;db.system,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - profiler.setContext(localRootSpanId, spanId, 0L, 0x9999L); - - ThreadContext ctx = profiler.getThreadContext(); - boolean result = ctx.setContextAttribute(0, "GET /api/users"); - assertTrue(result, "setContextAttribute should succeed"); - - result = ctx.setContextAttribute(1, "postgresql"); - assertTrue(result, "setContextAttribute for second key should succeed"); - - // Verify attribute values round-trip correctly through attrs_data - assertEquals("GET /api/users", ctx.readContextAttribute(0), "http.route should round-trip"); - assertEquals("postgresql", ctx.readContextAttribute(1), "db.system should round-trip"); - - // Verify trace context is still intact after attribute writes - assertEquals(spanId, ctx.getSpanId(), "SpanId should match after setAttribute"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match after setAttribute"); - } - - /** - * Tests that attrs_data overflow is handled gracefully (returns false, no crash). - */ - @Test - public void testOtelModeAttributeOverflow() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-overflow", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=k0;k1;k2;k3;k4,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - profiler.setContext(0x2L, 0x1L, 0L, 0x3L); - - ThreadContext ctx = profiler.getThreadContext(); - - // LRS is a fixed 18-byte entry (key=0, len=16, 16 hex value bytes). - // Available for custom attrs: 612 - 18 = 594 bytes. - // Each 255-char attr = 257 bytes. Two fit (514 ≤ 594); third overflows (771 > 594). - StringBuilder sb = new StringBuilder(255); - for (int i = 0; i < 255; i++) sb.append('x'); - String longValue = sb.toString(); - assertTrue(ctx.setContextAttribute(0, longValue), "First long attr should fit"); - assertTrue(ctx.setContextAttribute(1, longValue), "Second long attr should fit"); - assertFalse(ctx.setContextAttribute(2, longValue), "Third long attr should overflow"); - - // Short values should still work for remaining slots - assertTrue(ctx.setContextAttribute(3, "short"), "Short attr after overflow should work"); - } - - /** - * Tests sequential context updates including boundary values and clearing. - * Verifies that each setContext overwrites the previous values, that MAX_VALUE - * round-trips correctly, and that clearContext resets both IDs to zero. - */ - @Test - public void testSequentialContextUpdates() { - profiler.setContext(2L, 1L, 0, 1L); - assertEquals(1L, profiler.getThreadContext().getSpanId()); - assertEquals(2L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(20L, 10L, 0, 10L); - assertEquals(10L, profiler.getThreadContext().getSpanId()); - assertEquals(20L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(200L, 100L, 0, 100L); - assertEquals(100L, profiler.getThreadContext().getSpanId()); - assertEquals(200L, profiler.getThreadContext().getRootSpanId()); - - long maxValue = Long.MAX_VALUE; - profiler.setContext(maxValue, maxValue, 0, maxValue); - assertEquals(maxValue, profiler.getThreadContext().getSpanId(), "SpanId should be MAX_VALUE"); - assertEquals(maxValue, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be MAX_VALUE"); - - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "SpanId should be zero after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be zero after clear"); - } - - @Test - public void testThreadIsolation() throws InterruptedException { - long threadASpanId = 1000L; - long threadARootSpanId = 1001L; - profiler.setContext(threadARootSpanId, threadASpanId, 0, threadASpanId); - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - - final long threadBSpanId = 2000L; - final long threadBRootSpanId = 2001L; - final AssertionError[] threadBError = {null}; - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(threadBRootSpanId, threadBSpanId, 0, threadBSpanId); - assertEquals(threadBSpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadBRootSpanId, profiler.getThreadContext().getRootSpanId()); - } catch (AssertionError e) { - threadBError[0] = e; - } - }, "TestThread-B"); - - threadB.start(); - threadB.join(); - - if (threadBError[0] != null) throw threadBError[0]; - - // Thread A's context must be unaffected - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - } - - /** - * Tests that a direct span-to-span transition (no clearContext in between) - * does not leak custom attributes from the previous span. - */ - @Test - public void testSpanTransitionClearsAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-transition", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=http.route,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - // Span A: set a custom attribute - profiler.setContext(0x1L, 0x1L, 0L, 0x1L); - ThreadContext ctx = profiler.getThreadContext(); - ctx.setContextAttribute(0, "/api/spanA"); - - // Transition directly to span B without clearing - profiler.setContext(0x2L, 0x2L, 0L, 0x2L); - - // Span A's attribute must not be visible in span B's context - assertNull(ctx.readContextAttribute(0), "Custom attribute must be cleared on span transition"); - } - - /** - * Stress-tests the OTEP context path with many sequential writes to catch - * buffer corruption or stale-value leaks over repeated updates. - */ - @Test - public void testRepeatedContextWrites() { - for (int i = 1; i <= 1000; i++) { - long spanId = (long) i; - long rootSpanId = (long) (i + 10000); - profiler.setContext(rootSpanId, spanId, 0L, spanId); - assertEquals(spanId, profiler.getThreadContext().getSpanId(), - "spanId mismatch at iteration " + i); - assertEquals(rootSpanId, profiler.getThreadContext().getRootSpanId(), - "rootSpanId mismatch at iteration " + i); - } - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "spanId should be 0 after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "rootSpanId should be 0 after clear"); - } - - /** - * Tests that the per-thread attribute cache isolates threads correctly. - * "FB" and "Ea" have equal hashCode() (both 2236), so they map to the same - * cache slot. With per-thread caches each thread owns its slot independently. - */ - @Test - public void testAttributeCacheIsolation() throws Exception { - Path jfrFile = Files.createTempFile("otel-attr-cache-iso", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=attr0,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - final String valueA = "FB"; // hashCode = 2236, slot 188 - final String valueB = "Ea"; // hashCode = 2236, same slot - final AssertionError[] errors = {null, null}; - final CountDownLatch bothWritten = new CountDownLatch(2); - - Thread threadA = new Thread(() -> { - try { - profiler.setContext(1L, 1L, 0L, 1L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueA)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - // After thread B has written "Ea" to its own slot, A must still read "FB" - assertEquals(valueA, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[0] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheA"); - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(2L, 2L, 0L, 2L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueB)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - assertEquals(valueB, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[1] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheB"); - - threadA.start(); - threadB.start(); - threadA.join(10_000); - threadB.join(10_000); - - if (errors[0] != null) throw errors[0]; - if (errors[1] != null) throw errors[1]; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java deleted file mode 100644 index 95c8fecd50..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java +++ /dev/null @@ -1,641 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.IntStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.ContextSetter; -import static com.datadoghq.profiler.MoreAssertions.DICTIONARY_PAGE_SIZE; -import static com.datadoghq.profiler.MoreAssertions.assertBoundedBy; -import com.datadoghq.profiler.Platform; - -public class TagContextTest extends AbstractProfilerTest { - - @BeforeEach - void assumeNotJ9() { - // On J9, ProfiledThread (and thus the OTEP TLS buffer) is not allocated until the thread - // is registered for wall-clock profiling, so initializeContextTLS0() returns null and - // ThreadContext creation throws. These tests require a live ThreadContext from the start. - Assumptions.assumeTrue(!Platform.isJ9()); - } - - @RetryingTest(10) - public void test() throws InterruptedException { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag1")); - - // Use session-unique prefix so each @RetryingTest attempt registers fresh values in the - // native Dictionary. Without this, on musl (no JVM fork) the per-thread attrCacheKeys - // persists across retries: cache hits skip registerConstant0(), leaving - // dictionary_context_keys=0 on every retry after the first. - String pfx = Long.toHexString(System.nanoTime()) + "_"; - String[] strings = IntStream.range(0, 10).mapToObj(i -> pfx + i).toArray(String[]::new); - for (int i = 0; i < strings.length * 10; i++) { - work(contextSetter, "tag1", strings[i % strings.length]); - } - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - Map weightsByTagValue = new HashMap<>(); - long droppedSamplesCount = 0; - long droppedSamplesWeight = 0; - long totalSamplesCount = 0; - long totalSamplesWeight = 0; - try { - for (IItemIterable wallclockSamples : events) { - IMemberAccessor weightAccessor = WEIGHT.getAccessor(wallclockSamples.getType()); - // this will become more generic in the future - IMemberAccessor tag1Accessor = TAG_1.getAccessor(wallclockSamples.getType()); - assertNotNull(tag1Accessor); - IMemberAccessor tag2Accessor = TAG_2.getAccessor(wallclockSamples.getType()); - assertNotNull(tag2Accessor); - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String stacktrace = stacktraceAccessor.getMember(sample); - if (!stacktrace.contains("sleep")) { - // we don't know the context has been set for sure until the sleep has started - continue; - } - - long weight = weightAccessor.getMember(sample).longValue(); - totalSamplesCount++; - totalSamplesWeight += weight; - - if (stacktrace.contains("")) { - // track dropped samples statistics but skip for weight distribution calculation - droppedSamplesCount++; - droppedSamplesWeight += weight; - continue; - } - - String tag = tag1Accessor.getMember(sample); - weightsByTagValue.computeIfAbsent(tag, v -> new AtomicLong()) - .addAndGet(weight); - assertNull(tag2Accessor.getMember(sample)); - } - } - long sum = 0; - long[] weights = new long[strings.length]; - System.out.println("Found tag values: " + weightsByTagValue.keySet()); - for (int i = 0; i < strings.length; i++) { - AtomicLong weight = weightsByTagValue.get(strings[i]); - assertNotNull(weight, "Weight for " + strings[i] + " not found. Found: " + weightsByTagValue.keySet()); - weights[i] = weightsByTagValue.get(strings[i]).get(); - sum += weights[i]; - } - double avg = (double) sum / weights.length; - for (int i = 0; i < weights.length; i++) { - assertTrue(Math.abs(weights[i] - avg) < 0.15 * weights[i], strings[i] - + " more than 15% from mean"); - } - - // now check we have settings to unbundle the dynamic columns - IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); - Set recordedContextAttributes = new HashSet<>(); - for (IItemIterable activeSetting : activeSettings) { - IMemberAccessor nameAccessor = JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); - IMemberAccessor valueAccessor = JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); - for (IItem item : activeSetting) { - String name = nameAccessor.getMember(item); - if ("contextattribute".equals(name)) { - recordedContextAttributes.add(valueAccessor.getMember(item)); - } - } - } - assertEquals(3, recordedContextAttributes.size()); - assertTrue(recordedContextAttributes.contains("tag1")); - assertTrue(recordedContextAttributes.contains("tag2")); - assertTrue(recordedContextAttributes.contains("tag3")); - - // Verify counters from JFR serialized data (not live process counters which are reset) - Map jfrCounters = new HashMap<>(); - for (IItemIterable counterEvent : verifyEvents("datadog.ProfilerCounter")) { - IMemberAccessor nameAccessor = NAME.getAccessor(counterEvent.getType()); - IMemberAccessor countAccessor = COUNT.getAccessor(counterEvent.getType()); - for (IItem item : counterEvent) { - String name = nameAccessor.getMember(item); - jfrCounters.put(name, countAccessor.getMember(item).longValue()); - } - } - - assertFalse(jfrCounters.isEmpty()); - assertEquals(strings.length, jfrCounters.get("dictionary_context_keys")); - } finally { - // Print statistics about dropped samples for debugging - double dropRate = totalSamplesCount > 0 ? (100.0 * droppedSamplesCount / totalSamplesCount) : 0.0; - double dropWeightRate = totalSamplesWeight > 0 ? (100.0 * droppedSamplesWeight / totalSamplesWeight) : 0.0; - System.out.printf("Sample statistics: %d total (%d dropped, %.2f%%), weight %d total (%d dropped, %.2f%%)%n", - totalSamplesCount, droppedSamplesCount, dropRate, - totalSamplesWeight, droppedSamplesWeight, dropWeightRate); - } - } - - /** - * Reads the current value of {@code tag} via {@link ThreadContext#readContextAttribute} - * — the only readback path retained on the Java side (test-only). - */ - private String readTag(ContextSetter contextSetter, String tag) { - return profiler.getThreadContext().readContextAttribute(contextSetter.offsetOf(tag)); - } - - @Test - public void testSnapshotRestore() throws Exception { - // J9 does not initialize ThreadContext for non-profiled threads; skip. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - // Initially both slots are empty - assertNull(readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - - // Set a value and read it back - assertTrue(contextSetter.setContextValue("tag1", "before")); - assertEquals("before", readTag(contextSetter, "tag1")); - - // Snapshot the string, overwrite, then restore - String saved = readTag(contextSetter, "tag1"); - assertTrue(contextSetter.setContextValue("tag1", "inside")); - assertEquals("inside", readTag(contextSetter, "tag1")); - - // Restore via setContextValue - assertTrue(contextSetter.setContextValue("tag1", saved)); - assertEquals("before", readTag(contextSetter, "tag1")); - - // put/clear/put cycle: verify offset stability across state transitions - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertTrue(contextSetter.setContextValue("tag1", "after")); - assertEquals("after", readTag(contextSetter, "tag1")); - - // tag2 was never set; readContextAttribute returns null - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testAttrsDataOverflow() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - char[] chars = new char[255]; - java.util.Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - int overflowIndex = -1; - for (int i = 1; i <= 10; i++) { - if (!contextSetter.setContextValue("tag" + i, bigValue)) { - overflowIndex = i; - break; - } - } - assertTrue(overflowIndex >= 0, "Expected at least one write to overflow attrs_data"); - assertNull(readTag(contextSetter, "tag" + overflowIndex), - "Overflowed slot must read null — the entry never landed in attrs_data"); - } - - @Test - public void testPutClearsCustomSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "before-put")); - assertEquals("before-put", readTag(contextSetter, "tag1")); - - // setContext() triggers setContextDirect which resets attrs_data_size to the LRS entry only, - // dropping all user attribute entries — so scanning attrs_data for tag1 returns null. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "tag1 must be null after setContext resets attrs_data"); - } - - @Test - public void testCrossSlotIsolation() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - assertTrue(contextSetter.clearContextValue("tag2")); - assertEquals("v1", readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testReapplyByIdAndBytes() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - String value = "app-managed"; - - // Set the attribute the normal way, then capture both the constant ID (sidecar) and the - // UTF-8 bytes — exactly what dd-trace-java retains for the reapply hot path. - assertTrue(contextSetter.setContextValue("tag1", value)); - int[] ids = contextSetter.snapshotTags(); - int savedId = ids[slot]; - assertNotEquals(0, savedId); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = value.getBytes(StandardCharsets.UTF_8); - - // setContext (span activation) wipes both views. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "attrs_data must be wiped by setContext"); - assertEquals(0, contextSetter.snapshotTags()[slot], "sidecar must be wiped by setContext"); - - // Reapply by ID + bytes restores BOTH views. - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals(value, readTag(contextSetter, "tag1"), "attrs_data must be restored"); - assertEquals(savedId, contextSetter.snapshotTags()[slot], "sidecar must be restored"); - } - - @Test - public void testReapplyByIdAndBytesRejectsBadArgs() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "v")); - int id = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id); - - // Null arrays and length mismatch must throw. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(null, new byte[1][])); - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[1], null)); - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[2], new byte[3][])); - - // A slot with constantId > 0 requires non-null bytes within the size limit. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {null, null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after NPE on active utf8[i]"); - - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {new byte[256], null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after IAE on oversized utf8[i]"); - - // 255 bytes is the boundary and must be accepted. - // Register the 255-byte value so its constant ID matches the bytes we pass. - byte[] ok255 = new byte[255]; - Arrays.fill(ok255, (byte) 'x'); - assertTrue(contextSetter.setContextValue("tag1", new String(ok255, StandardCharsets.UTF_8))); - int id255 = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id255); - assertTrue(contextSetter.setContextValuesByIdAndBytes( - new int[] {id255, 0}, new byte[][] {ok255, null})); - } - - @Test - public void testReapplyByIdAndBytesReplacesExistingValue() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Capture the ID + bytes for "first". - assertTrue(contextSetter.setContextValue("tag1", "first")); - int[] idsFirst = contextSetter.snapshotTags(); - byte[][] bytesFirst = new byte[idsFirst.length][]; - bytesFirst[slot] = "first".getBytes(StandardCharsets.UTF_8); - - // Overwrite the live slot with a different value. - assertTrue(contextSetter.setContextValue("tag1", "second")); - assertEquals("second", readTag(contextSetter, "tag1")); - - // Reapply "first" by ID + bytes over the live "second" — exercises the - // compact-then-insert path in replaceOtepAttribute. - assertTrue(contextSetter.setContextValuesByIdAndBytes(idsFirst, bytesFirst)); - assertEquals("first", readTag(contextSetter, "tag1")); - assertEquals(idsFirst[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesAfterClear() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "live")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "live".getBytes(StandardCharsets.UTF_8); - - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertEquals(0, contextSetter.snapshotTags()[slot]); - - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals("live", readTag(contextSetter, "tag1")); - assertEquals(ids[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesClearedRecord() throws Exception { - // Verifies that setContextValuesByIdAndBytes never resurrects a cleared (span-less) record. - // A cleared record has valid=0 and no trace/span context; re-publishing it would expose - // attribute values with no associated trace, which is meaningless to the signal handler. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a live record with tag1 set. - profiler.setContext(1L, 42L, 0L, 43L); - assertTrue(contextSetter.setContextValue("tag1", "will-be-cleared")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "will-be-cleared".getBytes(StandardCharsets.UTF_8); - - // Drive valid=0 via the all-zero clear path (clearContext → put(0,0,0,0) → no attach()). - profiler.clearContext(); - // readContextAttribute respects valid=0 and returns null, confirming the record is dark. - assertNull(readTag(contextSetter, "tag1")); - - // Reapply must return false and must not resurrect the cleared record. - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "setContextValuesByIdAndBytes must return false when the record is cleared (valid=0)"); - assertNull(readTag(contextSetter, "tag1"), - "cleared record must not be resurrected by setContextValuesByIdAndBytes"); - } - - @Test - public void testReapplyByIdAndBytesOverflowRollback() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Register one 255-byte value to obtain a valid constant ID. - char[] chars = new char[255]; - Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - assertTrue(contextSetter.setContextValue("tag1", bigValue)); - int bigId = contextSetter.snapshotTags()[contextSetter.offsetOf("tag1")]; - assertNotEquals(0, bigId); - byte[] bigBytes = bigValue.getBytes(StandardCharsets.UTF_8); - - // Reapply the same 255-byte value to all 10 slots — attrs_data cannot hold them all. - int[] ids = new int[10]; - byte[][] bytes = new byte[10][]; - Arrays.fill(ids, bigId); - Arrays.fill(bytes, bigBytes); - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "10 x 255-byte values must overflow attrs_data"); - - // The last slot certainly overflowed: its sidecar must be zeroed and attrs_data empty. - int lastSlot = contextSetter.offsetOf("tag10"); - assertEquals(0, contextSetter.snapshotTags()[lastSlot], - "overflowed slot's sidecar must be zeroed"); - assertNull(readTag(contextSetter, "tag10"), - "overflowed slot must read null — the entry never landed in attrs_data"); - - // Slots processed before the overflow are durably written — false does not mean - // the record is unchanged. At least tag1 (slot 0) must retain the new value. - int firstSlot = contextSetter.offsetOf("tag1"); - assertEquals(bigId, contextSetter.snapshotTags()[firstSlot], - "slot 0 processed before overflow must have its sidecar durably written"); - assertEquals(bigValue, readTag(contextSetter, "tag1"), - "slot 0 processed before overflow must be readable via attrs_data"); - } - - // ----------------------------------------------------------------------- - // Acceptance tests for the MAX_CUSTOM_SLOTS guard fixes - // ----------------------------------------------------------------------- - - /** - * Test 1: setContextValuesByIdAndBytes must throw IllegalArgumentException immediately when - * the arrays are longer than MAX_CUSTOM_SLOTS (10), and must not perform - * any partial write before the rejection. - */ - @Test - public void testSetContextValuesByIdAndBytesRejectsArraysLongerThanMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a known value and capture its constant ID. - assertTrue(contextSetter.setContextValue("tag1", "original")); - int savedId = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, savedId); - - // Build arrays of length 11 (> MAX_CUSTOM_SLOTS = 10). - int[] ids = new int[11]; - byte[][] utf8 = new byte[11][]; - ids[0] = savedId; - utf8[0] = "original".getBytes(StandardCharsets.UTF_8); - // All other entries remain 0 / null. - - // The call must be rejected with an exception before any write. - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(ids, utf8), - "setContextValuesByIdAndBytes must throw when array length > MAX_CUSTOM_SLOTS"); - - // No partial write: the sidecar for slot 0 must be unchanged. - assertEquals(savedId, contextSetter.snapshotTags()[slot], - "sidecar must not be modified before the length guard fires"); - } - - /** - * Test 2: setContextValuesByIdAndBytes must accept arrays of exactly - * MAX_CUSTOM_SLOTS (10) and return true, restoring all sidecar values. - */ - @Test - public void testSetContextValuesByIdAndBytesAcceptsExactlyMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Set all 10 attributes to distinct values and capture constant IDs + bytes. - int[] savedIds = new int[10]; - byte[][] savedBytes = new byte[10][]; - for (int i = 0; i < 10; i++) { - String value = "val" + i; - assertTrue(contextSetter.setContextValue("tag" + (i + 1), value)); - savedBytes[i] = value.getBytes(StandardCharsets.UTF_8); - } - int[] snapshot = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - savedIds[i] = snapshot[i]; - assertNotEquals(0, savedIds[i], "tag" + (i + 1) + " must have a non-zero sidecar ID"); - } - - // Wipe all slots via setContext (span activation). - profiler.setContext(1L, 42L, 0L, 43L); - - // Reapply with exactly-10-element arrays — must succeed. - assertTrue(contextSetter.setContextValuesByIdAndBytes(savedIds, savedBytes), - "setContextValuesByIdAndBytes must return true for arrays of length == MAX_CUSTOM_SLOTS"); - - // All 10 sidecar IDs must be restored. - int[] restored = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - assertEquals(savedIds[i], restored[i], - "sidecar for tag" + (i + 1) + " must be restored after reapply"); - } - } - - /** - * Test 3: snapshotTags(int[]) with an oversized buffer (length > attributes.size()) - * must write the managed indices [0, attributes.size()) with the current sidecar values, - * and zero out the extra indices [attributes.size(), snapshot.length). - */ - @Test - public void testSnapshotTagsOversizedBufferCopiesAndZerosExtras() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - - // Verify no-arg overload returns valid IDs. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Oversized buffer: length 5 > attributes.size() == 2. - int[] oversized = new int[5]; - Arrays.fill(oversized, -1); - contextSetter.snapshotTags(oversized); - - // Managed indices [0, attributes.size()) must contain the current sidecar values. - assertEquals(canonical[0], oversized[0], - "oversized buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], oversized[1], - "oversized buffer[1] must match no-arg snapshotTags()[1]"); - - // Extra indices [attributes.size(), snapshot.length) must be zeroed. - for (int i = 2; i < oversized.length; i++) { - assertEquals(0, oversized[i], - "oversized buffer element [" + i + "] must be zeroed by snapshotTags"); - } - - // No-arg overload must still work correctly. - int[] check = contextSetter.snapshotTags(); - assertEquals(canonical[0], check[0]); - assertEquals(canonical[1], check[1]); - } - - /** - * Test 4: snapshotTags(int[]) with an undersized buffer (length < attributes.size()) - * must be a no-op — existing no-op semantics must be preserved. - */ - @Test - public void testSnapshotTagsUndersizedBufferIsNoOp() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag3")); - - assertTrue(contextSetter.setContextValue("tag1", "a")); - assertTrue(contextSetter.setContextValue("tag2", "b")); - assertTrue(contextSetter.setContextValue("tag3", "c")); - - // Undersized buffer: length 1 < attributes.size() == 3. - int[] undersized = new int[1]; - undersized[0] = -1; - contextSetter.snapshotTags(undersized); - - assertEquals(-1, undersized[0], - "undersized buffer must not be written by snapshotTags"); - } - - /** - * Test 5: snapshotTags(int[]) with an exact-size buffer (length == attributes.size()) - * must copy the current sidecar values correctly. - */ - @Test - public void testSnapshotTagsExactSizeBufferCopiesCorrectly() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "x")); - assertTrue(contextSetter.setContextValue("tag2", "y")); - - // No-arg overload to obtain expected values. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Exact-size buffer: length 2 == attributes.size() == 2. - int[] exact = new int[2]; - contextSetter.snapshotTags(exact); - - assertEquals(canonical[0], exact[0], - "exact-size buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], exact[1], - "exact-size buffer[1] must match no-arg snapshotTags()[1]"); - } - - private void work(ContextSetter contextSetter, String contextAttribute, String contextValue) - throws InterruptedException { - assertTrue(contextSetter.setContextValue(contextAttribute, contextValue)); - checkTagValues(contextSetter, contextAttribute); - Thread.sleep(10); - assertTrue(contextSetter.clearContextValue(contextAttribute)); - } - - private void checkTagValues(ContextSetter contextSetter, String contextAttribute) { - int[] tags = contextSetter.snapshotTags(); - // expects tag1/tag2/tag3 - change this if the tested tags change - int offset = Integer.parseInt(contextAttribute.substring(3)) - 1; - for (int i = 0; i < tags.length; i++) { - if (i == offset) { - assertNotEquals(0, tags[i]); - } else { - assertEquals(0, tags[i]); - } - } - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter=0,attributes=tag1;tag2;tag3"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java index 2595ed0c6e..a5c7bc1928 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java @@ -32,14 +32,13 @@ public void smokeTest() throws Exception { doThreadFiltering(); } - @SuppressWarnings("deprecation") private void doThreadFiltering() throws Exception { Future[] futures = new Future[1000]; for (int i = 0; i < futures.length; i++) { int id = i; futures[i] = executorService.submit(() -> { profiler.addThread(); - profiler.setContext(id, 42); + profiler.setTraceContext(42, id + 1, 0, id + 1, -1, null, -1, null); try { Thread.sleep(2); } catch(InterruptedException e) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java index 2201f7cac0..845311ca2c 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java @@ -118,11 +118,7 @@ public void testNoSigsegvInWriteStackTracesAfterClassUnload() throws Exception { assertTrue(Files.size(dumpFile) > 0, "Profiler produced no output — SIGSEGV during writeStackTraces is suspected"); } finally { - try { - profiler.stop(); - } finally { - profiler.resetThreadContext(); - } + profiler.stop(); } } finally { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java index 88cc6a472f..03d4a6505f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java @@ -41,14 +41,13 @@ private Task(JavaProfiler profiler) { } @Override - @SuppressWarnings("deprecation") public void run() { - profiler.setContext(1, 2); + profiler.setTraceContext(2, 1, 0, 1, -1, null, -1, null); long now = profiler.getCurrentTicks(); if (profiler.isThresholdExceeded(9, start, now)) { profiler.recordQueueTime(start, now, getClass(), QueueTimeTest.class, ArrayBlockingQueue.class, 10, origin); } - profiler.clearContext(); + profiler.clearTraceContext(); } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java index 43d840b4a4..e646762ecd 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java @@ -134,11 +134,11 @@ public void tracedSleepingThreadIsSampled() throws InterruptedException { Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); registerCurrentThreadForWallClockProfiling(); - profiler.setContext(0x5100L, 0x5101L, 0L, 0x5101L); + profiler.setTraceContext(0x5100L, 0x5101L, 0L, 0x5101L, -1, null, -1, null); try { Thread.sleep(300); } finally { - profiler.clearContext(); + profiler.clearTraceContext(); } stopProfiler(); @@ -186,14 +186,13 @@ public void suppressionCounterIsZeroWhenPrecheckDisabled() throws Exception { } /** - * Recreates the steady state left after a previous test initialized and then removed the Java - * ThreadContext: the native ProfiledThread still owns an initialized OTEP record, but the - * record is cleared and invalid. + * Recreates the steady state left after a previous test activated and then cleared the trace + * context: the native ProfiledThread still owns an initialized OTEP record, but the record is + * cleared and invalid. */ private void leaveClearedInitializedContext() { - profiler.setContext(0x7700L, 0x7701L, 0L, 0x7701L); - profiler.clearContext(); - profiler.resetThreadContext(); + profiler.setTraceContext(0x7700L, 0x7701L, 0L, 0x7701L, -1, null, -1, null); + profiler.clearTraceContext(); } @Override diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java index a2caa1b8a5..7ae94290db 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java @@ -62,7 +62,7 @@ public void precheckAndParkSuppressionWorkTogether() throws Exception { registerCurrentThreadForWallClockProfiling(); long spanId = 0x1111L; long rootSpanId = 0x2222L; - profiler.setContext(rootSpanId, spanId, 0, 0); + profiler.setTraceContext(rootSpanId, spanId, 0, 0, -1, null, -1, null); ready.countDown(); ProfilerOwnedBlockHooks.parkEnter(profiler); long parkedUntil = System.nanoTime() + 280_000_000L; @@ -71,7 +71,7 @@ public void precheckAndParkSuppressionWorkTogether() throws Exception { } ProfilerOwnedBlockHooks.parkExit( profiler, System.identityHashCode(this), 0L); - profiler.clearContext(); + profiler.clearTraceContext(); }, "combined-parked"); diff --git a/doc/architecture/TLSContext.md b/doc/architecture/TLSContext.md index 0a17829dce..caf6988f06 100644 --- a/doc/architecture/TLSContext.md +++ b/doc/architecture/TLSContext.md @@ -10,9 +10,12 @@ with active traces. The system uses OTEL profiling signal conventions ([OTEP #4947](https://github.com/open-telemetry/oteps/pull/4947)) as its -sole context storage format. Java code writes tracing context into -thread-local `DirectByteBuffer`s mapped to native structs. Two consumer -paths read this context concurrently: +sole context storage format. Java code writes tracing context through a +small set of native JNI primitives (`setTraceContext`/`clearTraceContext`/ +`setContextValue`/`clearContextValue`) that resolve the current carrier's +`OtelThreadContextRecord` and mutate it directly — there is no cached +per-thread Java-side buffer. Two consumer paths read this context +concurrently: 1. **DD signal handler (SIGPROF)** — reads integer tag encodings and root-span ID from the sidecar buffer, and span ID from the OTEL record @@ -21,33 +24,27 @@ paths read this context concurrently: `otel_thread_ctx_v1` TLS symbol via ELF dynsym and read the `OtelThreadContextRecord` directly. -All writes from Java are zero-JNI on the hot path (cache-hit case), -using `DirectByteBuffer` with explicit memory ordering. A detach/attach -publication protocol ensures readers see either a complete old record or -a complete new record, never a torn intermediate state. - -For benchmark data, see -[ThreadContext Benchmark Report](../performance/reports/thread-context-benchmark-2026-03-21.md). +Each write resolves the current carrier's record inside a single JNI call — +race-free by construction under virtual-thread migration, since a JNI native +frame pins a mounted virtual thread to its carrier for the call's duration. +A detach/attach publication protocol ensures readers see either a complete +old record or a complete new record, never a torn intermediate state. ## Core Design Principles -1. **Zero-Copy Shared Memory** — Java writes to `DirectByteBuffer`s - mapped to native `ProfiledThread` fields; no data copying between - Java and native heaps. -2. **Signal Handler Safety** — all signal-handler reads use lock-free +1. **Signal Handler Safety** — all signal-handler reads use lock-free atomic loads with acquire semantics; no allocation, no locks, no syscalls. -3. **Detach/Attach Publication Protocol** — the `valid` flag is cleared - before mutation and set after, with `storeFence` barriers between - steps. The TLS pointer is set permanently at thread init. -4. **Two-Phase Attribute Registration** — string attribute values are - registered in the native Dictionary once via JNI; subsequent uses - are zero-JNI ByteBuffer writes from a per-thread encoding cache. -5. **Platform Independence** — correct on both strong (x86/TSO) and - weak (ARM) memory models via explicit `storeFence` / volatile write - barriers. -6. **Low Overhead** — typical span lifecycle write ~30 ns, sidecar - encoding read ~2 ns (no syscalls, no locks). +2. **Detach/Attach Publication Protocol** — the `valid` flag is cleared + before mutation and set after, with release fences between steps. The + TLS pointer is set permanently at thread init. +3. **Two-Phase Attribute Registration** — string attribute values are + registered in the native Dictionary once via JNI (`ContextValueCache`); + subsequent uses of the same value reuse the cached encoding. +4. **Platform Independence** — correct on both strong (x86/TSO) and + weak (ARM) memory models via explicit release fences. +5. **Low Overhead** — one JNI call per context operation; no cached buffer + to invalidate or reattach across thread migration. ## Architecture @@ -58,30 +55,29 @@ For benchmark data, see │ Application Thread │ ├─────────────────────────────────────────────────────────────────────┤ │ │ -│ Tracer calls ThreadContext.put(lrs, spanId, trHi, trLo) │ +│ Tracer calls JavaProfiler.setTraceContext(lrs, spanId, trHi, trLo, │ +│ slot0, v0, slot1, v1) │ │ │ │ │ ▼ │ │ ┌───────────────────────────────────────────────────────────────┐ │ -│ │ setContextDirect() │ │ -│ │ 1. detach() — valid ← 0, storeFence │ │ -│ │ 2. ctxBuffer.putLong(traceIdOffset, reverseBytes(trHi)) │ │ -│ │ ctxBuffer.putLong(traceIdOffset+8, reverseBytes(trLo)) │ │ -│ │ ctxBuffer.putLong(spanIdOffset, reverseBytes(spanId)) │ │ +│ │ setTraceContext0() (single JNI call) │ │ +│ │ 1. detach() — valid ← 0, release fence │ │ +│ │ 2. record->trace_id/span_id ← big-endian encode │ │ │ │ 3. tag_encodings[0..9] ← 0 │ │ │ │ attrs_data_size ← LRS_ENTRY_SIZE (keeps fixed LRS at [0]) │ │ -│ │ 4. ctxBuffer.putLong(lrsOffset, lrs) │ │ -│ │ writeLrsHex(lrs) — update fixed LRS entry in attrs_data │ │ -│ │ 5. attach() — storeFence, valid ← 1 │ │ +│ │ 4. sidecar local_root_span_id ← lrs; write LRS hex entry │ │ +│ │ 5. write slot0/slot1 activation attributes (if any) │ │ +│ │ 6. attach() — release fence, valid ← 1 │ │ │ └───────────────────────────────────────────────────────────────┘ │ │ │ │ │ ▼ │ │ ┌──────────────────────────────────────────────────────────────┐ │ -│ │ Unified ctxBuffer (688B, single DirectByteBuffer) │ │ +│ │ ProfiledThread (native, per-thread heap allocation) │ │ │ │ ┌──────────────────────┐ ┌───────────────────────────┐ │ │ │ │ │ OtelThreadContextRec │ │ tag_encodings[10] (u32) │ │ │ │ │ │ trace_id[16] (BE) │ │ local_root_span_id (u64) │ │ │ │ │ │ span_id[8] (BE) │ └───────────────────────────┘ │ │ -│ │ │ valid (u8)│ offsets 640..688 in ctxBuffer │ │ +│ │ │ valid (u8)│ contiguous sidecar region │ │ │ │ │ reserved (u8)│ │ │ │ │ │ attrs_data_size(u16)│ ┌──────────────────────────────┐ │ │ │ │ │ attrs_data[612] │ │ TLS pointer (8B) │ │ │ @@ -105,25 +101,16 @@ For benchmark data, see ├──────────────────────────────────────────────────────────────────────┤ │ │ │ JavaProfiler │ -│ ├─ ThreadLocal tlsContextStorage │ -│ ├─ initializeContextTLS0(long[] metadata) → ByteBuffer (688B) │ -│ └─ registerConstant0(String value) → int encoding │ -│ │ -│ ThreadContext (per thread) │ -│ ├─ ctxBuffer (688B DirectByteBuffer — record + sidecar contiguous)│ -│ ├─ put(lrs, spanId, trHi, trLo) → setContextDirect() │ -│ ├─ setContextAttribute(keyIdx, value) → setContextAttributeDirect │ -│ ├─ snapshot(byte[], int) / restore(byte[], int) ← nested scopes │ -│ └─ Per-thread caches: │ -│ └─ attrCache[CACHE_SIZE]: String → {int encoding, byte[] utf8}│ +│ ├─ setTraceContext(lrs, spanId, trHi, trLo, slot0, v0, slot1, v1) │ +│ ├─ clearTraceContext() │ +│ ├─ setContextValue(slot, value) / clearContextValue(slot) │ +│ └─ contextValueCache: ContextValueCache │ │ │ -│ BufferWriter (memory ordering abstraction) │ -│ ├─ BufferWriter8 (Java 8: Unsafe) │ -│ │ ├─ putOrderedLong / putOrderedInt │ -│ │ └─ storeFence → Unsafe.storeFence() │ -│ └─ BufferWriter9 (Java 9+: VarHandle) │ -│ ├─ setRelease │ -│ └─ storeFence → VarHandle.storeStoreFence() │ +│ ContextValueCache (process-wide, shared across every carrier / │ +│ virtual thread — no per-thread instance) │ +│ ├─ resolve(value) → {encoding, utf8} │ +│ ├─ direct-mapped table keyed by value.hashCode() & 0xFF │ +│ └─ registerConstant0(String value) → int encoding (JNI) │ └──────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────┐ @@ -179,7 +166,6 @@ Offset Size Field Description Total: 48 bytes ``` -The sidecar fields are exposed to Java as a single `DirectByteBuffer`. Tag encodings are integer IDs from the profiler's `Dictionary` constant pool — the signal handler writes them directly into JFR events without any string lookup. @@ -202,7 +188,7 @@ Each entry in `attrs_data` is encoded as: ### Problem Two concurrent readers may observe the record at any point during a -Java-side mutation: +native-side mutation: 1. **SIGPROF signal handler** — interrupts the writing thread mid-sequence, runs on the same thread. @@ -215,28 +201,31 @@ a partially-written record. ### Protocol ``` -Java writer timeline: +Native writer timeline (setTraceContext0 / setContextValue0): ────────────────────────────────────────────────────────────────── Time 0: detach() - ctxBuffer.put(validOffset, 0) ← mark invalid - storeFence() ← drain store buffer + record->valid ← 0 ← mark invalid + release fence ← drain store buffer Time 1: Mutate record fields - ctxBuffer.putLong(traceIdOffset, ...) - ctxBuffer.putLong(spanIdOffset, ...) - tag_encodings[0..9] ← 0 ← zero tag encodings (offsets 640..680) - attrs_data_size ← LRS_ENTRY_SIZE ← keep only fixed LRS entry at attrs_data[0] - ctxBuffer.putLong(lrsOffset, lrs) ← update LRS at offset 680 - writeLrsHex(lrs) ← update LRS hex entry in attrs_data + record->trace_id / span_id ← big-endian encode + tag_encodings[0..9] ← 0 ← zero tag encodings (setTraceContext0 only) + attrs_data_size ← LRS_ENTRY_SIZE ← keep only fixed LRS entry at attrs_data[0] + sidecar local_root_span_id ← lrs ← update LRS sidecar field + writeLrsHex(lrs) ← update LRS hex entry in attrs_data ⚡ SIGPROF may arrive here — handler sees valid=0, skips record Time 2: attach() - storeFence() ← ensure writes visible - ctxBuffer.put(validOffset, 1) ← mark valid + release fence ← ensure writes visible + record->valid ← 1 ← mark valid ────────────────────────────────────────────────────────────────── ``` +`clearContextValue0` is the one exception: it preserves whatever `valid` +value it found (removing a single attribute must not resurrect a record +that a preceding `clearTraceContext0()` intentionally left detached). + ### Reader: DD Signal Handler ```cpp @@ -267,7 +256,7 @@ for (int i = 0; i < 8; i++) { val = (val << 8) | record->span_id[i]; } span_id = val; ``` -The acquire fence pairs with the writer's `storeFence` + `valid=1` +The acquire fence pairs with the writer's release fence + `valid=1` sequence, ensuring all record field writes are visible if `valid` reads as 1. @@ -292,74 +281,41 @@ different reasons: - The **signal handler** runs on the same thread as the writer. The CPU presents its own stores in program order, so CPU store-buffer reordering is not a - concern. The JIT compiler can still reorder stores arbitrarily, so a compiler + concern. The compiler can still reorder stores arbitrarily, so a compiler barrier is required. - The **external OTEP profiler** (e.g. eBPF using scheduler events) attaches a `sched_switch` tracepoint that fires on the same CPU that was executing the thread. The Linux scheduler acquires `rq_lock` before the tracepoint fires, which includes a full hardware memory barrier (`smp_mb__before_spinlock` on ARM). By the time the eBPF probe runs, all prior user-space stores from that - thread are globally visible — including any stores ordered by `DMB ISHST`. - -In both cases `storeFence` serves as a compiler barrier that prevents the JIT -from sinking record field writes past the `valid=1` store. On ARM it also emits -`DMB ISHST`, which is required to order field writes before `valid=1` at the -hardware level — this is not a mere side effect. - -### Barrier Taxonomy - -| Operation | Java 8 | Java 9+ | ARM | x86 | -|-----------|--------|---------|-----|-----| -| `storeFence` | `Unsafe.storeFence` | `VarHandle.storeStoreFence` | DMB ISHST (~2 ns) | compiler barrier (free) | - -On x86, `storeFence` is a compiler-only barrier (TSO guarantees hardware -store ordering for free; Java 9+ `VarHandle.storeStoreFence` emits no -hardware instruction on x86). On ARM it compiles to `DMB ISHST` (~2 ns). - -### Why storeFence, Not fullFence + thread are globally visible. -The detach/attach protocol only requires store-store ordering — all -operations on the hot path are writes. There are no load-dependent -ordering requirements on the writer side. `storeFence` (~2 ns on ARM) -is sufficient; a full fence (~50 ns on ARM) would be wasteful. +In both cases the release fence prevents the compiler from sinking record +field writes past the `valid=1` store, and on ARM emits the hardware barrier +required to order field writes before `valid=1`. ## Initialization -### Per-Thread TLS Setup +### Self-Initializing on First Write -When a thread first accesses its `ThreadContext` via the `ThreadLocal`: - -```java -// JavaProfiler.initializeThreadContext() -long[] metadata = new long[6]; -ByteBuffer buffer = initializeContextTLS0(metadata); -return new ThreadContext(buffer, metadata); -``` - -The native `initializeContextTLS0` (in `javaApi.cpp`): +There is no separate TLS-initialization call. The first `setTraceContext0` +or `setContextValue0` on a thread that hasn't written context before: 1. Gets the calling thread's `ProfiledThread` (creates one if needed). 2. Sets `otel_thread_ctx_v1` permanently to the thread's `OtelThreadContextRecord` (triggering TLS slot init on musl). -3. Fills the `metadata` array with absolute offsets into the unified - buffer (computed via `offsetof` for record fields; `OTEL_MAX_RECORD_SIZE - + DD_TAGS_CAPACITY*sizeof(u32) = 680` for the LRS offset), so Java code - writes to the correct positions regardless of struct packing changes. -4. Creates a single `DirectByteBuffer` spanning the contiguous 688-byte - region: `_otel_ctx_record` (640 B) followed immediately by - `_otel_tag_encodings` (40 B) and `_otel_local_root_span_id` (8 B). - Contiguity is enforced by `alignas(8)` on `_otel_ctx_record` plus - `sizeof(OtelThreadContextRecord)` being a multiple of 8. -5. Returns the single buffer. - -This is the only JNI call in the initialization path. After this, all -hot-path operations are pure Java ByteBuffer writes into offset regions -of the one buffer. +3. Marks the thread's context initialized, so `ContextApi::get()` and the + wall-clock sampler's precheck start considering the record. + +A thread that only ever calls the all-native write API is therefore fully +visible to both the DD signal handler and external OTEP readers from its +very first write — no priming call is needed. ### Signal-Safe TLS Access -Signal handlers cannot call `initializeContextTLS0` (it may allocate). The -read path uses a pre-initialized pointer: +Signal handlers never call the write primitives (they may allocate on the +Dictionary-registration path). The read path uses a pre-initialized +pointer: ```cpp // ProfiledThread::currentSignalSafe() — no allocation, no TLS lazy init @@ -371,51 +327,32 @@ if (thrd == nullptr || !thrd->isContextInitialized()) { ## Two-Phase Attribute Registration -String attributes are set via `ThreadContext.setContextAttribute(keyIndex, value)`. -The hot path avoids JNI by splitting the work into two phases: +String attributes are set via `JavaProfiler.setContextValue(slot, value)` +(or as activation attributes via `setTraceContext`). Registration is split +into two phases so that repeated values avoid a JNI call: ### Phase 1: Registration (cache miss) -On the first call with a new string value: - -```java -encoding = registerConstant0(value); // JNI → Dictionary lookup -utf8 = value.getBytes(UTF_8); // one allocation, cached -attrCacheEncodings[slot] = encoding; -attrCacheBytes[slot] = utf8; -attrCacheKeys[slot] = value; // cache is per-thread; no fence needed -``` - -`registerConstant0` crosses JNI once to register the value in the -native `Dictionary` and returns an integer encoding. +On the first call with a new string value, `ContextValueCache.resolve` +crosses JNI once via `registerConstant0` to register the value in the +native Dictionary and cache the resulting `{encoding, utf8}` pair, +keyed by `value.hashCode() & 0xFF`. -### Phase 2: Cached Write (cache hit, zero JNI) +### Phase 2: Cached Resolution (cache hit) -On subsequent calls with the same string: - -```java -if (value.equals(attrCacheKeys[slot])) { - encoding = attrCacheEncodings[slot]; // int read - utf8 = attrCacheBytes[slot]; // ref read -} -// Both sidecar and OTEP attrs_data are written inside the detach/attach window -// so a signal handler never sees a new sidecar encoding alongside old attrs_data. -detach(); -ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * 4, encoding); -replaceOtepAttribute(otepKeyIndex, utf8); -attach(); -``` - -The cache is a 256-slot direct-mapped structure keyed by -`value.hashCode() & 0xFF`. Collisions evict the old entry (benign — -causes a redundant `registerConstant0` call). In production web -applications with 5–50 unique attribute values, the hit rate is -effectively 100%. +On subsequent calls with the same string, `ContextValueCache.resolve` +returns the cached `{encoding, utf8}` pair with no JNI call; the caller +still crosses JNI once to write the value into the current record (there +is no per-thread cached buffer to write into directly). Collisions in the +256-slot direct-mapped cache evict the old entry (benign — causes a +redundant `registerConstant0` call on the next use of the evicted value). +In production web applications with 5–50 unique attribute values, the hit +rate is effectively 100%. ## Signal Handler Read Path `Recording::writeCurrentContext()` executes in the SIGPROF handler and -reads context in bounded time (~15 ns) with no allocation: +reads context in bounded time with no allocation: 1. `ContextApi::get(spanId, rootSpanId)`: - `ProfiledThread::currentSignalSafe()` — cached pointer, no TLS @@ -432,73 +369,43 @@ reads context in bounded time (~15 ns) with no allocation: No dictionary lookup, no string comparison, no allocation. The encodings written to JFR events are resolved later during JFR parsing. -## Performance - -### Write Path Costs (arm64, Java 25) - -| Operation | ns/op | Path | -|-----------|------:|------| -| `clearContext` | 5.0 | detach + zero fields | -| `setContextFull` | 11.1 | detach + 3 putLong + attach | -| `setAttrCacheHit` | 10.7 | cache lookup + sidecar write + detach/attach | -| `spanLifecycle` | 30.4 | `setContextFull` + `setAttrCacheHit` | - -### Multi-Threaded Scaling - -| Benchmark | 1 thread | 2 threads | 4 threads | -|-----------|----------|-----------|-----------| -| `setContextFull` | 11.1 ns | 11.1 ns | 11.7 ns | -| `spanLifecycle` | 30.4 ns | 30.7 ns | 32.2 ns | - -No false sharing: each thread's `OtelThreadContextRecord` and sidecar -are embedded in its own heap-allocated `ProfiledThread`. - -### Instrumentation Budget - -At ~35 ns per span (`spanLifecycle` 30.4 ns + `clearContext` 5.0 ns), a single thread -can sustain ~28 million span transitions per second. For a web -application at 100K requests/second, this is <0.004% of CPU time. - -Full benchmark data and analysis: -[thread-context-benchmark-2026-03-21.md](../performance/reports/thread-context-benchmark-2026-03-21.md) - ## Testing ### Integration Tests (Java) -`ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java`: +`ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java`: + +- Verifies `setTraceContext`/`clearTraceContext`/`setContextValue`/ + `clearContextValue` round-trip correctly, using package-private + `JavaProfiler.test*` accessors (read via reflection) as the read oracle — + each wraps a native primitive that reads the current thread's + `OtelThreadContextRecord` directly, with no cached buffer involved. +- Covers span-transition attribute reset, attribute TLV encoding/overflow, + and thread isolation. + +`ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java`: -- **testOtelStorageModeContext** — context round-trips correctly with JFR running. -- **testOtelModeCustomAttributes** — verifies attribute TLV encoding in - `attrs_data` via `setContextAttribute`. -- **testOtelModeAttributeOverflow** — overflow of `attrs_data` is handled - gracefully (returns false, no crash). -- **testSequentialContextUpdates** — repeated writes with varying values, - `Long.MAX_VALUE` round-trip, and `clearContext` resetting both IDs to zero. -- **testThreadIsolation** — concurrent writes from multiple threads, - validating thread-local isolation. -- **testSpanTransitionClearsAttributes** — direct span-to-span transition - without `clearContext` does not leak custom attributes from the previous span. +- Regression test that the all-native write path self-initializes OTEL TLS + on first write — a thread that never calls anything but the all-native + API must still be visible to the wall-clock sampler. `ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java`: -- **test** — validates context propagation through wall-clock profiling +- Validates context propagation through wall-clock profiling samples and JFR event correlation across cstack modes. -`ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java`: +`ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java`: -- **test** — validates integer tag/attribute context propagation through - profiling samples. +- Validates `ContextValueCache` resolution, hash-collision eviction, + oversized-value rejection, and `clear()` semantics directly. ### JMH Benchmarks -`ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java`: +`ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java`: -- Single-threaded: `setContextFull`, `setAttrCacheHit`, `spanLifecycle`, - `clearContext`, `getContext`. -- Multi-threaded: `setContextFull_2t/4t`, `spanLifecycle_2t/4t` — - `@Threads(2)` and `@Threads(4)` variants to verify linear scaling - (absence of false sharing). +- Perf-guard for the full per-scope activate+deactivate cycle + (`setTraceContext`+`clearTraceContext`) as dd-trace-java drives it, on + both platform and mounted virtual threads. ## OTEP References diff --git a/doc/plans/2026-07-02-all-native-context-storage-design.md b/doc/plans/2026-07-02-all-native-context-storage-design.md deleted file mode 100644 index 7b299b94b7..0000000000 --- a/doc/plans/2026-07-02-all-native-context-storage-design.md +++ /dev/null @@ -1,316 +0,0 @@ -# Design note: all-native OTEL context storage (eliminating the carrier-owned DirectByteBuffer) - -- **Status:** Proposal (for review by java-profiler + dd-trace-java owners) -- **Date:** 2026-07-02 -- **Related:** PROF-15271 (carrier-scoping fix, commit `94686da`), dd-trace-java #11646 (reapply-on-mount) -- **Scope:** cross-repo — `java-profiler` (this repo) and `dd-trace-java` - -## TL;DR - -The OTEL trace-context write path assumes native OS threads map 1:1 to Java threads. Virtual -threads break that assumption and cause a use-after-free that can crash the JVM. The shipped -carrier-scoping fix (`94686da`) shrinks the crash window but does not close it. - -This note proposes making every context write go through a **native (JNI) call that resolves the -current carrier inside the call** — where the mounted virtual thread is *pinned to its carrier for -the duration by the native frame*, making the write **race-free by construction**. This deletes the -`DirectByteBuffer` / `CarrierThreadLocal` / reflection machinery entirely rather than adding more to -it. - -A JMH campaign (six benchmark classes, JDK 21, glibc x64) establishes the key result: the all-native -path is a **net win on the real per-scope-activation cycle** — but only when the per-activation -context is written in **one combined call**. With the current fine-grained call sequence it would -regress ~1.6×. The recommendation is therefore all-native **plus a combined per-activation API**, -rolled out via expand → migrate → contract. - -The size of the win depends on which DBB baseline it is measured against (see §4 for the full -breakdown and the two figures): - -- Against the **prototype microbenchmark** baseline (§4), which caches the `ThreadContext` handle and - so pays the per-call `currentContext()` / `CarrierThreadLocal` lookup only once, the combined - native cycle is **~7–10% faster** (128 vs 138 ns). -- Against the **production API** baseline — the shipped `ContextCombinedBenchmark`, where the DBB - cycle is the six real `setContext` / `setContextAttribute` / `clearContextAttribute` calls - dd-trace-java actually makes, each re-resolving `currentContext()` — the combined native cycle - (two JNI calls, no per-call lookup) is **~28% faster** (~136 vs ~189 ns). - -Both are the same effect from opposite ends: the fewer JNI transitions the combined call needs, the -more of the DBB path's repeated per-call lookup cost it removes. The ~28% figure is the one that -reflects real tracer usage; the ~7–10% is the conservative lower bound with the lookup amortized away. - -Consumer audit: the entire public context API has **exactly one production consumer** (the -dd-trace-java profiler bridge) and no in-repo external consumers, so the migration risk is internal -coordination only. - ---- - -## 1. Background & problem - -The OTEL context record (`OtelThreadContextRecord`, 640 B + tag-encoding sidecar + LRS = 688 B) is -embedded in the **carrier's** native `ProfiledThread`. It is written by Java (through a -`DirectByteBuffer` "window" over that native memory) and read by the sampler (an async-signal -handler bound to the carrier OS thread, via `ProfiledThread::getOtelContextRecord()` — the sampler -never touches the Java buffer). - -The write path was designed for platform threads, where a Java thread == a native thread for life. -Under Loom this breaks: - -- A `DirectByteBuffer` cached per *virtual* thread is bound to whatever carrier was mounted at first - use. When the vthread migrates, writes land on the wrong carrier (misattribution — the - "split-context" problem). -- Worse: when the original carrier's OS thread exits, its `ProfiledThread` is `delete`d - (`thread.cpp` `freeKey`), freeing the record. The dangling buffer keeps being written — a - **use-after-free** into memory the JVM has reused, observed as a SIGSEGV in - `ThreadsSMRSupport::free_list`. - -**What shipped (`94686da`, PROF-15271):** context storage was moved to -`jdk.internal.misc.CarrierThreadLocal`, so a mounted vthread resolves to its *current* carrier's -record per call. This eliminates the steady-state misattribution and shrinks the UAF window. - -**Why it's incomplete:** there is still a TOCTOU window between resolving the buffer and the write -completing. Today that window is effectively closed on the write path (the writes are plain -`ByteBuffer` ops with no yield point, and JDK 21 does not preemptively unmount vthreads at arbitrary -safepoints). But it reopens if/when Loom adds general preemptive scheduling — which is on the -roadmap. We want a fix that is correct by construction, not one that depends on current scheduler -behavior. - ---- - -## 2. Core insight - -Two observations reframe the problem: - -1. **Crash-safety is about record *lifetime*, not write *timing*.** A write landing on the "wrong" - carrier is only misattribution (non-fatal). The *only* fatal case is writing into a record whose - backing `ProfiledThread` has been freed. So the fix must ensure the write never targets freed - memory — either by never freeing it (arena) or by resolving a live target at write time (native). - -2. **A native frame pins a mounted virtual thread to its carrier.** JDK's freeze logic returns - `freeze_pinned_native` the moment it walks onto a native/JNI frame - (`continuationFreezeThaw.cpp`). So *while a JNI call executes, the vthread cannot unmount* — not - by voluntary yield, not by forced preemption. A native method that resolves - `ProfiledThread::current()` and writes the record therefore targets the **live current carrier**, - with no cached pointer and no migration window. Race-free by construction. - -This is why "all-native" is not just a performance question — it is the *most correct* design: it -removes the cached-buffer concept that caused the bug. - ---- - -## 3. Options considered - -| Option | Idea | Verdict | -|---|---|---| -| **A. Arena records** | Allocate records from a slab never returned to the allocator; `freeKey` recycles a slot instead of freeing memory. Stale writes then hit valid profiler memory (UAF → bounded misattribution). | Viable fallback. Keeps the fast zero-JNI DBB writes and is crash-safe with **no tracer change** — but adds an allocator (the "buffer management" we want to avoid) and keeps `CarrierThreadLocal`. | -| **B. All-native writes** | Every context write is a JNI call that resolves the current carrier (pinned by the native frame) and writes in C. | **Recommended.** Race-free by construction; deletes the DBB/CTL/reflection subsystem. Net perf win with a combined API (§4–5). | -| **C. JVMTI mount/unmount events** | Use `VirtualThreadMount/Unmount` extension events to refresh a cached buffer on mount. | Rejected. Cannot close the TOCTOU (the event can't reach a value already loaded into a Java frame), and enabling `can_support_virtual_threads` imposes a process-wide slow-path tax on *all* vthread transitions. | -| **D. In-buffer handshake** | A lock/generation word in the record; writer acquires it, `freeKey` waits for release before freeing. | Rejected. The lock lives in the memory being freed (bootstrap UAF: you can't safely touch the lock to learn the object is dead), and making `freeKey` wait converts the safety bug into a liveness stall under preemption. Collapses into "A + seqlock." | -| **E. Reflective pinning** (`disableSuspendAndPreempt`) | Borrow the JVM's own `notifyJvmtiDisableSuspend` + `Continuation.pin` to make the write critical-section unmount-proof. | Rejected. `notifyJvmtiDisableSuspend` is a private, boolean-**toggle**-with-assert (not a nesting counter) — driving it from outside `java.lang` risks corrupting per-thread VM state; unstable internals; per-call overhead on a hot path. | - -Option B is chosen because it is simultaneously the most-correct (native-frame pinning), the -lowest-maintenance (deletes code rather than adding an allocator), and — per the benchmarks — at -least perf-neutral on the real production surface. - ---- - -## 4. Benchmark evidence - -Full campaign: JDK 21 (`21.0.10`), glibc linux-x64, JMH `AverageTime` ns/op, 3 forks × 3+3 iters, -carrier vs thread storage mode confirmed per fork (no silent fallback). Prototypes are -benchmark-only (`*Native0` methods, not wired into the production API). Numbers are carrier mode -unless noted; treat as order-of-magnitude (per-fork JIT variance is ~1–6 ns on the sub-50 ns paths). - -**Component costs (corrected from initial estimates):** - -- `CarrierThreadLocal.get()` ≈ **3.9 ns** (plain `ThreadLocal` ≈ 3.3 ns) — the lookup is cheap, not - the ~10–25 ns first guessed. Carrier-scoping added ~no per-call cost. -- A Java→native JNI transition ≈ **~15 ns** (backed out from the scalar native path). So a - transition is *more* expensive than the TL lookup, not less. -- The DBB `setContext` *write* ≈ **18.8 ns** — ~30 bounds-checked `ByteBuffer` ops + 2 fences. This - (not the lookup) is where the DBB scalar cost lives. - -**Path-by-path (DBB vs all-native):** - -| Path | DBB | all-native | Verdict | -|---|---|---|---| -| scalar `setContext` (per span) | ~30 | ~27 | native wins ~11% | -| single `setContextAttribute` (cache hit) | **28.5** (zero-JNI) | 59.0 | native **loses ~2.1×** | -| single `clearContextAttribute` (fair) | **9.4** (zero-JNI) | 21.0 | native **loses ~2.2×** | -| batch reapply, 5 attrs, per-slot | 337 | 480 | native loses (object-array JNI) | -| batch reapply, 5 attrs, **flattened** | 337 | **81** precomp / 109 rebuild | native wins **3–4×** | -| `snapshot`+`restore` (688 B pair) | **45** (inlined memcpy) | 137 | native **loses ~3×** | -| **combined activate+deactivate cycle** | **138.5** | **128.3** | **native wins ~7% (carrier), ~10% (thread)** | - -Sub-results for the combined cycle: native *activate* alone loses (105.8 vs 73.5 — one combined call -vs 3 zero-JNI writes), but native *deactivate* wins decisively (~22 ns for one `clearFullContext` vs -~65 ns for `setContext(0,0,0,0)` + 2× `clearContextAttribute`); the full cycle nets in native's -favor. vthread mirrors platform (native 131.3 vs DBB 140.8), with a ~1–3 ns mount penalty; native is -mount- and mode-independent throughout. - -> **Prototype vs production baseline (why the PR quotes ~28%, not ~7–10%).** The `138.5` DBB figure -> above is the *prototype* microbenchmark: it caches the `ThreadContext` handle, so the DBB cycle -> pays the `currentContext()` / `CarrierThreadLocal` lookup only once. The shipped perf-guard, -> `ContextCombinedBenchmark`, instead drives the *production* API — the six real -> `setContext`/`setContextAttribute`/`clearContextAttribute` calls dd-trace-java makes, each -> re-resolving `currentContext()` — so its DBB baseline is higher (~189 ns) and the combined native -> cycle (two JNI calls, no repeated lookup) comes out **~28% faster** (~136 ns; ~24% in thread mode). -> Same effect, two baselines: ~7–10% is the conservative lower bound with the lookup amortized away, -> ~28% is the figure against real tracer usage. `ContextCombinedBenchmark` is the one to trust going -> forward; the prototype rows are retained for the path-by-path decomposition only. - -**The unifying principle (holds across all six classes):** - -> All-native **wins** when one JNI call replaces *multiple* DBB operations; it **loses** when it adds -> a JNI transition to a *single* operation the DBB already does zero-JNI. - -This is why batching matters: the per-activation context (4 longs + a couple of named attributes) is -a natural batch, so one combined native call amortizes the transition and beats the fine-grained DBB -sequence. Conversely, `snapshot/restore` is already a single inlined `memcpy` on the DBB side, so -native can only add cost there — but (see §6) that path is unused in production. - ---- - -## 5. Proposed design - -**Make context writes all-native, and expose a combined per-activation API.** - -Production per scope activation/deactivation (verified in §6) is: - -- activate: `setContext(rootSpan, span, trHi, trLo)` + `setContextValue(spanName)` + - `setContextValue(resourceName)` -- deactivate: `setContext(0,0,0,0)` + 2× `clearContextValue` - -Replace this fine-grained sequence with two batched native calls. The activation attributes are a -*variable, config-derived* set, so the API must not hardcode which/how many attributes there are: -dd-trace-java's attribute list is configurable (`orderedContextAttributes` = the `attributes=` -config plus appended `operation`/`resource`), and the operation/resource **slot offsets come from -`offsetOf(...)`, not fixed indices**. Today activation writes exactly two span-derived attributes -(operation + resource — the only values available from the `ProfilerContext`; other configured -attributes like `http.route` are set outside activation by instrumentations and are wiped per -activation so they don't leak between spans). So the combined call carries the scalar context plus a -small set of `(offset, value)` pairs: - -``` -// activate: one JNI call writes scalar context + N span-derived (offset,value) attributes, -// under one detach/attach. Explicit-arity overload for the common case avoids per-call allocation: -void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long traceIdLow, - int off0, CharSequence val0, int off1, CharSequence val1); -// deactivate: one JNI call clears -void clearTraceContext(); -``` - -- **Do not** hardcode `spanName`/`resourceName` or their slots — pass explicit offsets (the tracer - gets them from `operationNameOffset()`/`resourceNameOffset()`), so the API tracks the configurable - attribute model and survives a future third span-derived attribute (service, span kind). -- **Avoid per-activation allocation.** Prefer allocation-free explicit-arity overloads (the 2-pair - form above matches today exactly) over a naive `int[]`/`Object[]` variant, which would add GC - pressure on the activation hot path. If the span-derived attribute count ever grows past what - overloads cover, add a general array form backed by a tracer-reused scratch buffer. -- The `CharSequence → (encoding, utf8)` value cache (today inside `ThreadContext`) moves to a small - Java-side helper in `java-profiler` that resolves each value and invokes the native call — keeping - the OTEP record layout and the cache encapsulated; the tracer passes offsets + values. -- App-driven, sporadic attributes (e.g. `http.route`, set by instrumentations after activation) - keep a single-attribute native setter. This is the one path where native is ~2× the zero-JNI DBB - write — accepted because it is *not* per-activation (see §7). -- Reads (`getSpanId`, etc.) become JNI calls (~20 ns) — fine, they are cold/test-only (§6). -- The sampler is unchanged: it already reads the native record directly, not via the DBB. -- The OTEP record layout and the `otel_thread_ctx_v1` discovery symbol are unchanged, so external / - eBPF readers are unaffected. - -Why combined rather than keeping the fine-grained API all-native: the benchmarks show the -fine-grained all-native cycle regresses ~1.6× (~6 transitions/cycle), while the combined cycle wins -~7–10% (2 transitions/cycle). Batching is what makes all-native a win rather than a regression. - ---- - -## 6. Consumer audit (de-risking removal) - -Callers of the public context API were enumerated across `java-profiler` (all modules) and -`dd-trace-java`. Result: - -- **Exactly one production consumer:** the dd-trace-java bridge - (`DatadogProfilingIntegration` / `DatadogProfiler` / `ContextSetter`), which calls only - `setContext(4-arg)` and single `setContextValue`/`clearContextValue`. -- **No other production consumers** in either repo. No in-repo non-Datadog consumers. Classes live - in `com.datadoghq.profiler`; the jar is bundled into `dd-java-agent` (no `module-info` exports - found). -- **Verified unused by production** (contradicting an earlier automated pass): the batch setter - `setContextAttributesByIdAndBytes` (grep-empty outside tests), `DatadogProfiler.snapshot()` (the - `int[]` tag snapshot — no production callers), and `ThreadContext.snapshot/restore` + java-profiler's - `ScopeStack` (dd-trace-java has its own `ScopeStack` and re-derives parent context by re-calling - `setContext`; it never calls the profiler's snapshot/restore). -- **Test/benchmark-only:** `getThreadContext`, `resetThreadContext`, `contextStorageMode`, - `readContextAttribute`, `readTraceId`, `getSpanId`, `getRootSpanId`. - -**Removal risk:** entirely internal Datadog coordination (java-profiler ↔ dd-trace-java). No -external breakage identified. Caveat: if `ddprof-lib` is published as a standalone Maven artifact -that external users could compile against, add a deprecation window as insurance (§7). - -Consequence for the design: the `snapshot/restore` ~3× regression and the per-slot batch regression -are **moot** — neither path is on the production tracer surface. The only production paths that -matter are the ones the combined API covers, plus the sporadic single app-attribute. - ---- - -## 7. Migration plan (expand → migrate → contract) - -1. **Expand (java-profiler).** Add the all-native combined API (`setTraceContext` / - `clearTraceContext`), the single-attribute native setter for app tags, and native reads, - alongside the existing DBB APIs. Mark the DBB context methods `@Deprecated` pointing at the new - API. Release. Additive and backward-compatible — bumping to this version changes nothing - operationally. -2. **Migrate (dd-trace-java).** Switch `DatadogProfilingIntegration` to the combined API. Requires - java-profiler ≥ the phase-1 version. **This is where the crash fix and the perf win land.** -3. **Contract (java-profiler).** Once all consumers are on the new API, delete the - DBB / `CarrierThreadLocal` / `OtelContextStorage` subsystem (and, if desired, the unused - `ScopeStack` / `snapshot` / `restore` / batch-setter). Breaking release, gated on a - consumers-migrated confirmation. - -**Sequencing notes / caveats:** - -- **The crash fix lands in phase 2, not phase 1.** Phase 1 is purely additive; the UAF persists on - the old DBB path until the tracer switches. Until then, carrier-scoping (`94686da`) remains the - mitigation. If crash urgency is high, phase 2 is the milestone. -- **Coexistence is safe.** Old (DBB) and new (native) paths write the *same* underlying record; the - native path ignores the DBB; the tracer switches wholesale, so no thread interleaves both. No - dual-write coherence problem. -- **aarch64 memory model.** The native detach/attach uses a release fence + relaxed `valid` store, - mirroring `Unsafe.storeFence` + plain put. Trivially equivalent on x86; verify on aarch64 before - shipping the native path there. -- **Accepted minor regression.** Sporadic app-driven single attributes are ~2× the zero-JNI DBB - write on native. They are not per-activation, so the absolute impact is small; not designed around. - ---- - -## 8. Alternatives explicitly rejected (and why) - -- **Keep DBB + arena (Option A)** — perf-optimal and crash-safe with no tracer change, but adds an - allocator and keeps `CarrierThreadLocal`; contradicts the "no buffer management / most-correct" - goal. Retain as the fallback if the tracer API cannot change. -- **Reflective JVM pinning (Option E)** — corrupts VM state risk (toggle-with-assert), unstable - internals, hot-path overhead. -- **JVMTI mount events (Option C)** — cannot close the TOCTOU; global transition tax. -- **Fine-grained all-native (no combined call)** — measured ~1.6× per-cycle regression; batching is - required. - -## 9. Open questions / follow-ups - -- Confirm `ddprof-lib` Maven publication policy (external-consumer risk for phase 3). -- Decide whether to also remove the unused `ScopeStack` / `snapshot` / `restore` / batch setter in - phase 3, or retain for a possible future reapply-on-mount design (#11646). -- Re-run the combined-cycle benchmark on aarch64 and on a real dd-trace-java workload (chaos harness - under a patched `dd-java-agent`) to validate the ~7–10% win outside the microbenchmark. -- Split-context coherence (#11646) remains a separate tracer-side concern; this change makes it safe - but does not address it. - -## Appendix: benchmark experiment (reproducibility) - -The full JMH campaign — the six benchmark classes -(`ContextWritePathBenchmark`, `ContextAttrBenchmark`, `ContextVThreadBenchmark`, -`ContextScopeBenchmark`, `ContextSingleAttrBenchmark`, `ContextCombinedBenchmark`) and the -benchmark-only native prototypes they exercise (`*Native0` in `javaApi.cpp` with wrappers in -`JavaProfiler.java`) — is preserved on branch **`context-storage-benchmark-experiment`** -(`origin`). That scaffolding is intentionally *not* carried into the productionization branch; the -production API is the pared-down subset (combined per-activation write + single-attribute write + -value cache). Run any benchmark from that branch with -`./gradlew :ddprof-stresstest:jmh -PjmhInclude=""`. diff --git a/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md b/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md deleted file mode 100644 index fdc7f9d4fe..0000000000 --- a/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md +++ /dev/null @@ -1,89 +0,0 @@ -# Phase 2 — migrate dd-trace-java onto the all-native context API - -- **Date:** 2026-07-09 -- **Status:** Plan (execution in progress) -- **Related:** PR #631 (phase 1, java-profiler), PROF-15271, PROF-15361 (follow-up: batch reapply), - design note `2026-07-02-all-native-context-storage-design.md` (§5/§7) -- **Scope:** cross-repo — `java-profiler` (small read-API addition) and `dd-trace-java` (the migration) - -## Goal - -Switch dd-trace-java's profiler bridge off the deprecated DirectByteBuffer (DBB) context APIs onto -the all-native API added in phase 1 (`setTraceContext` / `clearTraceContext` / `setContextValue` / -`clearContextValue`). This is where the virtual-thread use-after-free fix and the per-activation -perf win land (3 JNI calls → 1 on activation). After this, no production code uses the DBB path. - -## Consumption loop (verified) - -- java-profiler is `com.datadoghq:ddprof:1.47.0-SNAPSHOT`; dd-trace-java pins `ddprof = "1.46.0"` and - `-PddprofUseSnapshot=true` derives next-minor → `1.47.0-SNAPSHOT` (exact match). `mavenLocal()` is - already first in dd-trace-java's repositories. -- Dev loop: java-profiler `./gradlew publishToMavenLocal`; dd-trace-java builds/tests with - `-PddprofUseSnapshot=true`. dd-trace-java uses **dependency locking**, so a full build needs - `--write-locks`. - -## Decisions locked (this session) - -1. **`setContextValue` publishes (`valid=1`).** Setting a value makes it visible even with no active - span — app context is visible independent of spans (dd-trace-java PR #11646). `clearContextValue` - preserves the prior valid (removing a value must not resurrect a deactivated record). Mirrors the - DBB single-setter vs bulk-setter asymmetry the tracer's reapply was built on. (Done: `5beb6987c`.) -2. **`setTraceContext` rejects `spanId == 0`** (IllegalArgumentException, thin Java wrapper; native - asserts). Clearing is `clearTraceContext`. Out-of-range slots throw; data conditions (null / - oversized / Dictionary full) stay soft. -3. **Reapply stays per-slot (#1), no batch API.** A per-slot native `setContextValue` loop preserves - the current shape and is adequate for typical app-attr cardinality (0–2). A native batch is - deferred to PROF-15361 and only if measurement warrants (JNI accessors are VM transitions too, so - a `byte[][]` batch would not clearly win; a flattened primitive-array batch would, but is premature). -4. **Design note §5 "wipe and let instrumentation re-set" is superseded** by #11646's reapply model; - we preserve reapply-on-activation/deactivation. - -## API mapping (old DBB → new all-native) - -| dd-trace-java use (DBB) | replacement | -| --- | --- | -| `profiler.setContext(root,span,hi,lo)` + `setContextValue(op)` + `setContextValue(res)` | one `profiler.setTraceContext(root,span,hi,lo, opOff,opVal, resOff,resVal)` | -| `profiler.setContext(0,0,0,0)` (clear) | `profiler.clearTraceContext()` | -| `ContextSetter.setContextValue` / `clearContextValue` (→ DBB `setContextAttribute`) | `profiler.setContextValue(slot,v)` / `clearContextValue(slot)` (native) | -| `ContextSetter.setContextValuesByIdAndBytes` (batch reapply) | per-slot native `setContextValue` loop | -| `ContextSetter.snapshotTags` / `JavaProfiler.copyTags` (DBB read; resets via `ThreadContext` ctor) | new native `copyContextTags` read (option A) | -| `ContextSetter.offsetOf` | **kept** — pure-Java `attributes.indexOf`, no DBB | - -## java-profiler change (option A — needed for the test oracle) - -`snapshot()` in dd-trace-java (test-only per the consumer audit) reads tag encodings via -`copyTags` → `currentContext().copyCustoms` — a DBB read whose `ThreadContext` ctor **resets** the -record, clobbering native writes. Add a small **native** read instead: - -- `JavaProfiler.copyContextTags(int[] out)` + `native copyContextTags0(int[])` — copies the current - thread's `enc[]` sidecar tag encodings directly from `ProfiledThread::current()` (no `ThreadContext`, - no reset). Test/introspection only. Add a java-profiler test. - -## dd-trace-java edits (per file) - -- `DatadogProfiler`: - - add `setTraceContext(...)` / `clearTraceContext()` (each: native call + `reapplyAppContext()`). - - `setContextValue(int,String)` / `clearContextValue(int)` → native `profiler.*`. - - `reapplyAppContext()` → per-slot `profiler.setContextValue(i, snapshot.stringAt(i))` loop (drop the - batch + valid=0 fallback; native `setContextValue` publishes uniformly). - - `syncNativeAppContext()` → native `setContextValue`/`clearContextValue`. - - `recordAppContextValue` → drop `snapshotTags`/`contextScratch` (no encoding capture); simplify - `AppContextSnapshot` to strings-only (`nonZeroCount` by string presence; drop `ids`/`utf8`). - - `snapshot()` → `profiler.copyContextTags(scratch)` (native read). - - keep `ContextSetter` only for `offsetOf`. -- `DatadogProfilingIntegration`: activate → one `setTraceContext(...)` folding op+resource; close / - `clearContext()` → `clearTraceContext()`. -- `DatadogProfilingScope`: unchanged in shape (save/restore app context); its writes now land native. - -## Verification - -- java-profiler: `publishToMavenLocal` (1.47.0-SNAPSHOT, incl. the new native read); its own tests green. -- dd-trace-java: `:dd-java-agent:agent-profiling:profiling-ddprof` build + `DatadogProfilerTest`, - `DatadogProfilerConfigTest`, `DatadogProfilerRecordingTest` green with `-PddprofUseSnapshot=true` - (`--write-locks` as needed). Smoke: `dd-smoke-tests/profiling-integration-tests`. - -## Follow-ups - -- PROF-15361: batch reapply (flattened primitive-array), only if reapply shows up in profiling. -- Phase 3 (java-profiler): remove the DBB / `ThreadContext` / `ContextSetter` / `OtelContextStorage` - subsystem once dd-trace-java is on the new API. dd-trace-java then needs its own `offsetOf` map. From 94ffac41c03faa9aeacb43ee11f437b7bce33860 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 13:12:44 +0200 Subject: [PATCH 02/10] Restore minimal ContextSetter for dd-trace-java compat; finish BufferWriter removal dd-trace-java's DatadogProfiler still depends on ContextSetter.offsetOf()/size() (pure-Java attribute-slot resolution, no DBB/JNI) per its Phase 2 migration (DataDog/dd-trace-java#11899). Deleting the class outright in Phase 3 broke that consumer; restore just the two pure-Java methods it uses. BufferWriter/BufferWriter8/BufferWriter9 lost their only caller (ThreadContext) in Phase 3 but were left behind as dead code, along with the Java-9 versioned-sources/multi-release-jar wiring and sourceCompatibility javadoc exemption that existed solely to support them. Remove all of it. --- ddprof-lib/build.gradle.kts | 25 -- .../com/datadoghq/profiler/BufferWriter.java | 180 --------- .../com/datadoghq/profiler/BufferWriter8.java | 55 --- .../com/datadoghq/profiler/ContextSetter.java | 52 +++ .../com/datadoghq/profiler/BufferWriter9.java | 56 --- .../datadoghq/profiler/BufferWriterTest.java | 373 ------------------ 6 files changed, 52 insertions(+), 689 deletions(-) delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java delete mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java create mode 100644 ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java delete mode 100644 ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java delete mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts index b39dbfc249..627491aef1 100644 --- a/ddprof-lib/build.gradle.kts +++ b/ddprof-lib/build.gradle.kts @@ -12,7 +12,6 @@ plugins { id("com.datadoghq.native-build") id("com.datadoghq.gtest") id("com.datadoghq.scanbuild") - id("com.datadoghq.versioned-sources") } val libraryName = "ddprof" @@ -54,23 +53,11 @@ gtest { failFast.set(true) } -// Java configuration - using sourceCompatibility (not --release 8) -// because BufferWriter8 needs access to internal sun.nio.ch package java { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } -// Configure versioned sources for runtime version-specific implementations -versionedSources { - versions { - register("java9") { - release.set(9) - minToolchainVersion.set(11) // Compile Java 9 code with JDK 11+ - } - } -} - // Test configuration tasks.test { onlyIf { @@ -89,14 +76,6 @@ val copyExternalLibs by tasks.registering(Copy::class) { } } -// Gradle 9 requires explicit dependency: compileJava9Java uses mainSourceSet.output -// which includes the copyExternalLibs destination directory -afterEvaluate { - tasks.named("compileJava9Java") { - dependsOn(copyExternalLibs) - } -} - // Create JAR tasks for each build configuration using nativeBuild extension utilities // Uses afterEvaluate to discover configurations dynamically from NativeBuildExtension afterEvaluate { @@ -133,7 +112,6 @@ afterEvaluate { } from(sourceSets.main.get().output.classesDirs) - versionedSources.configureJar(this) from(nativeBuild.libraryTargetBase(name)) { include("**/*") // Exclude debug symbols from production JAR @@ -169,7 +147,6 @@ tasks.jar { // Source JAR val sourcesJar by tasks.registering(Jar::class) { from(sourceSets.main.get().allJava) - versionedSources.configureSourceJar(this) archiveBaseName.set(libraryName) archiveClassifier.set("sources") archiveVersion.set(componentVersion) @@ -178,8 +155,6 @@ val sourcesJar by tasks.registering(Jar::class) { // Javadoc configuration tasks.withType().configureEach { dependsOn(copyExternalLibs) - // Allow javadoc to access internal sun.nio.ch package used by BufferWriter8 - (options as StandardJavadocDocletOptions).addStringOption("-add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") } // Javadoc JAR diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java deleted file mode 100644 index d9fb237f52..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.ByteBuffer; - -/** - * Version-agnostic wrapper for direct ByteBuffer memory operations with explicit memory ordering guarantees. - * - *

This class provides low-level memory access operations on direct ByteBuffers with precise control - * over memory ordering semantics. It abstracts differences between Java 8 (using sun.misc.Unsafe) and - * Java 9+ (using VarHandle APIs) to provide consistent behavior across JVM versions. - * - *

The class supports two types of memory ordering: - *

    - *
  • Ordered writes (release semantics): Prevents reordering with prior writes but allows - * subsequent operations to be reordered before this write. More efficient than volatile writes.
    • - *
  • Volatile writes (full barrier): Prevents reordering with both prior and subsequent - * operations. Ensures writes are completed before subsequent operations begin.
    • - *
- * - *

Signal Handler Safety: The primary use case is protecting write sequences from being - * observed in inconsistent states by async signal handlers (e.g., SIGPROF). When a signal interrupts - * a write sequence on the same thread, the signal handler must see either the complete write sequence - * or recognize the write is in-progress. This requires careful memory ordering even though both writer - * and reader execute on the same thread. - * - *

This is primarily used by the profiler for writing thread-local context data (span IDs, checksums) - * to direct ByteBuffers that are shared between Java and native code via JNI. - * - *

Thread Safety: This class is thread-safe. Individual write operations provide their own - * memory ordering guarantees as documented. - */ -public final class BufferWriter { - /** - * Service Provider Interface for version-specific buffer write implementations. - * - *

Implementations of this interface provide the actual low-level memory access operations: - *

    - *
  • {@code BufferWriter8} - Java 8 implementation using sun.misc.Unsafe
    • - *
  • {@code BufferWriter9} - Java 9+ implementation using VarHandle
    • - *
- * - *

The implementation is selected automatically based on the runtime Java version. - */ - public interface Impl { - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - void writeOrderedLong(ByteBuffer buffer, int offset, long value); - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - void writeInt(ByteBuffer buffer, int offset, int value); - - /** - * Executes a store-store memory fence. - * - *

Ensures that stores before the fence are visible before stores after it. - * Cheaper than fullFence on ARM (~5ns vs ~50ns) since it only orders - * stores, not loads. Sufficient for publication protocols where the writer - * needs to ensure data writes are visible before a flag/pointer write. - */ - void storeFence(); - } - - private final Impl impl; - - /** - * Creates a new BufferWriter with the appropriate implementation for the current Java version. - * - *

The implementation is selected based on the runtime Java version: - *

    - *
  • Java 8: Uses {@code BufferWriter8} which leverages sun.misc.Unsafe for memory operations
    • - *
  • Java 9+: Uses {@code BufferWriter9} which leverages VarHandle for memory operations
    • - *
- * - *

The implementation is loaded reflectively to avoid compile-time dependencies on - * version-specific APIs. - * - * @throws RuntimeException if the implementation class cannot be loaded or instantiated - */ - public BufferWriter() { - try { - if (Platform.isJavaVersion(8)) { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter8").getConstructor().newInstance(); - } else { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter9").getConstructor().newInstance(); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

This is commonly used for writing context fields (span IDs, root span IDs) in a write - * sequence that is protected by a volatile sentinel value. Ensures that if a signal handler - * interrupts after this write, all prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - impl.writeOrderedLong(buffer, offset, value); - } - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - public void writeOrderedInt(ByteBuffer buffer, int offset, int value) { - impl.writeInt(buffer, offset, value); - } - - /** - * Executes a store-store memory fence. - * - *

Ensures stores before the fence are globally visible before stores after it. - * On ARM this emits DMB ISHST (~2 ns); on x86 it is a compiler-only barrier. - */ - public void storeFence() { - impl.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java deleted file mode 100644 index 08af580522..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; - -import java.lang.reflect.Field; -import java.nio.ByteBuffer; - -public final class BufferWriter8 implements BufferWriter.Impl { - private static final Unsafe UNSAFE; - static { - Unsafe unsafe = null; - // a safety and testing valve to disable unsafe access - if (Platform.isJavaVersion(8)) { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - unsafe = (Unsafe) f.get(null); - } catch (Exception ignore) { - // On Java 8 will never happen - } - } - UNSAFE = unsafe; - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - UNSAFE.putOrderedLong(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - UNSAFE.putOrderedInt(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void storeFence() { - UNSAFE.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java new file mode 100644 index 0000000000..2ef1161bac --- /dev/null +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Resolves configured context attribute names to their fixed slot indices in the all-native + * context record. Deduplicates and truncates the configured attribute list to {@link + * JavaProfiler#MAX_CONTEXT_SLOTS} slots; callers use {@link #offsetOf} to translate an attribute + * name into the slot index passed to {@link JavaProfiler#setContextValue}. + */ +public class ContextSetter { + + private final List attributes; + + public ContextSetter(JavaProfiler profiler, List attributes) { + Set unique = new HashSet<>(attributes); + this.attributes = new ArrayList<>(unique.size()); + for (int i = 0; i < Math.min(attributes.size(), JavaProfiler.MAX_CONTEXT_SLOTS); i++) { + String attribute = attributes.get(i); + if (unique.remove(attribute)) { + this.attributes.add(attribute); + } + } + } + + public int offsetOf(String attribute) { + return attributes.indexOf(attribute); + } + + /** Number of (deduplicated, truncated) context attribute slots. */ + public int size() { + return attributes.size(); + } +} diff --git a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java b/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java deleted file mode 100644 index 9fcf77c4e5..0000000000 --- a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public final class BufferWriter9 implements BufferWriter.Impl { - private static final VarHandle LONG_VIEW_VH; - private static final VarHandle INT_VIEW_VH; - - static { - try { - // Create a view into ByteBuffer as if it were a long array - // The VarHandle coordinates are (ByteBuffer, int index) where index is in bytes - LONG_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - long[].class, ByteOrder.nativeOrder()); - INT_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - int[].class, ByteOrder.nativeOrder()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedLong) - LONG_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedInt) - INT_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void storeFence() { - VarHandle.storeStoreFence(); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java deleted file mode 100644 index 853b06431d..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Comprehensive tests for {@link BufferWriter} class. - * - *

This test validates: - *

    - *
  • Correct implementation selection based on Java version
    • - *
  • Basic write and read operations with ordered (release) semantics
    • - *
  • Release (ordered) semantics for single-threaded signal handler safety
    • - *
  • Int and long write operations
    • - *
  • Various offsets and buffer positions
    • - *
  • Edge cases and boundary conditions
    • - *
- */ -public class BufferWriterTest { - - private BufferWriter bufferWriter; - - @BeforeEach - public void setUp() { - bufferWriter = new BufferWriter(); - } - - /** - * Helper method to create a direct ByteBuffer with native byte order. - * BufferWriter implementations use native byte order, so tests must read in the same order. - */ - private ByteBuffer createBuffer(int capacity) { - return ByteBuffer.allocateDirect(capacity).order(ByteOrder.nativeOrder()); - } - - /** - * Tests that the correct implementation is loaded based on Java version. - */ - @Test - public void testCorrectImplementationLoaded() { - assertNotNull(bufferWriter, "BufferWriter instance should not be null"); - - // Verify implementation can be used without throwing exceptions - ByteBuffer buffer = createBuffer(16); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 0, 42L)); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 8, 99L)); - } - - /** - * Tests basic ordered long write and read functionality at offset 0. - */ - @Test - public void testWriteOrderedLongAtOffsetZero() { - ByteBuffer buffer = createBuffer(16); - long expectedValue = 0x123456789ABCDEF0L; - - bufferWriter.writeOrderedLong(buffer, 0, expectedValue); - long actualValue = buffer.getLong(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written value"); - } - - /** - * Tests basic ordered int write and read functionality. - */ - @Test - public void testWriteOrderedInt() { - ByteBuffer buffer = createBuffer(16); - int expectedValue = 0x12345678; - - bufferWriter.writeOrderedInt(buffer, 0, expectedValue); - int actualValue = buffer.getInt(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written int value"); - } - - /** - * Tests ordered long write operations at various offsets within the buffer. - */ - @Test - public void testWriteOrderedLongAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 8-byte aligned offsets - int[] offsets = {0, 8, 16, 24, 32, 40, 48, 56}; - long[] expectedValues = { - 0x1111111111111111L, - 0x2222222222222222L, - 0x3333333333333333L, - 0x4444444444444444L, - 0x5555555555555555L, - 0x6666666666666666L, - 0x7777777777777777L, - 0x8888888888888888L - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedLong(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - long actualValue = buffer.getLong(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Buffer value at offset %d should match", offsets[i])); - } - } - - /** - * Tests int write operations at various offsets within the buffer. - */ - @Test - public void testWriteIntAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 4-byte aligned offsets - int[] offsets = {0, 4, 8, 12, 16, 20, 24, 28}; - int[] expectedValues = { - 0x11111111, - 0x22222222, - 0x33333333, - 0x44444444, - 0x55555555, - 0x66666666, - 0x77777777, - 0x88888888 - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedInt(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - int actualValue = buffer.getInt(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Int value at offset %d should match", offsets[i])); - } - } - - /** - * Tests writing special long values (min, max, zero, negative). - */ - @Test - public void testSpecialLongValues() { - ByteBuffer buffer = createBuffer(40); - long[] specialValues = { - 0L, // Zero - Long.MIN_VALUE, // Minimum long - Long.MAX_VALUE, // Maximum long - -1L, // All bits set - 0xDEADBEEFCAFEBABEL // Arbitrary pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 8; - bufferWriter.writeOrderedLong(buffer, offset, specialValues[i]); - long actualValue = buffer.getLong(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests writing special int values (min, max, zero, negative). - */ - @Test - public void testSpecialIntValues() { - ByteBuffer buffer = createBuffer(24); - int[] specialValues = { - 0, // Zero - Integer.MIN_VALUE, // Minimum int - Integer.MAX_VALUE, // Maximum int - -1, // All bits set - 0xDEADBEEF, // Arbitrary pattern - 0x12345678 // Another pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 4; - bufferWriter.writeOrderedInt(buffer, offset, specialValues[i]); - int actualValue = buffer.getInt(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special int value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests that multiple consecutive writes work correctly without interference. - */ - @Test - public void testConsecutiveWrites() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write multiple times to the same offset - bufferWriter.writeOrderedLong(buffer, offset, 100L); - assertEquals(100L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 200L); - assertEquals(200L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 300L); - assertEquals(300L, buffer.getLong(offset)); - } - - /** - * Tests that writes to adjacent locations don't interfere with each other. - */ - @Test - public void testNonInterference() { - ByteBuffer buffer = createBuffer(32); - - long value1 = 0x1111111111111111L; - long value2 = 0x2222222222222222L; - long value3 = 0x3333333333333333L; - - bufferWriter.writeOrderedLong(buffer, 0, value1); - bufferWriter.writeOrderedLong(buffer, 8, value2); - bufferWriter.writeOrderedLong(buffer, 16, value3); - - assertEquals(value1, buffer.getLong(0), "First value should not be affected"); - assertEquals(value2, buffer.getLong(8), "Second value should not be affected"); - assertEquals(value3, buffer.getLong(16), "Third value should not be affected"); - } - - /** - * Tests writing to a buffer at maximum valid offset for longs. - */ - @Test - public void testMaximumValidOffsetLong() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 8; // 8 bytes for a long - - long expectedValue = 0xFEDCBA9876543210L; - bufferWriter.writeOrderedLong(buffer, maxValidOffset, expectedValue); - - long actualValue = buffer.getLong(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Value at maximum offset should be written correctly"); - } - - /** - * Tests writing to a buffer at maximum valid offset for ints. - */ - @Test - public void testMaximumValidOffsetInt() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 4; // 4 bytes for an int - - int expectedValue = 0xFEDCBA98; - bufferWriter.writeOrderedInt(buffer, maxValidOffset, expectedValue); - - int actualValue = buffer.getInt(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Int value at maximum offset should be written correctly"); - } - - /** - * Tests that overwriting values works correctly. - */ - @Test - public void testOverwrite() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write initial pattern - bufferWriter.writeOrderedLong(buffer, offset, 0xAAAAAAAAAAAAAAAAL); - assertEquals(0xAAAAAAAAAAAAAAAAL, buffer.getLong(offset)); - - // Overwrite with different pattern - bufferWriter.writeOrderedLong(buffer, offset, 0x5555555555555555L); - assertEquals(0x5555555555555555L, buffer.getLong(offset)); - - // Overwrite with zeros - bufferWriter.writeOrderedLong(buffer, offset, 0L); - assertEquals(0L, buffer.getLong(offset)); - } - - /** - * Tests parallel writes to different offsets from multiple threads. - */ - @Test - public void testParallelWrites() throws InterruptedException { - ByteBuffer buffer = createBuffer(128); - int numThreads = 8; - Thread[] threads = new Thread[numThreads]; - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch doneLatch = new CountDownLatch(numThreads); - - for (int i = 0; i < numThreads; i++) { - final int threadIndex = i; - final int offset = threadIndex * 16; - final long expectedValue = (long) threadIndex * 0x1111111111111111L; - - threads[i] = new Thread(() -> { - try { - startLatch.await(5, TimeUnit.SECONDS); - bufferWriter.writeOrderedLong(buffer, offset, expectedValue); - doneLatch.countDown(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - threads[i].start(); - } - - // Start all threads simultaneously - startLatch.countDown(); - - // Wait for all threads to complete - assertTrue(doneLatch.await(5, TimeUnit.SECONDS), - "All threads should complete within timeout"); - - // Verify all values were written correctly - for (int i = 0; i < numThreads; i++) { - int offset = i * 16; - long expectedValue = (long) i * 0x1111111111111111L; - long actualValue = buffer.getLong(offset); - assertEquals(expectedValue, actualValue, - String.format("Thread %d's value at offset %d should be correct", i, offset)); - } - - for (Thread thread : threads) { - thread.join(1000); - } - } - - /** - * Tests that the buffer's original position and limit are not affected by writes. - */ - @Test - public void testBufferStatePreservation() { - ByteBuffer buffer = createBuffer(64); - buffer.position(10); - buffer.limit(50); - - int originalPosition = buffer.position(); - int originalLimit = buffer.limit(); - - bufferWriter.writeOrderedLong(buffer, 16, 0x123456789ABCDEF0L); - bufferWriter.writeOrderedLong(buffer, 24, 0xFEDCBA9876543210L); - bufferWriter.writeOrderedInt(buffer, 32, 0x12345678); - bufferWriter.writeOrderedInt(buffer, 36, 0x9ABCDEF0); - - assertEquals(originalPosition, buffer.position(), - "Buffer position should not be affected by writes"); - assertEquals(originalLimit, buffer.limit(), - "Buffer limit should not be affected by writes"); - } -} From 25e3f5adadf76f5b7c8b7665f5b3a99545ff0447 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 13:44:15 +0200 Subject: [PATCH 03/10] Fix chaos harness default and antagonist per review feedback vthread-context-cascade targeted a use-after-free specific to the removed legacy DBB per-carrier-cached conduit; the all-native path resolves each thread's context record fresh via ProfiledThread::current() on every call, so that failure class no longer exists and there's no equivalent antagonist to swap in. Drop the now-unknown name from run-chaos-harness.sh's profiler+tracer default list so profiler+tracer chaos CI runs stop aborting with "unknown antagonist". ReapplyContextValueAntagonist.workerLoop also ignored setContextValue's boolean return; fail fast on false so a dictionary-overflow or other unexpected failure surfaces instead of silently losing coverage. --- .../profiler/chaos/ReapplyContextValueAntagonist.java | 11 +++++++++-- utils/run-chaos-harness.sh | 8 +++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java index 50ec02f77f..e20312d000 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java @@ -100,15 +100,22 @@ private void workerLoop() { // Span activation resets all custom slots. profiler.setTraceContext(localRootSpanId, spanId, 0, traceIdLow, -1, null, -1, null); for (int i = 0; i < SLOT_COUNT; i++) { - profiler.setContextValue(i, ROUTES[i % ROUTES.length]); + checkSetContextValue(profiler.setContextValue(i, ROUTES[i % ROUTES.length]), i); } for (int i = 0; i < SLOT_COUNT; i += 2) { profiler.clearContextValue(i); } for (int i = 0; i < SLOT_COUNT; i += 2) { - profiler.setContextValue(i, ROUTES[(i + ThreadLocalRandom.current().nextInt(ROUTES.length)) % ROUTES.length]); + int route = (i + ThreadLocalRandom.current().nextInt(ROUTES.length)) % ROUTES.length; + checkSetContextValue(profiler.setContextValue(i, ROUTES[route]), i); } profiler.clearTraceContext(); } } + + private static void checkSetContextValue(boolean succeeded, int slot) { + if (!succeeded) { + throw new IllegalStateException("setContextValue failed for slot " + slot); + } + } } diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh index ba8f695208..c43143908a 100755 --- a/utils/run-chaos-harness.sh +++ b/utils/run-chaos-harness.sh @@ -136,14 +136,12 @@ fi case $CONFIG in profiler) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context and - # vthread-context-cascade (which is driven entirely by @Trace-annotated - # methods) are excluded here. + # @Trace is a no-op without the tracer, so trace-context is excluded here. DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" ;; profiler+tracer) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" ;; *) echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 @@ -163,7 +161,7 @@ case $ALLOCATOR in # Logged so a glibc-detected corruption abort can be reproduced with the # same perturb byte (the value is otherwise random per run). echo "MALLOC_PERTURB_=${MALLOC_PERTURB_}" - # thread-churn/dump-storm/vthread-context-cascade cycle many short-lived + # thread-churn/dump-storm/vthread-churn cycle many short-lived # threads; glibc's per-thread arenas are slow to trim back to the OS, # which was inflating container RSS past the OOM limit on aarch64 # (mirrors the tcmalloc/jemalloc tuning below). From 31a1488e5c63ed095ee7f08aa65e3405e32bfc39 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 13:47:20 +0200 Subject: [PATCH 04/10] Remove remaining residue from legacy DBB context removal run-threadcontext-benchmark.sh ran a JMH class (ThreadContextBenchmark) that no longer exists; delete it. Fix two javadoc references to methods deleted in the same change: clearTraceContext()'s {@link #clearContext()} and TraceContextAntagonist's description of the tracer's activation path, now setTraceContext/clearTraceContext. --- .../com/datadoghq/profiler/JavaProfiler.java | 2 +- .../scripts/run-threadcontext-benchmark.sh | 26 ------------------- .../chaos/TraceContextAntagonist.java | 2 +- 3 files changed, 2 insertions(+), 28 deletions(-) delete mode 100755 ddprof-stresstest/scripts/run-threadcontext-benchmark.sh diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 83a9f6d73a..c86bd40765 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -263,7 +263,7 @@ public void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long e1 == null ? -1 : slot1, e1 == null ? 0 : e1.encoding, e1 == null ? null : e1.utf8); } - /** Combined per-scope-deactivation clear (replaces {@link #clearContext()} on the native path). */ + /** Clears the trace context on span deactivation. */ public void clearTraceContext() { clearTraceContext0(); } diff --git a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh b/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh deleted file mode 100755 index 86608bf7b6..0000000000 --- a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Run ThreadContext performance benchmark -# Compares JNI-based vs DirectByteBuffer-based implementations - -set -e - -JAVA_TEST_HOME="${JAVA_TEST_HOME:-$JAVA_HOME}" - -echo "Building JMH benchmark JAR..." -./gradlew :ddprof-stresstest:jmhJar - -echo "" -echo "Running ThreadContext benchmark..." -echo "This will take several minutes as it runs multiple configurations with 1, 2, 4, 8, and 16 threads" -echo "" - -# Run the benchmark with specific pattern to match our ThreadContext tests -"${JAVA_TEST_HOME}/bin/java" -jar ddprof-stresstest/build/libs/stresstests.jar \ - "ThreadContextBenchmark.*" \ - -rf json \ - -rff build/threadcontext-benchmark-results.json - -echo "" -echo "Benchmark complete!" -echo "Results saved to: build/threadcontext-benchmark-results.json" - diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java index 8abee751b7..570c1275b0 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java @@ -21,7 +21,7 @@ /** * Drives nested {@link Trace}-annotated invocations from a pool of worker * threads. Under a {@code dd-java-agent}-instrumented JVM the tracer drives - * {@code JavaProfiler.setContext}/{@code clearContext} on every span + * {@code JavaProfiler.setTraceContext}/{@code clearTraceContext} on every span * activation, so a high enter/exit rate stresses that path against signal * delivery and stack walking. * From e0b0e5bb41ce9ab2525a894bda594e4e4d326a77 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 13:52:31 +0200 Subject: [PATCH 05/10] Make setContextValue failure in chaos antagonist actually surface Main's own javadoc documents the harness's only failure signal as a non-zero process exit; an exception thrown on a background pool thread doesn't produce one on its own, and submit()'s discarded Future would have swallowed it entirely. Switch to execute() and halt the process on failure so an unexpected setContextValue false actually fails the job, per the intent of the original fail-fast fix. --- .../profiler/chaos/ReapplyContextValueAntagonist.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java index e20312d000..a8a84320ee 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java @@ -68,7 +68,7 @@ public String name() { public void start() { running = true; for (int i = 0; i < workerCount; i++) { - pool.submit(this::workerLoop); + pool.execute(this::workerLoop); } } @@ -113,9 +113,14 @@ private void workerLoop() { } } + // Per Main's class javadoc, the harness's only failure signal is a non-zero process exit; + // an exception here would otherwise die silently on this daemon worker thread (and be + // swallowed entirely if the task were submitted via ExecutorService.submit and its Future + // never inspected). Halt immediately so an unexpected failure actually fails the CI job. private static void checkSetContextValue(boolean succeeded, int slot) { if (!succeeded) { - throw new IllegalStateException("setContextValue failed for slot " + slot); + System.err.println("[chaos] reapply-context-value: setContextValue failed for slot " + slot); + Runtime.getRuntime().halt(1); } } } From 6fd6733f681888e55269881e08dbbf9fc29ca7de Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 29 Jul 2026 14:53:35 +0200 Subject: [PATCH 06/10] Clear stale trace context before re-enabling wall-clock on reused executor threads Removing the priming setContext(0,0) call also removed its incidental side effect of clearing a reused worker's leftover context before addThread() re-enables wall-clock sampling, letting a signal in the window before ContextTask.run() activates the new context attribute to the previous task's span. --- .../java/com/datadoghq/profiler/context/ContextExecutor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java index 22e3864200..97d0faaa7b 100644 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java +++ b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java @@ -24,6 +24,10 @@ protected RunnableFuture newTaskFor(Runnable runnable, T value) { @Override protected void beforeExecute(Thread t, Runnable r) { super.beforeExecute(t, r); + // Clear any context left over from this worker's previous task before wall-clock + // profiling is re-enabled below, so a signal in the window before ContextTask.run() + // activates the new context can't be attributed to the previous task's span. + profiler.clearTraceContext(); profiler.addThread(); } From d3dc0a23ffcdbd1e4454523fcd6d4e210067e1b4 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 30 Jul 2026 17:01:53 +0200 Subject: [PATCH 07/10] Address review nits: String over CharSequence, drop stale DBB comment, tidy import setTraceContext/setContextValue take String now instead of CharSequence since the value was immediately converted via toString() before touching the cache either way. Also drops a comment referencing the removed DirectByteBuffer mechanism and swaps a java.util.Arrays FQN for an import in the cache test. Co-Authored-By: Claude Sonnet 5 --- .../java/com/datadoghq/profiler/JavaProfiler.java | 13 ++++++------- .../java/com/datadoghq/profiler/OTelContext.java | 2 +- .../datadoghq/profiler/ContextValueCacheTest.java | 5 +++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index c86bd40765..edcdb6943c 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -225,8 +225,7 @@ public void removeThread() { // ---- All-native context write API (OTEP #4947) -------------------------------------------- // Each of these resolves the current carrier's OTEP record inside a single JNI call per - // operation — no cached DirectByteBuffer, so they are race-free under virtual-thread migration - // (see the design note and setTraceContext0 et al.). + // operation, so they are race-free under virtual-thread migration. /** * Combined per-scope-activation write: full trace/span context plus up to two span-derived @@ -249,7 +248,7 @@ public void removeThread() { * non-negative {@code slotN} is {@code >= MAX_CONTEXT_SLOTS} (out of range) */ public void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long traceIdLow, - int slot0, CharSequence v0, int slot1, CharSequence v1) { + int slot0, String v0, int slot1, String v1) { if (spanId == 0) { throw new IllegalArgumentException( "spanId must be non-zero; use clearTraceContext() to clear the trace context"); @@ -281,9 +280,9 @@ public void clearTraceContext() { * full * @throws IllegalArgumentException if {@code slot} is out of range */ - public boolean setContextValue(int slot, CharSequence value) { + public boolean setContextValue(int slot, String value) { requireValidSlot(slot); - ContextValueCache.Entry e = value == null ? null : contextValueCache.resolve(value.toString()); + ContextValueCache.Entry e = value == null ? null : contextValueCache.resolve(value); if (e == null) { clearContextValue0(slot); return false; @@ -335,11 +334,11 @@ private static void requireValidSlot(int slot) { // (skip sentinel), the value is null, or the value cannot be represented (oversized / Dictionary // full). A non-negative out-of-range slot is rejected earlier by requireActivationSlot, so it // never reaches here and never registers the value in the permanent native Dictionary. - private ContextValueCache.Entry resolveContextValue(int slot, CharSequence value) { + private ContextValueCache.Entry resolveContextValue(int slot, String value) { if (slot < 0 || value == null) { return null; } - return contextValueCache.resolve(value.toString()); + return contextValueCache.resolve(value); } /** diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java index 3ed5dbc191..f3eebf1917 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java @@ -235,7 +235,7 @@ public ProcessContext readProcessContext() { * element is null, the entire process context publish is * skipped - no context is published (a warning is logged) and * no exception is thrown. Order must match the indices used - * with {@link JavaProfiler#setContextValue(int, CharSequence)}. + * with {@link JavaProfiler#setContextValue(int, String)}. * * @throws NullPointerException if {@code attributeKeys} is null * diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java index f025023976..7daa29ba1e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.Arrays; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -57,7 +58,7 @@ public void resolveHitsCacheOnSecondLookup() throws IOException { public void oversizedValueResolvesToNull() throws IOException { loadLibrary(); char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES + 1]; - java.util.Arrays.fill(chars, 'x'); + Arrays.fill(chars, 'x'); assertNull(cache.resolve(new String(chars))); } @@ -65,7 +66,7 @@ public void oversizedValueResolvesToNull() throws IOException { public void maxSizeValueResolves() throws IOException { loadLibrary(); char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES]; - java.util.Arrays.fill(chars, 'x'); + Arrays.fill(chars, 'x'); String value = new String(chars); ContextValueCache.Entry e = cache.resolve(value); assertNotNull(e); From 0dfc51a34f1c08a163134250efd87830e4557c52 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 30 Jul 2026 20:04:44 +0200 Subject: [PATCH 08/10] Clean up dead carrier-scoping build logic, default the reapply-context-value chaos antagonist, and close the custom-attribute JFR test gap carrierExportJvmArgs()/testJvmMajorVersionFromRelease() and the two --add-exports call sites in ProfilerTestPlugin.kt, ContextCombinedBenchmark's jvmArgsAppend, and run-chaos-harness.sh's --add-exports line all existed solely to resolve jdk.internal.misc.CarrierThreadLocal for OtelContextStorage's now-deleted carrier-scoped mode; removed as dead weight. Adds reapply-context-value to both DEFAULT_ANTAGONISTS lists so the chaos harness exercises the antagonist that specifically targets the UAF class this PR removes, by default rather than only via ANTAGONISTS_OVERRIDE. Adds CustomContextAttributeSamplingTest, porting the weight-distribution/ jdk.ActiveSetting/dictionary_context_keys assertions from the deleted TagContextTest onto the all-native setContextValue API, closing the coverage gap between a value written through that API and it surfacing as a correctly-named, correctly-weighted JFR field. Co-Authored-By: Claude Sonnet 5 --- .../datadoghq/profiler/ProfilerTestPlugin.kt | 71 +-------- .../throughput/ContextCombinedBenchmark.java | 3 +- .../CustomContextAttributeSamplingTest.java | 149 ++++++++++++++++++ utils/run-chaos-harness.sh | 5 +- 4 files changed, 158 insertions(+), 70 deletions(-) create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index 4d85bc78ba..f12db89906 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -16,7 +16,6 @@ import org.gradle.api.provider.Property import org.gradle.api.tasks.Exec import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.testing.Test -import java.io.File import java.time.Duration import javax.inject.Inject @@ -79,60 +78,6 @@ import javax.inject.Inject */ class ProfilerTestPlugin : Plugin { - /** - * Major version of the *test* JVM, read from its `release` file (`JAVA_VERSION="..."`) rather - * than by executing the launcher. - * - * Executing `$JAVA_TEST_HOME/bin/java -version` (PlatformUtils.testJvmMajorVersion()) is - * unreliable here: in the musl split-JDK matrix it has been observed to report the build JDK - * (21) even when the test JVM is JDK 8, which put a JDK-21-only `--add-exports` onto a JDK-8 - * launcher and aborted it. Reading the `release` file is a pure file read of the same - * JAVA_TEST_HOME the executable is resolved from — deterministic, no subprocess, no exec-format - * or PATH hazards. Returns 0 when it cannot be determined (missing/old `release`), so callers - * fail safe: they omit the flag, the profiler degrades to thread-scoped storage, and the - * carrier-scoping tests skip — never an abort. - */ - private fun testJvmMajorVersionFromRelease(): Int = try { - val release = File(PlatformUtils.testJavaHome(), "release") - val version = release.takeIf { it.isFile } - ?.readLines() - ?.firstOrNull { it.startsWith("JAVA_VERSION=") } - ?.substringAfter('=')?.trim()?.trim('"') - // "1.8.0_452" -> 8 ; "21.0.5" -> 21 - val parts = version?.split('.').orEmpty() - val majorToken = when { - parts.isEmpty() -> "" - parts[0] == "1" && parts.size > 1 -> parts[1] - else -> parts[0] - } - majorToken.takeWhile { it.isDigit() }.toIntOrNull() ?: 0 - } catch (e: Exception) { - 0 - } - - /** - * JVM args required to enable carrier-scoped OTEL context storage - * (`OtelContextStorage.Mode.CARRIER`), or an empty list when the test JVM does not support it. - * - * Carrier scoping resolves `jdk.internal.misc.CarrierThreadLocal`, which lives in a - * non-exported package, so it needs `--add-exports java.base/jdk.internal.misc=ALL-UNNAMED`. - * That type only exists on JDK 21+, and the flag *aborts* a Java 8 JVM ("Unrecognized option"), - * so it is gated on the version of the actual test JVM. - * - * MUST be evaluated at task execution time (inside doFirst), not configuration time: the test - * JVM is selected via JAVA_TEST_HOME, which the CI only makes resolvable at execution time (see - * the `executable` assignments below). - */ - private fun carrierExportJvmArgs(project: Project): List { - val major = testJvmMajorVersionFromRelease() - val enabled = major >= 21 - project.logger.info( - "ddprof: carrier --add-exports gate — testJavaHome={}, detected major={}, flag {}", - PlatformUtils.testJavaHome(), major, if (enabled) "ADDED" else "omitted" - ) - return if (enabled) listOf("--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED") else emptyList() - } - override fun apply(project: Project) { val extension = project.extensions.create( "profilerTest", @@ -311,8 +256,6 @@ class ProfilerTestPlugin : Plugin { testTask.doFirst { val allArgs = mutableListOf() allArgs.addAll(testConfig.standardJvmArgs) - // Version-gated at execution time, when the real test JVM is resolvable. - allArgs.addAll(carrierExportJvmArgs(project)) if (extension.nativeLibDir.isPresent) { allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") @@ -386,8 +329,6 @@ class ProfilerTestPlugin : Plugin { // JVM args allArgs.addAll(testConfig.standardJvmArgs) - // Version-gated at execution time, when the real test JVM (JAVA_TEST_HOME) is resolvable. - allArgs.addAll(carrierExportJvmArgs(project)) if (extension.nativeLibDir.isPresent) { allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") } @@ -748,12 +689,12 @@ abstract class ProfilerTestExtension @Inject constructor( init { // Standard JVM arguments for profiler testing. - // NOTE: JDK-version-gated flags (e.g. the carrier-scoping --add-exports) must NOT be - // added here. This convention is computed at configuration time, where JAVA_TEST_HOME - // is not yet resolvable and PlatformUtils.testJavaHome() falls back to the *build* JDK - // (JAVA_HOME) — which misdetects in the musl split-JDK CI (build JDK 21, test JDK 8) and - // would emit a JDK-21 flag onto a JDK-8 test JVM. Version-gated flags are added at - // execution time in the task doFirst blocks instead (see ProfilerTestPlugin). + // NOTE: JDK-version-gated flags must NOT be added here. This convention is computed at + // configuration time, where JAVA_TEST_HOME is not yet resolvable and + // PlatformUtils.testJavaHome() falls back to the *build* JDK (JAVA_HOME) — which + // misdetects in the musl split-JDK CI (build JDK 21, test JDK 8) and would emit a + // JDK-21 flag onto a JDK-8 test JVM. Version-gated flags belong in the task doFirst + // blocks instead (see ProfilerTestPlugin), where the real test JVM is resolvable. standardJvmArgs.convention(listOf( "-Djdk.attach.allowAttachSelf", // Allow profiler to attach to self "-Djol.tryWithSudo=true", // JOL memory layout analysis diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java index d2c606a80b..ebc1d7be26 100644 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java +++ b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java @@ -46,8 +46,7 @@ */ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 3, warmups = 0, - jvmArgsAppend = {"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED"}) +@Fork(value = 3, warmups = 0) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 2) public class ContextCombinedBenchmark { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java new file mode 100644 index 0000000000..0a5b43c8f8 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler.context; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.ContextSetter; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junitpioneer.jupiter.RetryingTest; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end coverage for the custom-attribute {@code setContextValue}/{@code ContextSetter} -> + * JFR field pipeline, on the all-native context write path. + * + *

{@link com.datadoghq.profiler.ContextValueCacheTest} covers the value-cache unit in + * isolation, and {@link AllNativeContextSamplingTest} covers span/root-span propagation into JFR, + * but neither confirms that a value written via {@code setContextValue} actually surfaces as a + * correctly-named, correctly-weighted JFR field. This closes that gap. + */ +public class CustomContextAttributeSamplingTest extends AbstractProfilerTest { + + @BeforeEach + void assumeNotJ9() { + // On J9, ProfiledThread (and thus the OTEP TLS buffer) is not allocated until the thread + // is registered for wall-clock profiling, so a fresh context write throws. + Assumptions.assumeTrue(!Platform.isJ9()); + } + + @Override + protected String getProfilerCommand() { + return "wall=1ms,filter=0,attributes=tag1;tag2;tag3"; + } + + @RetryingTest(10) + public void customAttributeValueSurfacesInJfr() throws InterruptedException { + registerCurrentThreadForWallClockProfiling(); + ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag3")); + int slot = contextSetter.offsetOf("tag1"); + + // Session-unique prefix so each @RetryingTest attempt registers fresh values in the + // native Dictionary, matching dictionary_context_keys to this run's writes only. + String pfx = Long.toHexString(System.nanoTime()) + "_"; + String[] values = IntStream.range(0, 10).mapToObj(i -> pfx + i).toArray(String[]::new); + for (int i = 0; i < values.length * 10; i++) { + String value = values[i % values.length]; + assertTrue(profiler.setContextValue(slot, value)); + Thread.sleep(10); + profiler.clearContextValue(slot); + } + stopProfiler(); + + IItemCollection events = verifyEvents("datadog.MethodSample"); + Map weightsByTagValue = new HashMap<>(); + for (IItemIterable samples : events) { + IMemberAccessor weightAccessor = WEIGHT.getAccessor(samples.getType()); + IMemberAccessor tag1Accessor = TAG_1.getAccessor(samples.getType()); + IMemberAccessor tag2Accessor = TAG_2.getAccessor(samples.getType()); + if (tag1Accessor == null || tag2Accessor == null) { + continue; + } + IMemberAccessor stacktraceAccessor = + JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); + for (IItem sample : samples) { + String stacktrace = stacktraceAccessor.getMember(sample); + if (!stacktrace.contains("sleep") || stacktrace.contains("")) { + // Only count samples taken while the context was definitely active. + continue; + } + String tag = tag1Accessor.getMember(sample); + if (tag == null) { + continue; + } + weightsByTagValue.computeIfAbsent(tag, v -> new AtomicLong()) + .addAndGet(weightAccessor.getMember(sample).longValue()); + assertNull(tag2Accessor.getMember(sample), "tag2 was never set for this thread"); + } + } + for (String value : values) { + assertNotNull(weightsByTagValue.get(value), + "no sample carried tag1=" + value + ". Found: " + weightsByTagValue.keySet()); + } + + // jdk.ActiveSetting must enumerate the configured attribute names, unbundling the + // dynamic-column config into the recording. + IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); + Set recordedContextAttributes = new HashSet<>(); + for (IItemIterable activeSetting : activeSettings) { + IMemberAccessor nameAccessor = + JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); + IMemberAccessor valueAccessor = + JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); + for (IItem item : activeSetting) { + if ("contextattribute".equals(nameAccessor.getMember(item))) { + recordedContextAttributes.add(valueAccessor.getMember(item)); + } + } + } + assertEquals(3, recordedContextAttributes.size()); + assertTrue(recordedContextAttributes.contains("tag1")); + assertTrue(recordedContextAttributes.contains("tag2")); + assertTrue(recordedContextAttributes.contains("tag3")); + + // dictionary_context_keys must match the number of distinct values registered above. + Map jfrCounters = new HashMap<>(); + for (IItemIterable counterEvent : verifyEvents("datadog.ProfilerCounter")) { + IMemberAccessor nameAccessor = NAME.getAccessor(counterEvent.getType()); + IMemberAccessor countAccessor = COUNT.getAccessor(counterEvent.getType()); + for (IItem item : counterEvent) { + jfrCounters.put(nameAccessor.getMember(item), countAccessor.getMember(item).longValue()); + } + } + assertFalse(jfrCounters.isEmpty()); + assertEquals(values.length, jfrCounters.get("dictionary_context_keys")); + } +} diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh index c43143908a..742d3f36a7 100755 --- a/utils/run-chaos-harness.sh +++ b/utils/run-chaos-harness.sh @@ -137,11 +137,11 @@ case $CONFIG in profiler) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" # @Trace is a no-op without the tracer, so trace-context is excluded here. - DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm,reapply-context-value" ;; profiler+tracer) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm,reapply-context-value" ;; *) echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 @@ -249,7 +249,6 @@ CHAOS_START=$(date +%s) timeout "$((RUNTIME + 300))" \ java -javaagent:${PATCHED_AGENT} \ --add-opens java.base/java.lang=ALL-UNNAMED \ - --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ ${ENABLEMENT} \ -Ddd.profiling.upload.period=10 \ -Ddd.profiling.start-force-first=true \ From f80a9b3876e005e354cec9b0db0be995101e4ae5 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 31 Jul 2026 13:04:19 +0200 Subject: [PATCH 09/10] Keep setTraceContext's activation attributes as CharSequence, and make the cache honor it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dd-trace-java's DatadogProfiler.setTraceContext forwards ProfilerContext#getOperationName()/getResourceName() directly into this method, both declared CharSequence throughout DDSpanContext's public span API — UTF8BytesString and SubSequence exist specifically to avoid materializing a String for these fields on the tracing hot path. Narrowing to String here would break that call site at compile time on the next ddprof-lib version bump, and runs counter to the tracer's own allocation-avoidance idiom. setContextValue keeps its String param, since dd-trace-java always passes an actual String there. Since the whole point of accepting CharSequence is avoiding an eager String, ContextValueCache gains a resolve(CharSequence) overload that never calls toString() on a cache hit: contentHashCode() replicates String#hashCode()'s documented polynomial algorithm (part of the API contract, not an implementation detail) so a CharSequence and a content-equal cached String key land in the same slot, and String#contentEquals(CharSequence) compares without allocating. toString() is only paid on a genuine miss, where registerConstant0 needs a real String anyway. Co-Authored-By: Claude Sonnet 5 --- .../datadoghq/profiler/ContextValueCache.java | 39 ++++++++++++++ .../com/datadoghq/profiler/JavaProfiler.java | 4 +- .../profiler/ContextValueCacheTest.java | 53 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java index 8e8171959f..3977d7813f 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java @@ -97,6 +97,45 @@ Entry resolve(String value) { return ne; } + /** + * {@link CharSequence} counterpart of {@link #resolve(String)}, for callers (e.g. {@code + * setTraceContext}) whose value may be a non-{@code String} implementation — dd-trace-java + * passes {@code UTF8BytesString}/{@code SubSequence} spans-name views specifically to avoid + * forcing a {@code String} allocation on the hot path. The cache-hit path here never calls + * {@code toString()}: {@link #contentHashCode} replicates {@link String#hashCode()}'s + * algorithm so a {@code CharSequence} and a content-equal cached {@code String} key land in + * the same slot, and {@link String#contentEquals(CharSequence)} compares without allocating. + * {@code toString()} is only paid on a genuine miss, where a {@code String} is needed anyway + * to hand to {@code registerConstant0} and to store as the entry's key. + */ + Entry resolve(CharSequence value) { + if (value == null) { + return null; + } + if (value instanceof String) { + return resolve((String) value); + } + int slot = contentHashCode(value) & MASK; + Entry e = table.get(slot); + if (e != null && e.key.contentEquals(value)) { + return e; // hit — no allocation, no JNI + } + return resolve(value.toString()); + } + + /** + * Computes {@link String#hashCode()}'s polynomial ({@code s[0]*31^(n-1) + ... + s[n-1]}) over + * an arbitrary {@code CharSequence}, without materializing a {@code String}. + */ + private static int contentHashCode(CharSequence value) { + int h = 0; + int len = value.length(); + for (int i = 0; i < len; i++) { + h = 31 * h + value.charAt(i); + } + return h; + } + /** * Drops all cached entries. Called when a fresh recording session starts and the native * Dictionary is reset, so stale encodings from the previous session are not reused. A concurrent diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index edcdb6943c..74ebd9ac7c 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -248,7 +248,7 @@ public void removeThread() { * non-negative {@code slotN} is {@code >= MAX_CONTEXT_SLOTS} (out of range) */ public void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long traceIdLow, - int slot0, String v0, int slot1, String v1) { + int slot0, CharSequence v0, int slot1, CharSequence v1) { if (spanId == 0) { throw new IllegalArgumentException( "spanId must be non-zero; use clearTraceContext() to clear the trace context"); @@ -334,7 +334,7 @@ private static void requireValidSlot(int slot) { // (skip sentinel), the value is null, or the value cannot be represented (oversized / Dictionary // full). A non-negative out-of-range slot is rejected earlier by requireActivationSlot, so it // never reaches here and never registers the value in the permanent native Dictionary. - private ContextValueCache.Entry resolveContextValue(int slot, String value) { + private ContextValueCache.Entry resolveContextValue(int slot, CharSequence value) { if (slot < 0 || value == null) { return null; } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java index 7daa29ba1e..53e32ed5aa 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java @@ -113,4 +113,57 @@ public void clearDropsCachedEntries() throws IOException { assertNotNull(after); assertNotSame(before, after, "resolve after clear() must not return the pre-clear entry"); } + + @Test + public void charSequenceHitNeverCallsToString() throws IOException { + loadLibrary(); + ContextValueCache.Entry primed = cache.resolve("shared-value"); + assertNotNull(primed); + + ToStringCountingCharSequence value = new ToStringCountingCharSequence("shared-value"); + ContextValueCache.Entry hit = cache.resolve((CharSequence) value); + assertSame(primed, hit, "a content-equal CharSequence must hit the String-keyed entry"); + assertEquals(0, value.toStringCalls, "a cache hit must not materialize a String"); + } + + @Test + public void charSequenceMissMaterializesStringExactlyOnce() throws IOException { + loadLibrary(); + ToStringCountingCharSequence value = new ToStringCountingCharSequence("not-yet-cached"); + ContextValueCache.Entry e = cache.resolve((CharSequence) value); + assertNotNull(e); + assertEquals("not-yet-cached", e.key); + assertEquals(1, value.toStringCalls, "a miss must materialize a String exactly once"); + } + + /** A {@link CharSequence} that is not a {@link String}, counting {@link #toString()} calls. */ + private static final class ToStringCountingCharSequence implements CharSequence { + private final String value; + int toStringCalls = 0; + + ToStringCountingCharSequence(String value) { + this.value = value; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return value.subSequence(start, end); + } + + @Override + public String toString() { + toStringCalls++; + return value; + } + } } From 83c6d4ce89169c3197027d9dc4928c79e6a24197 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 31 Jul 2026 13:28:49 +0200 Subject: [PATCH 10/10] Remove ambiguous String/CharSequence overload pair on ContextValueCache.resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve(String) and resolve(CharSequence) as separate overloads meant dispatch depended on the caller's static type rather than the value's runtime type (e.g. JavaProfiler.resolveContextValue's CharSequence-typed local always binds to the CharSequence overload even when it happens to hold a String) — correct today only because of the explicit instanceof check, but a footgun for future callers who might assume Java picks the more specific runtime match. resolve is now the single entrypoint; the String-only fast path moves to a private resolveString helper. Co-Authored-By: Claude Sonnet 5 --- .../datadoghq/profiler/ContextValueCache.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java index 3977d7813f..619a85e22a 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java @@ -68,15 +68,43 @@ static final class Entry { /** * Resolves {@code value} to its cached {@code (encoding, utf8)} pair, registering it on a miss. + * The sole entrypoint — deliberately not overloaded on {@code String} alongside {@code + * CharSequence}, since Java resolves overloads by static (not runtime) type and a {@code + * String}-typed local would silently bind to a different method than a {@code + * CharSequence}-typed one holding the same value. {@link #resolveString} is the private, + * no-allocation fast path for callers (like this method, on a {@code String} runtime type) + * that already hold a {@code String}. + * + *

The cache-hit path never calls {@code toString()} on a non-{@code String} {@code + * CharSequence}: {@link #contentHashCode} replicates {@link String#hashCode()}'s algorithm so + * a {@code CharSequence} and a content-equal cached {@code String} key land in the same slot, + * and {@link String#contentEquals(CharSequence)} compares without allocating. This matters for + * callers (e.g. {@code setTraceContext}) whose value may be a non-{@code String} + * implementation — dd-trace-java passes {@code UTF8BytesString}/{@code SubSequence} span-name + * views specifically to avoid forcing a {@code String} allocation on the hot path. + * {@code toString()} is only paid on a genuine miss, where a {@code String} is needed anyway + * to hand to {@code registerConstant0} and to store as the entry's key. * * @return the entry, or {@code null} if the value cannot be represented — {@code null} input, a * UTF-8 encoding longer than {@value #MAX_VALUE_BYTES} bytes, or a full native Dictionary * (encoding {@code < 0}). Callers treat {@code null} as "no attribute" (skip / clear). */ - Entry resolve(String value) { + Entry resolve(CharSequence value) { if (value == null) { return null; } + if (value instanceof String) { + return resolveString((String) value); + } + int slot = contentHashCode(value) & MASK; + Entry e = table.get(slot); + if (e != null && e.key.contentEquals(value)) { + return e; // hit — no allocation, no JNI + } + return resolveString(value.toString()); + } + + private Entry resolveString(String value) { int slot = value.hashCode() & MASK; Entry e = table.get(slot); if (e != null && value.equals(e.key)) { @@ -97,32 +125,6 @@ Entry resolve(String value) { return ne; } - /** - * {@link CharSequence} counterpart of {@link #resolve(String)}, for callers (e.g. {@code - * setTraceContext}) whose value may be a non-{@code String} implementation — dd-trace-java - * passes {@code UTF8BytesString}/{@code SubSequence} spans-name views specifically to avoid - * forcing a {@code String} allocation on the hot path. The cache-hit path here never calls - * {@code toString()}: {@link #contentHashCode} replicates {@link String#hashCode()}'s - * algorithm so a {@code CharSequence} and a content-equal cached {@code String} key land in - * the same slot, and {@link String#contentEquals(CharSequence)} compares without allocating. - * {@code toString()} is only paid on a genuine miss, where a {@code String} is needed anyway - * to hand to {@code registerConstant0} and to store as the entry's key. - */ - Entry resolve(CharSequence value) { - if (value == null) { - return null; - } - if (value instanceof String) { - return resolve((String) value); - } - int slot = contentHashCode(value) & MASK; - Entry e = table.get(slot); - if (e != null && e.key.contentEquals(value)) { - return e; // hit — no allocation, no JNI - } - return resolve(value.toString()); - } - /** * Computes {@link String#hashCode()}'s polynomial ({@code s[0]*31^(n-1) + ... + s[n-1]}) over * an arbitrary {@code CharSequence}, without materializing a {@code String}.