diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index 4d85bc78ba..f12db89906 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -16,7 +16,6 @@ import org.gradle.api.provider.Property import org.gradle.api.tasks.Exec import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.testing.Test -import java.io.File import java.time.Duration import javax.inject.Inject @@ -79,60 +78,6 @@ import javax.inject.Inject */ class ProfilerTestPlugin : Plugin { - /** - * Major version of the *test* JVM, read from its `release` file (`JAVA_VERSION="..."`) rather - * than by executing the launcher. - * - * Executing `$JAVA_TEST_HOME/bin/java -version` (PlatformUtils.testJvmMajorVersion()) is - * unreliable here: in the musl split-JDK matrix it has been observed to report the build JDK - * (21) even when the test JVM is JDK 8, which put a JDK-21-only `--add-exports` onto a JDK-8 - * launcher and aborted it. Reading the `release` file is a pure file read of the same - * JAVA_TEST_HOME the executable is resolved from — deterministic, no subprocess, no exec-format - * or PATH hazards. Returns 0 when it cannot be determined (missing/old `release`), so callers - * fail safe: they omit the flag, the profiler degrades to thread-scoped storage, and the - * carrier-scoping tests skip — never an abort. - */ - private fun testJvmMajorVersionFromRelease(): Int = try { - val release = File(PlatformUtils.testJavaHome(), "release") - val version = release.takeIf { it.isFile } - ?.readLines() - ?.firstOrNull { it.startsWith("JAVA_VERSION=") } - ?.substringAfter('=')?.trim()?.trim('"') - // "1.8.0_452" -> 8 ; "21.0.5" -> 21 - val parts = version?.split('.').orEmpty() - val majorToken = when { - parts.isEmpty() -> "" - parts[0] == "1" && parts.size > 1 -> parts[1] - else -> parts[0] - } - majorToken.takeWhile { it.isDigit() }.toIntOrNull() ?: 0 - } catch (e: Exception) { - 0 - } - - /** - * JVM args required to enable carrier-scoped OTEL context storage - * (`OtelContextStorage.Mode.CARRIER`), or an empty list when the test JVM does not support it. - * - * Carrier scoping resolves `jdk.internal.misc.CarrierThreadLocal`, which lives in a - * non-exported package, so it needs `--add-exports java.base/jdk.internal.misc=ALL-UNNAMED`. - * That type only exists on JDK 21+, and the flag *aborts* a Java 8 JVM ("Unrecognized option"), - * so it is gated on the version of the actual test JVM. - * - * MUST be evaluated at task execution time (inside doFirst), not configuration time: the test - * JVM is selected via JAVA_TEST_HOME, which the CI only makes resolvable at execution time (see - * the `executable` assignments below). - */ - private fun carrierExportJvmArgs(project: Project): List { - val major = testJvmMajorVersionFromRelease() - val enabled = major >= 21 - project.logger.info( - "ddprof: carrier --add-exports gate — testJavaHome={}, detected major={}, flag {}", - PlatformUtils.testJavaHome(), major, if (enabled) "ADDED" else "omitted" - ) - return if (enabled) listOf("--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED") else emptyList() - } - override fun apply(project: Project) { val extension = project.extensions.create( "profilerTest", @@ -311,8 +256,6 @@ class ProfilerTestPlugin : Plugin { testTask.doFirst { val allArgs = mutableListOf() allArgs.addAll(testConfig.standardJvmArgs) - // Version-gated at execution time, when the real test JVM is resolvable. - allArgs.addAll(carrierExportJvmArgs(project)) if (extension.nativeLibDir.isPresent) { allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") @@ -386,8 +329,6 @@ class ProfilerTestPlugin : Plugin { // JVM args allArgs.addAll(testConfig.standardJvmArgs) - // Version-gated at execution time, when the real test JVM (JAVA_TEST_HOME) is resolvable. - allArgs.addAll(carrierExportJvmArgs(project)) if (extension.nativeLibDir.isPresent) { allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") } @@ -748,12 +689,12 @@ abstract class ProfilerTestExtension @Inject constructor( init { // Standard JVM arguments for profiler testing. - // NOTE: JDK-version-gated flags (e.g. the carrier-scoping --add-exports) must NOT be - // added here. This convention is computed at configuration time, where JAVA_TEST_HOME - // is not yet resolvable and PlatformUtils.testJavaHome() falls back to the *build* JDK - // (JAVA_HOME) — which misdetects in the musl split-JDK CI (build JDK 21, test JDK 8) and - // would emit a JDK-21 flag onto a JDK-8 test JVM. Version-gated flags are added at - // execution time in the task doFirst blocks instead (see ProfilerTestPlugin). + // NOTE: JDK-version-gated flags must NOT be added here. This convention is computed at + // configuration time, where JAVA_TEST_HOME is not yet resolvable and + // PlatformUtils.testJavaHome() falls back to the *build* JDK (JAVA_HOME) — which + // misdetects in the musl split-JDK CI (build JDK 21, test JDK 8) and would emit a + // JDK-21 flag onto a JDK-8 test JVM. Version-gated flags belong in the task doFirst + // blocks instead (see ProfilerTestPlugin), where the real test JVM is resolvable. standardJvmArgs.convention(listOf( "-Djdk.attach.allowAttachSelf", // Allow profiler to attach to self "-Djol.tryWithSudo=true", // JOL memory layout analysis diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts index b39dbfc249..627491aef1 100644 --- a/ddprof-lib/build.gradle.kts +++ b/ddprof-lib/build.gradle.kts @@ -12,7 +12,6 @@ plugins { id("com.datadoghq.native-build") id("com.datadoghq.gtest") id("com.datadoghq.scanbuild") - id("com.datadoghq.versioned-sources") } val libraryName = "ddprof" @@ -54,23 +53,11 @@ gtest { failFast.set(true) } -// Java configuration - using sourceCompatibility (not --release 8) -// because BufferWriter8 needs access to internal sun.nio.ch package java { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } -// Configure versioned sources for runtime version-specific implementations -versionedSources { - versions { - register("java9") { - release.set(9) - minToolchainVersion.set(11) // Compile Java 9 code with JDK 11+ - } - } -} - // Test configuration tasks.test { onlyIf { @@ -89,14 +76,6 @@ val copyExternalLibs by tasks.registering(Copy::class) { } } -// Gradle 9 requires explicit dependency: compileJava9Java uses mainSourceSet.output -// which includes the copyExternalLibs destination directory -afterEvaluate { - tasks.named("compileJava9Java") { - dependsOn(copyExternalLibs) - } -} - // Create JAR tasks for each build configuration using nativeBuild extension utilities // Uses afterEvaluate to discover configurations dynamically from NativeBuildExtension afterEvaluate { @@ -133,7 +112,6 @@ afterEvaluate { } from(sourceSets.main.get().output.classesDirs) - versionedSources.configureJar(this) from(nativeBuild.libraryTargetBase(name)) { include("**/*") // Exclude debug symbols from production JAR @@ -169,7 +147,6 @@ tasks.jar { // Source JAR val sourcesJar by tasks.registering(Jar::class) { from(sourceSets.main.get().allJava) - versionedSources.configureSourceJar(this) archiveBaseName.set(libraryName) archiveClassifier.set("sources") archiveVersion.set(componentVersion) @@ -178,8 +155,6 @@ val sourcesJar by tasks.registering(Jar::class) { // Javadoc configuration tasks.withType().configureEach { dependsOn(copyExternalLibs) - // Allow javadoc to access internal sun.nio.ch package used by BufferWriter8 - (options as StandardJavadocDocletOptions).addStringOption("-add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") } // Javadoc JAR diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 83f6045c1a..3f71a1b74c 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -710,51 +710,6 @@ Java_com_datadoghq_profiler_OTelContext_readProcessCtx0(JNIEnv *env, jclass unus #endif } -extern "C" DLLEXPORT jobject JNICALL -Java_com_datadoghq_profiler_JavaProfiler_initializeContextTLS0(JNIEnv* env, jclass unused, jlongArray metadata) { - // Initialize thread TLS if it has not yet done - ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); - assert(thrd != nullptr); - - if (!thrd->isContextInitialized()) { - ContextApi::initializeContextTLS(thrd); - } - - OtelThreadContextRecord* record = thrd->getOtelContextRecord(); - - // Contiguity of record + tag_encodings + LRS is enforced by alignas(8) on _otel_ctx_record - // plus sizeof(OtelThreadContextRecord) being a multiple of 8 (see thread.h). - // Compile-time alignment check always runs; runtime pointer-layout check is debug-only. - static_assert(DD_TAGS_CAPACITY * sizeof(u32) % alignof(u64) == 0, - "tag encodings array size must be aligned to u64 for contiguous sidecar layout"); -#ifdef DEBUG - uint8_t* record_start = reinterpret_cast(record); - uint8_t* sidecar_start = reinterpret_cast(thrd->getOtelTagEncodingsPtr()); - assert(sidecar_start == record_start + OTEL_MAX_RECORD_SIZE - && "_otel_ctx_record and _otel_tag_encodings must be contiguous"); -#endif - - // Fill metadata[6]: [VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - // ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET]. - // All offsets are absolute within the unified buffer returned below. - if (metadata != nullptr && env->GetArrayLength(metadata) >= 6) { - jlong meta[6]; - meta[0] = (jlong)offsetof(OtelThreadContextRecord, valid); - meta[1] = (jlong)offsetof(OtelThreadContextRecord, trace_id); - meta[2] = (jlong)offsetof(OtelThreadContextRecord, span_id); - meta[3] = (jlong)offsetof(OtelThreadContextRecord, attrs_data_size); - meta[4] = (jlong)offsetof(OtelThreadContextRecord, attrs_data); - meta[5] = (jlong)(OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32)); - env->SetLongArrayRegion(metadata, 0, 6, meta); - } - - // Single contiguous view over [record | tag_encodings | LRS] — used for per-field - // access and for bulk snapshot/restore. All three regions are in one ProfiledThread - // memory block. - size_t totalSize = OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32) + sizeof(u64); - return env->NewDirectByteBuffer((void*)record, (jlong)totalSize); -} - // --------------------------------------------------------------------------- // All-native context write API (OTEP #4947). // @@ -762,25 +717,21 @@ Java_com_datadoghq_profiler_JavaProfiler_initializeContextTLS0(JNIEnv* env, jcla // ProfiledThread::current(). Because a JNI native frame pins a mounted virtual thread to its // carrier for the duration of the call (the continuation cannot freeze across a native frame), // the record resolved here is guaranteed live and cannot migrate mid-write — there is no cached -// per-thread buffer to dangle. This replaces the DirectByteBuffer path (see ThreadContext), -// eliminating the virtual-thread use-after-free. +// per-thread buffer to dangle, and hence no virtual-thread use-after-free. // // Signal-safety / reader coherence: the sampler (ContextApi::get, wallClock.cpp) reads the same -// record on the same carrier, gated on record->valid. Each write follows the detach -> mutate -> -// attach protocol used by ThreadContext: store valid=0, release fence, mutate, release fence, -// store valid=1. On x86 the release fence is a compiler barrier; on aarch64 it is a real barrier -// pairing with the sampler's acquire load of valid in ContextApi::get. +// record on the same carrier, gated on record->valid. Each write follows a detach -> mutate -> +// attach protocol: store valid=0, release fence, mutate, release fence, store valid=1. On x86 the +// release fence is a compiler barrier; on aarch64 it is a real barrier pairing with the sampler's +// acquire load of valid in ContextApi::get. // --------------------------------------------------------------------------- -// Byte layout constants shared with ThreadContext (see otel_context.h / ThreadContext.java). static const int OTEL_LRS_ENTRY_SIZE = 18; // fixed attrs_data[0] entry: key(1)+len(1)+16 hex bytes // Writes the full fixed LRS attrs_data entry: header (key_index=0, length=16) at attrs_data[0..2) // plus the 16 hex value bytes at attrs_data[2..18). The combined write/clear entry points -// (setTraceContext0 / clearTraceContext0) call this so they establish the LRS entry themselves -// rather than relying on the ThreadContext ctor — i.e. they work on a record that only saw the -// ProfiledThread zero-init, the phase-2 pure-native case where no DirectByteBuffer / ThreadContext -// was ever created. Mirrors ThreadContext's LRS entry layout. +// (setTraceContext0 / clearTraceContext0) establish this entry themselves on a record that only +// ever saw the ProfiledThread zero-init. // // Note: the single-attribute path (setContextValue0) does NOT write this entry; it assumes a // preceding setTraceContext0 has already established it (the production order — app tags are set @@ -800,7 +751,7 @@ static inline void otelWriteLrsEntry(OtelThreadContextRecord* record, u64 v) { } // Compacts out the attrs_data entry with the given OTEP key index; returns the new size. -// Mirrors ThreadContext.compactOtepAttribute. Record must be detached. +// Record must be detached. static int otelCompactAttr(OtelThreadContextRecord* record, int otepKeyIndex) { int currentSize = record->attrs_data_size; uint8_t* d = record->attrs_data; @@ -825,7 +776,7 @@ static int otelCompactAttr(OtelThreadContextRecord* record, int otepKeyIndex) { } // Replaces/inserts an attribute value in attrs_data (record must be detached). Returns false on -// attrs_data overflow (nothing appended). Mirrors ThreadContext.replaceOtepAttribute. +// attrs_data overflow (nothing appended). static bool otelReplaceAttr(OtelThreadContextRecord* record, int otepKeyIndex, const uint8_t* utf8, int valueLen) { int currentSize = otelCompactAttr(record, otepKeyIndex); @@ -886,10 +837,10 @@ Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass un // clearTraceContext0. The public setTraceContext wrapper enforces this by throwing // IllegalArgumentException, so a zero span reaching here is a direct-JNI/contract violation. assert(spanId != 0 && "setTraceContext0 requires a non-zero span; use clearTraceContext0 to clear"); - // Publish the OTEP TLS pointer and mark the thread initialized on first native write, exactly - // as the DirectByteBuffer path does in initializeContextTLS0. Without this a thread that only - // ever uses the all-native API writes a record that ContextApi::get / the wallclock sampler - // ignore (both gate on isContextInitialized) and that external OTEP readers can't discover. + // Publish the OTEP TLS pointer and mark the thread initialized on first native write. Without + // this a thread that never explicitly initializes context writes a record that ContextApi::get + // / the wallclock sampler ignore (both gate on isContextInitialized) and that external OTEP + // readers can't discover. if (!thrd->isContextInitialized()) { ContextApi::initializeContextTLS(thrd); } @@ -931,7 +882,7 @@ Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass un } // Combined per-deactivation clear: zeros scalar context + custom slots and leaves the record -// detached (valid=0), mirroring the DBB clear path (setContext(0,0,0,0) + clearContextValue*). +// detached (valid=0). extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass unused) { ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); @@ -951,7 +902,7 @@ Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass *lrs = 0; record->attrs_data_size = (uint16_t)OTEL_LRS_ENTRY_SIZE; otelWriteLrsEntry(record, 0); - // clear path leaves valid=0 (no attach), mirroring ThreadContext.clearContextDirect. + // clear path leaves valid=0 (no attach) — a cleared context is not "active". } // Single pre-resolved attribute write (sidecar encoding + attrs_data value) in one detach/attach @@ -977,10 +928,9 @@ Java_com_datadoghq_profiler_JavaProfiler_setContextValue0(JNIEnv* env, jclass un // Publish the record (valid=1) after the write: setting a value means "make it visible", so an // attribute is observable to the sampler even with no active span (zero trace/span). This is the // app-context-independent-of-span model dd-trace-java relies on — app-owned attributes (e.g. - // http.route) and the reapply-after-deactivation path stay visible between spans. Mirrors the DBB - // single-attribute setter ThreadContext.setContextAttributeDirect, which likewise always - // re-attaches. (clearContextValue0, by contrast, preserves the prior valid: removing a value must - // not resurrect a deactivated record.) + // http.route) and the reapply-after-deactivation path stay visible between spans. + // (clearContextValue0, by contrast, preserves the prior valid: removing a value must not + // resurrect a deactivated record.) __atomic_store_n(&record->valid, (uint8_t)0, __ATOMIC_RELAXED); __atomic_thread_fence(__ATOMIC_RELEASE); @@ -1022,10 +972,7 @@ Java_com_datadoghq_profiler_JavaProfiler_clearContextValue0(JNIEnv* env, jclass } // Copies the current thread's custom-attribute sidecar tag encodings (enc[]) into out, reading the -// record directly via ProfiledThread::current() — no ThreadContext / DirectByteBuffer, so it does -// NOT reset the record the way the deprecated DBB copyTags does. This lets a caller observe -// encodings written through the all-native setContextValue path (introspection / tests); the DBB -// read went through ThreadContext, whose ctor resets the record and would clobber native writes. +// record directly via ProfiledThread::current() without mutating it. Introspection / test use. extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass unused, jintArray out) { if (out == nullptr) { @@ -1050,7 +997,7 @@ Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass un } extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_ThreadContext_registerConstant0(JNIEnv* env, jclass unused, jstring value) { +Java_com_datadoghq_profiler_ContextValueCache_registerConstant0(JNIEnv* env, jclass unused, jstring value) { JniString value_str(env, value); u32 encoding = Profiler::instance()->contextValueMap()->bounded_lookup( value_str.c_str(), value_str.length(), 1 << 16); @@ -1088,3 +1035,87 @@ Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) ContextApi::get(spanId, rootSpanId); TEST_LOG("===> Context: tid:%lu, spanId=%lu, rootSpanId=%lu", OS::threadId(), spanId, rootSpanId); } + +// ---- Test-only reads of the current thread's OTEP record ----------------------------------- +// Each reads the current carrier's record directly via ProfiledThread::current(), with no +// detach/attach (diagnostic-only, not on any signal-handler or hot write path). + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testGetSpanId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return 0; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + uint64_t beSpan; + memcpy(&beSpan, record->span_id, 8); + return (jlong)__builtin_bswap64(beSpan); +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testGetRootSpanId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return 0; + } + u32* enc = thrd->getOtelTagEncodingsPtr(); + u64* lrs = reinterpret_cast(enc + DD_TAGS_CAPACITY); + return (jlong)*lrs; +} + +extern "C" DLLEXPORT jstring JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testReadTraceId0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return nullptr; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + static const char HEXD[16] = + {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; + char hex[33]; + for (int i = 0; i < 16; i++) { + uint8_t b = record->trace_id[i]; + hex[i * 2] = HEXD[b >> 4]; + hex[i * 2 + 1] = HEXD[b & 0xF]; + } + hex[32] = '\0'; + return env->NewStringUTF(hex); +} + +extern "C" DLLEXPORT jstring JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testReadContextAttribute0(JNIEnv* env, jclass unused, jint slot) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr || slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { + return nullptr; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + int targetKey = slot + 1; + int size = record->attrs_data_size; + uint8_t* d = record->attrs_data; + int pos = 0; + while (pos + 2 <= size) { + int k = d[pos]; + int len = d[pos + 1]; + if (pos + 2 + len > size) { + break; + } + if (k == targetKey) { + char buf[256]; + memcpy(buf, d + pos + 2, len); + buf[len] = '\0'; + return env->NewStringUTF(buf); + } + pos += 2 + len; + } + return nullptr; +} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_testIsContextValid0(JNIEnv* env, jclass unused) { + ProfiledThread* thrd = ProfiledThread::current(); + if (thrd == nullptr) { + return JNI_FALSE; + } + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); + return __atomic_load_n(&record->valid, __ATOMIC_ACQUIRE) ? JNI_TRUE : JNI_FALSE; +} diff --git a/ddprof-lib/src/main/cpp/otel_context.h b/ddprof-lib/src/main/cpp/otel_context.h index 82a6701b2d..cee65b747b 100644 --- a/ddprof-lib/src/main/cpp/otel_context.h +++ b/ddprof-lib/src/main/cpp/otel_context.h @@ -57,8 +57,7 @@ struct __attribute__((packed)) OtelThreadContextRecord { uint8_t attrs_data[OTEL_MAX_ATTRS_DATA_SIZE]; }; static_assert(sizeof(OtelThreadContextRecord) == OTEL_MAX_RECORD_SIZE, - "OtelThreadContextRecord size must match OTEL_MAX_RECORD_SIZE (640 bytes); " - "update the Java constant ThreadContext.OTEL_MAX_RECORD_SIZE if the struct changes"); + "OtelThreadContextRecord size must match OTEL_MAX_RECORD_SIZE (640 bytes)"); // OTEP #4947 TLS pointer — MUST appear in dynsym for external profiler discovery DLLEXPORT extern thread_local OtelThreadContextRecord* otel_thread_ctx_v1; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 293a8c15a8..ff8f68a026 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -90,13 +90,10 @@ class ProfiledThread : public ThreadLocalData { #endif // alignas(8) + sizeof(OtelThreadContextRecord)==640 (multiple of 8) guarantee // _otel_tag_encodings sits at +640 with no padding, so the three fields form one - // 688-byte contiguous region exposed as a combined DirectByteBuffer. + // 688-byte contiguous region. alignas(8) OtelThreadContextRecord _otel_ctx_record; - // These two fields MUST be contiguous and 8-byte aligned — the JNI layer - // exposes them as a single DirectByteBuffer (sidecar), and VarHandle long - // views require 8-byte alignment for the buffer base address. + // 8-byte aligned so VarHandle long views over this region require no unaligned access. // Read invariant: sidecar readers must gate on record->valid (see ContextApi::get). - // ThreadContext.restore() relies on this to perform a bulk memcpy under valid=0. alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY]; u64 _otel_local_root_span_id; diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index eac9f3fd37..798da6c3d3 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -51,9 +51,9 @@ static inline bool hasKnownActiveTraceContext(ProfiledThread* thread) { } OtelThreadContextRecord* record = thread->getOtelContextRecord(); - // record->valid is not a context-presence bit. ThreadContext leaves cleared + // record->valid is not a context-presence bit. clearTraceContext0 leaves cleared // records invalid indefinitely, so gating on valid=1 disables wallprecheck for - // the common no-context state after a Java ThreadLocal reset. + // the common no-context state after deactivation. return loadSpanId(record) != 0; } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java deleted file mode 100644 index d9fb237f52..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.ByteBuffer; - -/** - * Version-agnostic wrapper for direct ByteBuffer memory operations with explicit memory ordering guarantees. - * - *

This class provides low-level memory access operations on direct ByteBuffers with precise control - * over memory ordering semantics. It abstracts differences between Java 8 (using sun.misc.Unsafe) and - * Java 9+ (using VarHandle APIs) to provide consistent behavior across JVM versions. - * - *

The class supports two types of memory ordering: - *

    - *
  • Ordered writes (release semantics): Prevents reordering with prior writes but allows - * subsequent operations to be reordered before this write. More efficient than volatile writes.
    • - *
  • Volatile writes (full barrier): Prevents reordering with both prior and subsequent - * operations. Ensures writes are completed before subsequent operations begin.
    • - *
- * - *

Signal Handler Safety: The primary use case is protecting write sequences from being - * observed in inconsistent states by async signal handlers (e.g., SIGPROF). When a signal interrupts - * a write sequence on the same thread, the signal handler must see either the complete write sequence - * or recognize the write is in-progress. This requires careful memory ordering even though both writer - * and reader execute on the same thread. - * - *

This is primarily used by the profiler for writing thread-local context data (span IDs, checksums) - * to direct ByteBuffers that are shared between Java and native code via JNI. - * - *

Thread Safety: This class is thread-safe. Individual write operations provide their own - * memory ordering guarantees as documented. - */ -public final class BufferWriter { - /** - * Service Provider Interface for version-specific buffer write implementations. - * - *

Implementations of this interface provide the actual low-level memory access operations: - *

    - *
  • {@code BufferWriter8} - Java 8 implementation using sun.misc.Unsafe
    • - *
  • {@code BufferWriter9} - Java 9+ implementation using VarHandle
    • - *
- * - *

The implementation is selected automatically based on the runtime Java version. - */ - public interface Impl { - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - void writeOrderedLong(ByteBuffer buffer, int offset, long value); - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - void writeInt(ByteBuffer buffer, int offset, int value); - - /** - * Executes a store-store memory fence. - * - *

Ensures that stores before the fence are visible before stores after it. - * Cheaper than fullFence on ARM (~5ns vs ~50ns) since it only orders - * stores, not loads. Sufficient for publication protocols where the writer - * needs to ensure data writes are visible before a flag/pointer write. - */ - void storeFence(); - } - - private final Impl impl; - - /** - * Creates a new BufferWriter with the appropriate implementation for the current Java version. - * - *

The implementation is selected based on the runtime Java version: - *

    - *
  • Java 8: Uses {@code BufferWriter8} which leverages sun.misc.Unsafe for memory operations
    • - *
  • Java 9+: Uses {@code BufferWriter9} which leverages VarHandle for memory operations
    • - *
- * - *

The implementation is loaded reflectively to avoid compile-time dependencies on - * version-specific APIs. - * - * @throws RuntimeException if the implementation class cannot be loaded or instantiated - */ - public BufferWriter() { - try { - if (Platform.isJavaVersion(8)) { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter8").getConstructor().newInstance(); - } else { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter9").getConstructor().newInstance(); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

This is commonly used for writing context fields (span IDs, root span IDs) in a write - * sequence that is protected by a volatile sentinel value. Ensures that if a signal handler - * interrupts after this write, all prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - impl.writeOrderedLong(buffer, offset, value); - } - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - public void writeOrderedInt(ByteBuffer buffer, int offset, int value) { - impl.writeInt(buffer, offset, value); - } - - /** - * Executes a store-store memory fence. - * - *

Ensures stores before the fence are globally visible before stores after it. - * On ARM this emits DMB ISHST (~2 ns); on x86 it is a compiler-only barrier. - */ - public void storeFence() { - impl.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java deleted file mode 100644 index 08af580522..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; - -import java.lang.reflect.Field; -import java.nio.ByteBuffer; - -public final class BufferWriter8 implements BufferWriter.Impl { - private static final Unsafe UNSAFE; - static { - Unsafe unsafe = null; - // a safety and testing valve to disable unsafe access - if (Platform.isJavaVersion(8)) { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - unsafe = (Unsafe) f.get(null); - } catch (Exception ignore) { - // On Java 8 will never happen - } - } - UNSAFE = unsafe; - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - UNSAFE.putOrderedLong(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - UNSAFE.putOrderedInt(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void storeFence() { - UNSAFE.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java index 09ed77f638..2ef1161bac 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java @@ -16,28 +16,24 @@ package com.datadoghq.profiler; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; /** - * DirectByteBuffer context wrapper. - * - * @deprecated Superseded by {@link JavaProfiler#setTraceContext} / - * {@link JavaProfiler#setContextValue} (all-native). Removed in phase 3. + * Resolves configured context attribute names to their fixed slot indices in the all-native + * context record. Deduplicates and truncates the configured attribute list to {@link + * JavaProfiler#MAX_CONTEXT_SLOTS} slots; callers use {@link #offsetOf} to translate an attribute + * name into the slot index passed to {@link JavaProfiler#setContextValue}. */ -@Deprecated public class ContextSetter { private final List attributes; - private final JavaProfiler profiler; public ContextSetter(JavaProfiler profiler, List attributes) { - this.profiler = profiler; Set unique = new HashSet<>(attributes); this.attributes = new ArrayList<>(unique.size()); - for (int i = 0; i < Math.min(attributes.size(), ThreadContext.MAX_CUSTOM_SLOTS); i++) { + for (int i = 0; i < Math.min(attributes.size(), JavaProfiler.MAX_CONTEXT_SLOTS); i++) { String attribute = attributes.get(i); if (unique.remove(attribute)) { this.attributes.add(attribute); @@ -45,71 +41,12 @@ public ContextSetter(JavaProfiler profiler, List attributes) { } } - public int[] snapshotTags() { - int[] snapshot = new int[attributes.size()]; - profiler.copyTags(snapshot); - return snapshot; - } - - /** - * Copies current sidecar encodings into {@code snapshot}. The array must have at least - * {@code attributes.size()} elements; arrays shorter than {@code attributes.size()} are - * silently ignored. Indices {@code [attributes.size(), snapshot.length)} are zeroed after - * copying to prevent stale data from leaking to the caller. - * Use the no-arg {@link #snapshotTags()} overload to obtain a correctly sized array. - */ - public void snapshotTags(int[] snapshot) { - if (snapshot.length >= attributes.size()) { - profiler.copyTags(snapshot); - Arrays.fill(snapshot, attributes.size(), snapshot.length, 0); - } - } - public int offsetOf(String attribute) { return attributes.indexOf(attribute); } - /** Number of (deduplicated, truncated) context attribute slots. Pure Java; no native/DBB read. */ + /** Number of (deduplicated, truncated) context attribute slots. */ public int size() { return attributes.size(); } - - public boolean setContextValue(String attribute, String value) { - return setContextValue(offsetOf(attribute), value); - } - - public boolean setContextValue(int offset, String value) { - if (offset >= 0) { - return profiler.setContextAttribute(offset, value); - } - return false; - } - - /** - * Re-applies attribute values from precomputed constant IDs and UTF-8 bytes, indexed by slot - * (as produced by {@link #snapshotTags(int[])}). Restores both the DD sidecar encoding and the - * OTEP attrs_data value for every slot whose constantId is {@code > 0}, in a single atomic - * publish — no String allocation, hashing, or cache lookup. Intended for re-applying - * application-managed context after a {@code setContext} span activation wipes the slots. - * - *

Partial-write on overflow. A {@code false} return does not mean the record is - * unchanged: slots that were written before an attrs_data overflow remain published. Overflowed - * slots are zeroed in both the sidecar and attrs_data views. Callers must not assume the record - * is unmodified when {@code false} is returned. - */ - public boolean setContextValuesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return profiler.setContextAttributesByIdAndBytes(constantIds, utf8); - } - - public boolean clearContextValue(String attribute) { - return clearContextValue(offsetOf(attribute)); - } - - public boolean clearContextValue(int offset) { - if (offset >= 0) { - profiler.clearContextAttribute(offset); - return true; - } - return false; - } } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java deleted file mode 100644 index 6aa18260d7..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextStorageMode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -/** - * Scope of the OTEL context {@link ThreadContext} storage actually in effect, as reported by - * {@link JavaProfiler#contextStorageMode()}. See {@link OtelContextStorage} for how it is - * selected. - */ -public enum ContextStorageMode { - /** Carrier-scoped via {@code jdk.internal.misc.CarrierThreadLocal} (JDK 21+). */ - CARRIER, - /** Legacy virtual-thread-scoped plain {@link ThreadLocal}. */ - THREAD -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java index 7a0eaafb9d..619a85e22a 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextValueCache.java @@ -40,9 +40,8 @@ * session become stale. {@link JavaProfiler} therefore calls {@link #clear()} on every {@code start} * command; a subsequent {@link #resolve} re-registers the value and re-caches its new encoding. * - *

Replaces the per-{@link ThreadContext} value cache: in the all-native model there is no - * per-thread {@code ThreadContext} instance to host it, and a global cache avoids duplicating the - * table across every carrier / virtual thread. + *

In the all-native model there is no per-thread instance to host this cache, so it is a + * single process-wide table shared across every carrier / virtual thread. */ final class ContextValueCache { @@ -69,15 +68,43 @@ static final class Entry { /** * Resolves {@code value} to its cached {@code (encoding, utf8)} pair, registering it on a miss. + * The sole entrypoint — deliberately not overloaded on {@code String} alongside {@code + * CharSequence}, since Java resolves overloads by static (not runtime) type and a {@code + * String}-typed local would silently bind to a different method than a {@code + * CharSequence}-typed one holding the same value. {@link #resolveString} is the private, + * no-allocation fast path for callers (like this method, on a {@code String} runtime type) + * that already hold a {@code String}. + * + *

The cache-hit path never calls {@code toString()} on a non-{@code String} {@code + * CharSequence}: {@link #contentHashCode} replicates {@link String#hashCode()}'s algorithm so + * a {@code CharSequence} and a content-equal cached {@code String} key land in the same slot, + * and {@link String#contentEquals(CharSequence)} compares without allocating. This matters for + * callers (e.g. {@code setTraceContext}) whose value may be a non-{@code String} + * implementation — dd-trace-java passes {@code UTF8BytesString}/{@code SubSequence} span-name + * views specifically to avoid forcing a {@code String} allocation on the hot path. + * {@code toString()} is only paid on a genuine miss, where a {@code String} is needed anyway + * to hand to {@code registerConstant0} and to store as the entry's key. * * @return the entry, or {@code null} if the value cannot be represented — {@code null} input, a * UTF-8 encoding longer than {@value #MAX_VALUE_BYTES} bytes, or a full native Dictionary * (encoding {@code < 0}). Callers treat {@code null} as "no attribute" (skip / clear). */ - Entry resolve(String value) { + Entry resolve(CharSequence value) { if (value == null) { return null; } + if (value instanceof String) { + return resolveString((String) value); + } + int slot = contentHashCode(value) & MASK; + Entry e = table.get(slot); + if (e != null && e.key.contentEquals(value)) { + return e; // hit — no allocation, no JNI + } + return resolveString(value.toString()); + } + + private Entry resolveString(String value) { int slot = value.hashCode() & MASK; Entry e = table.get(slot); if (e != null && value.equals(e.key)) { @@ -89,7 +116,7 @@ Entry resolve(String value) { if (utf8.length > MAX_VALUE_BYTES) { return null; } - int encoding = ThreadContext.registerConstant0(value); + int encoding = registerConstant0(value); if (encoding < 0) { return null; // Dictionary full } @@ -98,6 +125,19 @@ Entry resolve(String value) { return ne; } + /** + * Computes {@link String#hashCode()}'s polynomial ({@code s[0]*31^(n-1) + ... + s[n-1]}) over + * an arbitrary {@code CharSequence}, without materializing a {@code String}. + */ + private static int contentHashCode(CharSequence value) { + int h = 0; + int len = value.length(); + for (int i = 0; i < len; i++) { + h = 31 * h + value.charAt(i); + } + return h; + } + /** * Drops all cached entries. Called when a fresh recording session starts and the native * Dictionary is reset, so stale encodings from the previous session are not reused. A concurrent @@ -109,4 +149,7 @@ void clear() { table.set(i, null); } } + + /** Registers {@code value} in the native Dictionary, returning its encoding, or a negative value if full. */ + private static native int registerConstant0(String value); } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index eab2de7580..74ebd9ac7c 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -64,43 +64,16 @@ private static boolean isVirtualThread(Thread thread) { } } - // Storage for profiling context. Scoped to the carrier thread when available so a - // mounted virtual thread resolves to its current carrier's OTEP record (the record the - // sampler reads); falls back to plain thread-local storage otherwise. See - // OtelContextStorage for the mode selection and the rationale. - private final ThreadLocal tlsContextStorage = OtelContextStorage.create(); - // Process-wide value->(encoding, utf8) cache for the all-native context write path // (setTraceContext / setContextValue). See ContextValueCache. One instance on the singleton. private final ContextValueCache contextValueCache = new ContextValueCache(); // Number of custom attribute slots on the all-native path. Must equal the native // DD_TAGS_CAPACITY (context.h); kept as a literal (not derived via JNI) because it bounds - // array-slot checks that can run before the native library is loaded, and kept independent of - // ThreadContext so the phase-3 removal of ThreadContext does not strand this constant. Drift - // from the native value is caught at test time by MaxContextSlotsTest via maxContextSlots0(). + // array-slot checks that can run before the native library is loaded. Drift from the native + // value is caught at test time by MaxContextSlotsTest via maxContextSlots0(). static final int MAX_CONTEXT_SLOTS = 10; - /** - * Returns the calling thread's (or, in carrier mode, its current carrier's) - * {@link ThreadContext}, creating and caching it on first use. Replaces the previous - * {@code ThreadLocal.withInitial(...)} supplier: a carrier-scoped storage instance is - * built reflectively and cannot carry a supplier, so lazy initialization is done here. - * - *

Race-free without synchronization: a carrier runs at most one mounted virtual - * thread at a time and this method has no blocking point, so no unmount can occur - * mid-call. A redundant re-init could at worst produce a second {@link ThreadContext} - * over the same carrier record, which is harmless. - */ - private ThreadContext currentContext() { - ThreadContext ctx = tlsContextStorage.get(); - if (ctx == null) { - ctx = initializeThreadContext(); - tlsContextStorage.set(ctx); - } - return ctx; - } - private JavaProfiler() { } @@ -250,54 +223,9 @@ public void removeThread() { filterThreadRemove0(); } - /** - * Passing context identifier to a profiler. This ID is thread-local and is dumped in - * the JFR output only. 0 is a reserved value for "no-context". - * - *

Note: {@code rootSpanId} maps to {@code localRootSpanId} internally. A synthetic - * trace_id of {@code [0, spanId]} is written to the OTEP record. For correct W3C - * trace ID interop use {@link #setContext(long, long, long, long)}. - * - * @param spanId Span identifier that should be stored for current thread - * @param rootSpanId Local root span identifier (used for endpoint correlation) - * @deprecated Use {@link #setContext(long, long, long, long)} for full OTEP interop. - */ - @Deprecated - public void setContext(long spanId, long rootSpanId) { - currentContext().put(spanId, rootSpanId); - } - - /** - * Sets trace context with full 128-bit W3C trace ID, span ID, and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId Span identifier - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - * @deprecated DirectByteBuffer path; use {@link #setTraceContext} (all-native). Removed in phase 3. - */ - @Deprecated - public void setContext(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - currentContext().put(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Resets the current thread's context to zero (traceId=0, spanId=0, localRootSpanId=0). - * Custom context attributes are also cleared. - * - * @deprecated DirectByteBuffer path; use {@link #clearTraceContext} (all-native). Removed in phase 3. - */ - @Deprecated - public void clearContext() { - currentContext().put(0, 0, 0, 0); - } - // ---- All-native context write API (OTEP #4947) -------------------------------------------- - // Provisional names (subject to dd-trace-java coordination). These resolve the current carrier's - // OTEP record inside a single JNI call per operation — no cached DirectByteBuffer, so they are - // race-free under virtual-thread migration (see the design note and setTraceContext0 et al.). - // They coexist with the deprecated DirectByteBuffer path below; both write the same native - // record, and no thread uses both at once. + // Each of these resolves the current carrier's OTEP record inside a single JNI call per + // operation, so they are race-free under virtual-thread migration. /** * Combined per-scope-activation write: full trace/span context plus up to two span-derived @@ -334,7 +262,7 @@ public void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long e1 == null ? -1 : slot1, e1 == null ? 0 : e1.encoding, e1 == null ? null : e1.utf8); } - /** Combined per-scope-deactivation clear (replaces {@link #clearContext()} on the native path). */ + /** Clears the trace context on span deactivation. */ public void clearTraceContext() { clearTraceContext0(); } @@ -352,9 +280,9 @@ public void clearTraceContext() { * full * @throws IllegalArgumentException if {@code slot} is out of range */ - public boolean setContextValue(int slot, CharSequence value) { + public boolean setContextValue(int slot, String value) { requireValidSlot(slot); - ContextValueCache.Entry e = value == null ? null : contextValueCache.resolve(value.toString()); + ContextValueCache.Entry e = value == null ? null : contextValueCache.resolve(value); if (e == null) { clearContextValue0(slot); return false; @@ -375,10 +303,9 @@ public void clearContextValue(int slot) { /** * Copies the current thread's custom-attribute sidecar tag encodings into {@code out} (index = - * slot), reading the native record directly — no {@link ThreadContext} / DirectByteBuffer, so it - * does not reset the record. Unlike the deprecated DBB read path, this observes encodings written - * through the all-native {@link #setContextValue} path. Introspection / test use; entries beyond - * {@code MAX_CONTEXT_SLOTS} are left untouched. + * slot), reading the native record directly. Observes encodings written through the all-native + * {@link #setContextValue} path. Introspection / test use; entries beyond {@code + * MAX_CONTEXT_SLOTS} are left untouched. */ public void copyContextTags(int[] out) { copyContextTags0(out); @@ -411,70 +338,7 @@ private ContextValueCache.Entry resolveContextValue(int slot, CharSequence value if (slot < 0 || value == null) { return null; } - return contextValueCache.resolve(value.toString()); - } - - /** - * Sets a custom context attribute at the given slot offset for the current thread. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values return {@code false} - * @param value the string value to record; {@code null} returns {@code false} without - * writing; an empty string is written as a zero-length entry (not a clear — - * use {@link #clearContextAttribute(int)} to remove a value) - * @return true if the value was recorded; false if {@code offset} is out of range, - * {@code value} is null, the Dictionary is full, or {@code attrs_data} overflows - * for this slot - * @deprecated DirectByteBuffer path; use {@link #setContextValue} (all-native). Removed in phase 3. - */ - @Deprecated - public boolean setContextAttribute(int offset, String value) { - return currentContext().setContextAttribute(offset, value); - } - - /** - * Clears the custom context attribute at the given slot offset for the current thread. - * Zeros the sidecar encoding and removes it from OTEP {@code attrs_data}. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values are silently ignored - * @deprecated DirectByteBuffer path; use {@link #clearContextValue} (all-native). Removed in phase 3. - */ - @Deprecated - public void clearContextAttribute(int offset) { - currentContext().clearContextAttribute(offset); - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes for - * the current thread in a single detach/attach window. - * - *

    - *
  • Slots with {@code constantIds[i] <= 0} are skipped.
    • - *
  • Returns {@code false} without writing if the thread's record is not currently valid - * (span-less), to avoid resurrecting a cleared record.
    • - *
  • On {@code attrs_data} overflow, the overflowed slot's sidecar is zeroed and - * {@code false} is returned; slots written before the overflow are retained.
    • - *
- * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most 255 bytes - * (the OTEP attrs_data entry length field is one byte) for every slot - * whose {@code constantId > 0} - * @return true if every slot with {@code constantId > 0} was written; false on a cleared - * (span-less) record, or {@code attrs_data} overflow for any slot - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} is null - * @throws IllegalArgumentException if the arrays have different lengths, exceed the slot limit, - * or any active {@code utf8[i]} exceeds 255 bytes - * @deprecated DirectByteBuffer path; unused by dd-trace-java. Removed in phase 3. - */ - @Deprecated - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return currentContext().setContextAttributesByIdAndBytes(constantIds, utf8); - } - - @Deprecated - void copyTags(int[] snapshot) { - currentContext().copyCustoms(snapshot); + return contextValueCache.resolve(value); } /** @@ -589,15 +453,6 @@ public Map getDebugCounters() { return counters; } - private static ThreadContext initializeThreadContext() { - long[] metadata = new long[6]; - ByteBuffer buffer = initializeContextTLS0(metadata); - if (buffer == null) { - throw new IllegalStateException("Failed to initialize OTEL TLS — ProfiledThread not available"); - } - return new ThreadContext(buffer, metadata); - } - private static native boolean init0(); private native void stop0() throws IllegalStateException; private native String execute0(String command) throws IllegalArgumentException, IllegalStateException, IOException; @@ -635,22 +490,6 @@ private static ThreadContext initializeThreadContext() { private static native String getStatus0(); - /** - * Initializes context TLS for the current thread and returns a single DirectByteBuffer - * spanning the OTEP record + tag-encoding sidecar + LRS (688 bytes, contiguous in - * ProfiledThread). Sets otel_thread_ctx_v1 permanently to the thread's - * OtelThreadContextRecord. - * - * @param metadata output array filled with absolute offsets into the returned buffer: - * [0] VALID_OFFSET — offset of 'valid' field - * [1] TRACE_ID_OFFSET — offset of 'trace_id' field - * [2] SPAN_ID_OFFSET — offset of 'span_id' field - * [3] ATTRS_DATA_SIZE_OFFSET — offset of 'attrs_data_size' field - * [4] ATTRS_DATA_OFFSET — offset of 'attrs_data' field - * [5] LRS_OFFSET — offset of local_root_span_id - */ - private static native ByteBuffer initializeContextTLS0(long[] metadata); - // All-native context write primitives (OTEP #4947). Each resolves the current carrier's record // inside the JNI call (which pins a mounted virtual thread to its carrier), so there is no // cached per-thread buffer to dangle. See the native implementations in javaApi.cpp and the @@ -672,35 +511,6 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ private static native boolean consumeContextDictionaryReset0(); - /** - * Returns the {@link ThreadContext} for the current storage slot (the calling thread, or in - * {@link ContextStorageMode#CARRIER} its current carrier). - * - *

Do not cache the returned instance across a point where the calling thread may be - * unmounted and remounted on a different carrier (any blocking operation on a virtual - * thread). In carrier mode the returned context's buffer targets the carrier that was mounted - * at call time; after migration it no longer corresponds to the current carrier's record — the - * sampler reads the new carrier, and once the old carrier's OS thread exits the buffer dangles. - * Callers that write context (span/attributes) should re-fetch per use — the {@code setContext*} - * methods already do this internally via {@code currentContext()}. - * - * @deprecated DirectByteBuffer path (test/diagnostic only); the all-native API is stateless and - * exposes no per-thread handle. Removed in phase 3. - */ - @Deprecated - public ThreadContext getThreadContext() { - return currentContext(); - } - - /** - * Diagnostics/tests: the resolved OTEL context storage mode, as selected by - * {@code -D}{@value OtelContextStorage#MODE_PROPERTY} and the availability of - * {@code jdk.internal.misc.CarrierThreadLocal}. - */ - public ContextStorageMode contextStorageMode() { - return OtelContextStorage.modeOf(tlsContextStorage); - } - // --- test and debug utility methods /** @@ -711,13 +521,38 @@ public ContextStorageMode contextStorageMode() { public static native void dumpContext(); - /** - * Resets the cached ThreadContext for the current storage slot — the calling thread in - * {@link ContextStorageMode#THREAD}, or its current carrier in - * {@link ContextStorageMode#CARRIER}. The next call to {@link #getThreadContext()} - * or any {@code setContext} overload will re-create it with fresh OTEL TLS buffers. - */ - public void resetThreadContext() { - tlsContextStorage.remove(); + // ---- Test-only reads of the current thread's OTEP record ---------------------------------- + // Each resolves the current carrier's record directly (like the write primitives above) with + // no cached buffer and no per-thread Java object; introspection/test use only. + + /** Test-only: the current thread's span ID from the OTEP record. */ + long testGetSpanId() { + return testGetSpanId0(); } + + /** Test-only: the current thread's local root span ID from the OTEP record. */ + long testGetRootSpanId() { + return testGetRootSpanId0(); + } + + /** Test-only: the current thread's trace ID as a 32-char lowercase hex string. */ + String testReadTraceId() { + return testReadTraceId0(); + } + + /** Test-only: the current thread's custom attribute value at {@code slot}, or null if unset. */ + String testReadContextAttribute(int slot) { + return testReadContextAttribute0(slot); + } + + /** Test-only: whether the current thread's OTEP record is currently valid (published). */ + boolean testIsContextValid() { + return testIsContextValid0(); + } + + private static native long testGetSpanId0(); + private static native long testGetRootSpanId0(); + private static native String testReadTraceId0(); + private static native String testReadContextAttribute0(int slot); + private static native boolean testIsContextValid0(); } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java index 2110ac65c7..f3eebf1917 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java @@ -235,7 +235,7 @@ public ProcessContext readProcessContext() { * element is null, the entire process context publish is * skipped - no context is published (a warning is logged) and * no exception is thrown. Order must match the indices used - * with {@link ThreadContext#setContextAttribute(int, String)}. + * with {@link JavaProfiler#setContextValue(int, String)}. * * @throws NullPointerException if {@code attributeKeys} is null * diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java deleted file mode 100644 index 1bc399f9c6..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OtelContextStorage.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.util.Locale; - -/** - * Factory for the thread-local that backs {@link ThreadContext} storage, and the - * home of the context-storage mode selection. - * - *

Why this exists: the OTEP record a {@link ThreadContext} writes to is embedded in - * the carrier's native {@code ProfiledThread}, and that carrier's record is what - * the sampler (an async-signal handler bound to the carrier OS thread) reads. A plain - * {@link ThreadLocal} keys the {@code ThreadContext} — and therefore its - * {@code DirectByteBuffer} conduit — by the virtual thread, pinning it to - * whichever carrier was mounted at first use. That is wrong once the virtual thread - * migrates (writes land on the old carrier, so a sampler on the new carrier sees stale - * or empty context) and unsafe once the old carrier's OS thread exits (the record is - * freed while the buffer keeps being written — a use-after-free that can corrupt - * JVM-owned native memory). See {@code ThreadContext} and the design note. - * - *

{@link ContextStorageMode#CARRIER} scopes storage to the carrier via - * {@code jdk.internal.misc.CarrierThreadLocal} (JDK 21+), whose {@code get()/set()/remove()} - * operate on the current carrier's map even when called from a mounted virtual thread. A - * mounted virtual thread then always resolves to its current carrier's record, which is - * exactly what the sampler reads; storage lifetime matches the native record's lifetime, - * so the dangling-buffer window is eliminated. - * - *

{@code CarrierThreadLocal} lives in a non-exported package, so {@link ContextStorageMode#CARRIER} - * needs {@code --add-exports java.base/jdk.internal.misc=ALL-UNNAMED} at runtime (a - * {@code -javaagent} can grant this via {@code Instrumentation.redefineModule}). When the - * type is missing (older JDK) or inaccessible (export not granted), we degrade to - * {@link ContextStorageMode#THREAD} — today's plain {@code ThreadLocal} behavior — never failing hard. - * - *

The instance is held as a {@link ThreadLocal} (the supertype, available on the Java 8 - * baseline) and constructed reflectively, so calls dispatch virtually to the carrier-scoped - * overrides with no per-call reflection and no compile-time dependency on the internal type. - */ -final class OtelContextStorage { - - /** - * Selector system property: {@code auto} (default) | {@code carrier} | {@code thread}. - * Named under {@code ddprof.debug.*} because it is an internal knob, not part of the - * supported configuration surface. - *

    - *
  • {@code auto} — use {@link ContextStorageMode#CARRIER} when available, else - * {@link ContextStorageMode#THREAD} (logged loudly on JDK 21+, where the fallback is - * unsafe under virtual threads).
    • - *
  • {@code carrier} — require carrier scoping; {@link #create()} throws if unavailable.
    • - *
  • {@code thread} — force legacy behavior (disables carrier scoping entirely).
    • - *
- */ - static final String MODE_PROPERTY = "ddprof.debug.context.storage.mode"; - - private static final String INTERNAL_CTL = "jdk.internal.misc.CarrierThreadLocal"; - - private OtelContextStorage() {} - - /** - * The scope of a storage instance returned by {@link #create()} — a property of the - * instance itself (its concrete type), so there is no shared mutable state to leak - * between callers. - */ - static ContextStorageMode modeOf(ThreadLocal storage) { - return storage != null && INTERNAL_CTL.equals(storage.getClass().getName()) - ? ContextStorageMode.CARRIER : ContextStorageMode.THREAD; - } - - /** - * Build the backing thread-local according to {@link #MODE_PROPERTY}. - * - *
    - *
  • {@code thread} — always plain thread-scoped storage (legacy). Never throws.
    • - *
  • {@code carrier} — carrier scoping is required: throws if - * {@code CarrierThreadLocal} is not accessible, rather than silently reintroducing - * the virtual-thread-pinned storage this class exists to remove.
    • - *
  • {@code auto} (default) — carrier scoping when available; otherwise falls back to - * thread-scoped storage. The fallback is expected on JDK < 21 (silent); on a - * Loom-capable JVM it is logged loudly because it is the pre-fix behavior and is - * unsafe if virtual threads carry context.
    • - *
- * - *

Rationale for not failing hard by default: at profiler init we only know the JVM is - * Loom-capable, not whether virtual threads will actually route context here, and the - * runtime export that carrier scoping needs is granted by the agent — which may land after - * this library. Failing hard by default would break profiler startup for every JDK 21+ - * deployment (including non-Loom apps) until that grant is in place. Callers that can - * guarantee the export (and know they use Loom) should opt into {@code carrier} to get - * fail-fast behavior. - */ - static ThreadLocal create() { - // Resolved once per JavaProfiler instance — this is the tlsContextStorage field - // initializer, not a per-thread path. The per-thread get-or-init in JavaProfiler - // (currentContext) never reads this property, so parsing it here is not a hot path. - // Locale.ROOT: the values are ASCII keywords, so lower-casing must be locale-independent - // (a default-locale toLowerCase() maps "CARRIER" to "carrıer" under tr_TR, breaking the match). - String requested = System.getProperty(MODE_PROPERTY, "auto").trim().toLowerCase(Locale.ROOT); - boolean forceThread = "thread".equals(requested); - boolean requireCarrier = "carrier".equals(requested); - - if (forceThread) { - return new ThreadLocal<>(); - } - - // "auto" or "carrier": prefer carrier scoping when the internal type is reachable. - ThreadLocal carrier = tryCreateCarrierLocal(); - if (carrier != null) { - return carrier; - } - - // Carrier scoping unavailable (JDK < 21, or the jdk.internal.misc export not granted). - if (requireCarrier) { - throw new IllegalStateException("ddprof: -D" + MODE_PROPERTY + "=carrier requires " - + INTERNAL_CTL + ", which is not accessible. On JDK 21+ add " - + "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED (a -javaagent can grant " - + "this via Instrumentation.redefineModule). Falling back to thread-scoped storage " - + "would re-expose the virtual-thread context use-after-free; set -D" - + MODE_PROPERTY + "=thread to explicitly accept legacy thread-scoped storage."); - } - - // auto: degrade so profiling still loads, but be loud on Loom-capable JVMs where the - // fallback can misattribute or corrupt context under virtual threads. - if (Platform.isJavaVersionAtLeast(21)) { - System.out.println("[WARN] ddprof: carrier-scoped OTEL context storage is unavailable on a " - + "JDK 21+ JVM (" + INTERNAL_CTL + " not accessible); falling back to thread-scoped " - + "storage. Under virtual threads this can misattribute context and, if a carrier " - + "thread exits, corrupt native memory. Add " - + "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED, set -D" + MODE_PROPERTY - + "=carrier to fail fast instead, or -D" + MODE_PROPERTY + "=thread to silence this."); - } - return new ThreadLocal<>(); - } - - /** - * Returns a {@code CarrierThreadLocal} (as its {@link ThreadLocal} supertype) or - * {@code null} if the type is absent (JDK < 21) or not accessible (export not granted). - */ - @SuppressWarnings("unchecked") - private static ThreadLocal tryCreateCarrierLocal() { - try { - Class ctl = Class.forName(INTERNAL_CTL); - // Public no-arg constructor of a public type; the newInstance access check - // is what fails (IllegalAccessException) when the package is not exported. - Object instance = ctl.getConstructor().newInstance(); - return (ThreadLocal) instance; - } catch (Throwable t) { - // ClassNotFoundException (JDK < 21), IllegalAccessException (no --add-exports), - // or any other reflective failure — degrade silently to thread scoping. - return null; - } - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java deleted file mode 100644 index cf02c454a6..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler; - -import java.util.Arrays; - -/** - * Per-thread stack of {@link ThreadContext} snapshots for nested scopes. - * - *

Provides bulk save/restore of the full OTEP record + sidecar state via one memcpy per - * transition. Not thread-safe: a single stack instance must be accessed only from its - * owning thread. - * - *

Storage is tiered to keep shallow nesting allocation-free: - *

    - *
  • Depths 0 .. {@value #FAST_DEPTH}-1: one contiguous byte[] allocated eagerly.
    • - *
  • Depths {@value #FAST_DEPTH} and beyond: lazily allocated {@value #CHUNK_DEPTH}-slot - * chunks, each a single byte[]. Chunks are allocated once per depth band and reused.
    • - *
- * - * @deprecated DirectByteBuffer snapshot/restore path; unused by dd-trace-java (re-derives - * context). Removed in phase 3. - */ -@Deprecated -public final class ScopeStack { - private static final int FAST_DEPTH = 6; - private static final int CHUNK_DEPTH = 12; - private static final int SLOT_SIZE = ThreadContext.SNAPSHOT_SIZE; - - private final byte[] fast = new byte[FAST_DEPTH * SLOT_SIZE]; - // chunks[i] covers depths [FAST_DEPTH + i*CHUNK_DEPTH .. FAST_DEPTH + (i+1)*CHUNK_DEPTH). - private byte[][] chunks; - private int depth; - - public void enter(ThreadContext ctx) { - int d = depth; - ctx.snapshot(bufferFor(d), offsetFor(d)); - depth = d + 1; - } - - public void exit(ThreadContext ctx) { - int d = depth - 1; - if (d < 0) { - throw new IllegalStateException("ScopeStack underflow"); - } - ctx.restore(bufferFor(d), offsetFor(d)); - depth = d; - } - - /** Current nesting depth (number of outstanding {@link #enter} calls). */ - public int depth() { - return depth; - } - - private byte[] bufferFor(int d) { - if (d < FAST_DEPTH) { - return fast; - } - // chunkFor is idempotent: if this depth was previously populated (via a matching enter), - // it returns the existing chunk without allocating. - return chunkFor((d - FAST_DEPTH) / CHUNK_DEPTH); - } - - private static int offsetFor(int d) { - int slot = d < FAST_DEPTH ? d : (d - FAST_DEPTH) % CHUNK_DEPTH; - return slot * SLOT_SIZE; - } - - private byte[] chunkFor(int idx) { - byte[][] cs = chunks; - if (cs == null) { - cs = new byte[4][]; - chunks = cs; - } else if (idx >= cs.length) { - int newLen = cs.length; - while (newLen <= idx) { - newLen <<= 1; - } - cs = Arrays.copyOf(cs, newLen); - chunks = cs; - } - byte[] c = cs[idx]; - if (c == null) { - c = new byte[CHUNK_DEPTH * SLOT_SIZE]; - cs[idx] = c; - } - return c; - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java deleted file mode 100644 index 7c8af9bdff..0000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java +++ /dev/null @@ -1,644 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.Objects; - -/** - * Thread-local context for trace/span identification. - * - *

Uses OTEP #4947 TLS record for all context storage. - * Context is written directly to the OTEP record via DirectByteBuffer - * for minimal overhead. Only little-endian platforms are supported. - * - * @deprecated DirectByteBuffer context path; superseded by - * {@link JavaProfiler#setTraceContext} (all-native). Removed in phase 3. - */ -@Deprecated -public final class ThreadContext { - static final int MAX_CUSTOM_SLOTS = 10; - // Max UTF-8 byte length for a custom attribute value. Matches the 1-byte length - // field in the OTEP attrs_data entry header. Enforced up front in setContextAttribute - // so replaceOtepAttribute can assume the input always fits. - private static final int MAX_VALUE_BYTES = 255; - private static final int OTEL_MAX_RECORD_SIZE = 640; - private static final int SIDECAR_SIZE = MAX_CUSTOM_SLOTS * Integer.BYTES + Long.BYTES; // 48 - // Package-private so ScopeStack can size its byte[] scratch. - static final int SNAPSHOT_SIZE = OTEL_MAX_RECORD_SIZE + SIDECAR_SIZE; // 688 - private static final int LRS_OTEP_KEY_INDEX = 0; - // LRS is always a fixed 16-hex-char value in attrs_data (zero-padded u64). - // The entry header is 2 bytes (key_index + length), giving 18 bytes total. - private static final int LRS_FIXED_VALUE_LEN = 16; - private static final int LRS_ENTRY_SIZE = 2 + LRS_FIXED_VALUE_LEN; - private static final byte[] HEX_DIGITS = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; - - private static final BufferWriter BUFFER_WRITER = new BufferWriter(); - - // ---- Per-thread bounded direct-mapped cache for attribute values ---- - // Stores {int encoding, byte[] utf8Bytes} per entry. - // - encoding: Dictionary constant pool ID for DD JFR sidecar - // - utf8Bytes: UTF-8 value for OTEP attrs_data (external profilers) - // - // Instance (not static) so that only the owning thread ever reads or writes - // the cache arrays — no cross-thread races, no memory barriers needed. - // Collision evicts the old entry; a miss triggers one JNI registerConstant0() call. - // The Dictionary (contextValueMap) is never cleared, so encodings remain valid - // for the JVM lifetime. On profiler restart the ThreadContext instance is recreated, - // clearing the cache; the first miss per value pays one JNI call to re-populate. - private static final int CACHE_SIZE = 256; - private static final int CACHE_MASK = CACHE_SIZE - 1; - - // Attribute value cache: String value → {int encoding, byte[] utf8} - // Keyed by value string (not by keyIndex) — same string always maps to - // same encoding regardless of key slot. - private final String[] attrCacheKeys = new String[CACHE_SIZE]; - private final int[] attrCacheEncodings = new int[CACHE_SIZE]; - private final byte[][] attrCacheBytes = new byte[CACHE_SIZE][]; - - // OTEP record field offsets (from packed struct) - private final int validOffset; - private final int traceIdOffset; - private final int spanIdOffset; - private final int attrsDataSizeOffset; - private final int attrsDataOffset; - private final int maxAttrsDataSize; - private final int lrsOffset; // localRootSpanId offset in the unified buffer - // Base offset of the tag-encoding sidecar within the unified buffer. Every tag slot i - // lives at ctxBuffer[tagEncodingsOffset + i * Integer.BYTES]. Equal to OTEL_MAX_RECORD_SIZE. - private static final int TAG_ENCODINGS_OFFSET = OTEL_MAX_RECORD_SIZE; - - // Single buffer spanning [OTEP record | tag_encodings | LRS] — 688 bytes contiguous. - // Used for per-field access AND for bulk snapshot/restore memcpy. Position state is - // thread-confined to snapshot/restore, which reset it before each bulk op. - private final ByteBuffer ctxBuffer; - - /** - * Creates a ThreadContext from the single DirectByteBuffer returned by native initializeContextTLS0. - * - * @param ctxBuffer 688-byte unified buffer spanning record + tag_encodings + LRS - * @param metadata array with absolute offsets [VALID, TRACE_ID, SPAN_ID, - * ATTRS_DATA_SIZE, ATTRS_DATA, LRS] - */ - public ThreadContext(ByteBuffer ctxBuffer, long[] metadata) { - // Uses native order for uint16_t attrs_data_size (read by C as native uint16_t). - // trace_id/span_id are uint8_t[] arrays requiring big-endian — handled via Long.reverseBytes() - // in setContextDirect(). Only little-endian platforms are supported. - this.ctxBuffer = ctxBuffer.order(ByteOrder.nativeOrder()); - this.validOffset = (int) metadata[0]; - this.traceIdOffset = (int) metadata[1]; - this.spanIdOffset = (int) metadata[2]; - this.attrsDataSizeOffset = (int) metadata[3]; - this.attrsDataOffset = (int) metadata[4]; - this.maxAttrsDataSize = OTEL_MAX_RECORD_SIZE - this.attrsDataOffset; - this.lrsOffset = (int) metadata[5]; - if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { - throw new UnsupportedOperationException( - "ByteBuffer context path requires little-endian platform"); - } - // Zero sidecar + record to prevent stale encodings from a previous profiler session. - // The native ProfiledThread survives across sessions, so the buffer may hold - // old tag encodings and the record may hold old attrs_data. - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - this.ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - this.ctxBuffer.putLong(this.lrsOffset, 0); - this.ctxBuffer.put(this.validOffset, (byte) 0); - // Pre-initialize the fixed-size LRS entry at attrs_data[0..LRS_ENTRY_SIZE-1]: - // key_index=0, length=16, value=16 zero hex bytes. - // The entry is always present; updates overwrite only the 16 value bytes. - this.ctxBuffer.put(this.attrsDataOffset, (byte) LRS_OTEP_KEY_INDEX); - this.ctxBuffer.put(this.attrsDataOffset + 1, (byte) LRS_FIXED_VALUE_LEN); - for (int i = 0; i < LRS_FIXED_VALUE_LEN; i++) { - this.ctxBuffer.put(this.attrsDataOffset + 2 + i, (byte) '0'); - } - this.ctxBuffer.putShort(this.attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - } - - /** - * Returns the current span ID. - * Reads directly from the OTEP record buffer (big-endian bytes → native long). - */ - public long getSpanId() { - return Long.reverseBytes(ctxBuffer.getLong(spanIdOffset)); - } - - /** - * Returns the current local root span ID. - * Reads directly from the LRS region of ctxBuffer (native long). - */ - public long getRootSpanId() { - return ctxBuffer.getLong(lrsOffset); - } - - /** - * Sets trace context with 2-arg legacy API. - * Maps rootSpanId to localRootSpanId. Uses spanId as traceIdLow so that - * the OTEL clear check (all-zero) only fires when spanId is actually 0. - * Note: this produces a synthetic trace_id of [0, spanId] in the OTEP record. - * External OTEP readers will see this as a real W3C trace ID. Callers needing - * correct OTEP interop must use the 4-arg {@link #put(long, long, long, long)}. - * - *

Parameter mapping to the 4-arg API: {@code spanId} → {@code spanId}, - * {@code rootSpanId} → {@code localRootSpanId} (first argument). - * - * @deprecated Use {@link #put(long, long, long, long)} instead. - */ - @Deprecated - public long put(long spanId, long rootSpanId) { - put(rootSpanId, spanId, 0, spanId); - return 0; // Return type kept for ABI compatibility; value carries no meaning. - } - - /** - * Sets trace context with full 128-bit W3C trace ID and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId The span ID - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - */ - public void put(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - setContextDirect(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Clears a custom attribute: zeros the sidecar encoding and removes it from OTEP attrs_data. - */ - public void clearContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return; - } - int otepKeyIndex = keyIndex + 1; - detach(); - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - removeOtepAttribute(otepKeyIndex); - attach(); - } - - public void copyCustoms(int[] value) { - int len = Math.min(value.length, MAX_CUSTOM_SLOTS); - for (int i = 0; i < len; i++) { - value[i] = ctxBuffer.getInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES); - } - } - - /** - * Captures the full record + sidecar state into {@code scratch[offset..offset+SNAPSHOT_SIZE)}. - * Pair with {@link #restore} for nested-scope propagation. - * - *

The detach/memcpy/re-publish pair hides the bulk read from any signal handler going - * through {@code ContextApi::get} — while {@code valid=0}, sidecar reads are gated off. The - * pre-snapshot {@code valid} state is preserved in {@code scratch[offset + validOffset]} so - * {@link #restore} can replay it. If the record was already invalid (e.g. the all-zero clear - * path in {@link #setContextDirect} leaves {@code valid=0} with a stale {@code attrs_data_size} - * / {@code attrs_data}), the live buffer is left invalid after snapshot — re-publishing would - * expose a cleared-but-stale record. - */ - public void snapshot(byte[] scratch, int offset) { - byte priorValid = ctxBuffer.get(validOffset); - detach(); - // Cast to Buffer: ByteBuffer.position(int) only returns ByteBuffer since JDK 9 (covariant - // return). This source is compiled for Java 8 runtimes where the method lives on Buffer. - ((Buffer) ctxBuffer).position(0); - ctxBuffer.get(scratch, offset, SNAPSHOT_SIZE); - // Overwrite the valid byte in scratch (memcpy captured the post-detach 0) with the - // pre-snapshot value. restore() consults this to decide whether to re-attach. - scratch[offset + validOffset] = priorValid; - if (priorValid != 0) { - attach(); - } - } - - /** - * Restores a previously captured state. The detach/memcpy/conditional-attach pair hides the - * memcpy from readers going through {@link #ctxBuffer}'s valid flag ({@code ContextApi::get} - * in native code), which is the sole gate for sidecar reads (see {@code thread.h}). - * - *

The valid byte inside scratch is cleared to 0 for the duration of the memcpy so that - * even if the captured state had {@code valid=1}, the live buffer cannot transiently observe - * {@code valid=1} alongside partially-written fields. The captured value is restored into - * scratch after the memcpy so subsequent snapshot/restore cycles keep working, and - * {@link #attach} re-publishes only when the saved state was itself valid — matching the - * semantics of {@link #snapshot}. - */ - public void restore(byte[] scratch, int offset) { - int validIdx = offset + validOffset; - byte wasValid = scratch[validIdx]; - scratch[validIdx] = 0; - detach(); - ((Buffer) ctxBuffer).position(0); - ctxBuffer.put(scratch, offset, SNAPSHOT_SIZE); - if (wasValid != 0) { - scratch[validIdx] = wasValid; - attach(); - } - } - - /** - * Sets a custom attribute on the current thread's context by string value. - * Uses a per-thread encoding cache to avoid JNI for repeated values - * (the common case). On cache miss, a single JNI call registers the value - * in the native Dictionary; subsequent calls with the same value are - * zero-JNI ByteBuffer writes. - * - *

High-cardinality values are not supported. Each unique value - * permanently occupies one slot in the native Dictionary, which is bounded - * at 65536 entries across all threads for the JVM lifetime. Once exhausted, - * this method returns {@code false} and clears the attribute. Use only - * low-cardinality values (e.g. endpoint names, DB system names). UUIDs, - * request IDs, and other per-request-unique strings will exhaust the - * Dictionary and cause attributes to be silently dropped. - * - *

Value size limit. The UTF-8 encoding of {@code value} must fit in - * {@value #MAX_VALUE_BYTES} bytes (the OTEP attrs_data entry length field is one byte). - * Oversized values are rejected up front — they never reach the Dictionary or attrs_data. - * - * @param keyIndex Index into the registered attribute key map (0-based) - * @param value The string value for this attribute - * @return true if the attribute was set successfully, false if the value is too long, - * the Dictionary is full, attrs_data overflows, or keyIndex is out of range - */ - public boolean setContextAttribute(int keyIndex, String value) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS || value == null) { - return false; - } - return setContextAttributeDirect(keyIndex, value); - } - - /** - * Writes both the sidecar encoding (DD signal handler) and OTEP attrs_data - * UTF-8 value (external profilers) via ByteBuffer. - */ - private boolean setContextAttributeDirect(int keyIndex, String value) { - - // Resolve encoding + UTF-8 bytes from per-thread cache - int slot = value.hashCode() & CACHE_MASK; - int encoding; - byte[] utf8; - if (value.equals(attrCacheKeys[slot])) { - // Cache hit — the value was previously validated and cached; no re-check needed. - encoding = attrCacheEncodings[slot]; - utf8 = attrCacheBytes[slot]; - } else { - // Cache miss: encode UTF-8 and validate size BEFORE touching the Dictionary. - // Rejecting here avoids an orphan Dictionary entry (the native Dictionary is - // write-only for the JVM lifetime and cannot be undone). - utf8 = value.getBytes(StandardCharsets.UTF_8); - if (utf8.length > MAX_VALUE_BYTES) { - return false; - } - encoding = registerConstant0(value); - if (encoding < 0) { - // Dictionary full: clear sidecar AND remove the OTEP attrs_data entry - // so both views stay consistent (both report no value for this key). - clearContextAttribute(keyIndex); - return false; - } - attrCacheEncodings[slot] = encoding; - attrCacheBytes[slot] = utf8; - attrCacheKeys[slot] = value; - } - - // Write both sidecar and OTEP attrs_data inside the detach/attach window - // so a signal handler never sees a new sidecar encoding alongside old attrs_data. - detach(); - boolean written = writeSlot(keyIndex, encoding, utf8); - attach(); - return written; - } - - /** - * Writes one slot's sidecar encoding and OTEP attrs_data value. The caller must already hold - * the detach/attach window. On attrs_data overflow the old entry was compacted out and the new - * one couldn't fit, so the sidecar is zeroed to keep both views agreeing there is no value. - * - * @return true if the value was written; false on attrs_data overflow (sidecar left zeroed) - */ - private boolean writeSlot(int keyIndex, int encoding, byte[] utf8) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, encoding); - if (!replaceOtepAttribute(keyIndex + 1, utf8)) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - return false; - } - return true; - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes in a - * single detach/attach window, without Dictionary lookups or per-thread cache access. - * - *

Both arrays are indexed by key slot, matching the layout produced by - * {@link #copyCustoms(int[])}. For each slot {@code i} with {@code constantIds[i] > 0}, the - * sidecar encoding (DD signal handler) and the OTEP attrs_data value (external profilers) are - * written together; slots with {@code constantIds[i] <= 0} are left untouched. Performing all - * writes inside one detach/attach window means a signal handler never observes a partially - * re-applied set. - * - *

Intended for the reapply-app-context hot path: the caller already holds the constant IDs - * (from {@link #copyCustoms(int[])}) and the UTF-8 bytes (from the original Strings), so this - * does no String allocation, hashing, or cache lookup. The record is re-published only if it - * was valid before the call, so a cleared (span-less) record is not resurrected. - * - *

Caller contract. {@code constantIds[i]} must be an ID previously returned for the - * same value via {@link #copyCustoms(int[])} within the current profiler session, and - * {@code utf8[i]} must be that value's UTF-8 bytes. The (id, bytes) pairing is not verifiable - * here; a mismatch silently diverges the sidecar and attrs_data views. - * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most - * {@value #MAX_VALUE_BYTES} bytes for every slot whose constantId {@code > 0} - * @return true if every slot with {@code constantId > 0} was written successfully; false if - * the record was not valid before the call (nothing is published), or if any slot - * overflowed {@code attrs_data} (that slot's sidecar is zeroed). Note: a {@code false} - * return due to {@code attrs_data} overflow does not mean the record is - * unmodified — slots processed before the overflowed one are durably written. - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} (where {@code constantIds[i] > 0}) is null - * @throws IllegalArgumentException if the arrays have different lengths, - * {@code constantIds.length > MAX_CUSTOM_SLOTS}, or any - * active {@code utf8[i].length > MAX_VALUE_BYTES} - */ - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - Objects.requireNonNull(constantIds, "constantIds"); - Objects.requireNonNull(utf8, "utf8"); - if (constantIds.length != utf8.length) { - throw new IllegalArgumentException("constantIds and utf8 must have the same length"); - } - if (constantIds.length > MAX_CUSTOM_SLOTS) { - throw new IllegalArgumentException("constantIds.length exceeds MAX_CUSTOM_SLOTS"); - } - int len = constantIds.length; - // Validate active slots before touching the buffer so a bad input never leaves - // the record detached (valid=0) after an exception unwinds past attach(). - for (int i = 0; i < len; i++) { - if (constantIds[i] > 0) { - if (utf8[i] == null) { - throw new NullPointerException("utf8[" + i + "]"); - } - if (utf8[i].length > MAX_VALUE_BYTES) { - throw new IllegalArgumentException("utf8[" + i + "].length exceeds MAX_VALUE_BYTES"); - } - } - } - // Never resurrect a cleared (span-less) record: valid=0 means no reader can observe - // what we write, and re-publishing would expose a record with no trace/span context. - if (ctxBuffer.get(validOffset) == 0) { - return false; - } - detach(); - boolean allWritten = true; - for (int i = 0; i < len; i++) { - int constantId = constantIds[i]; - if (constantId <= 0) { - continue; - } - if (!writeSlot(i, constantId, utf8[i])) { - allWritten = false; - } - } - attach(); - return allWritten; - } - - /** - * Write context directly to the OTEP record via ByteBuffer. - * trace_id and span_id are OTEP big-endian byte arrays — Long.reverseBytes() - * converts from native LE to big-endian. - */ - private void setContextDirect(long localRootSpanId, long spanId, long trHi, long trLo) { - detach(); - - if (trHi == 0 && trLo == 0 && spanId == 0) { - clearContextDirect(); - return; - } - - // Write trace_id (big-endian) + span_id (big-endian) - ctxBuffer.putLong(traceIdOffset, Long.reverseBytes(trHi)); - ctxBuffer.putLong(traceIdOffset + 8, Long.reverseBytes(trLo)); - ctxBuffer.putLong(spanIdOffset, Long.reverseBytes(spanId)); - - // Reset custom attribute state so the previous span's values don't leak - // into this span. Callers set attributes again via setContextAttribute(). - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - // offset into ctxBuffer for tag-encoding slot i - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - // Reset attrs_data_size to contain only the fixed LRS entry, discarding - // any custom attribute entries written during the previous span. - ctxBuffer.putShort(attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - - // Update LRS sidecar and OTEP attrs_data inside the detach/attach window so a - // signal handler never sees the new LRS with old trace/span IDs. - ctxBuffer.putLong(lrsOffset, localRootSpanId); - writeLrsHex(localRootSpanId); - - attach(); - } - - // ---- LRS helpers ---- - - /** - * Zeros trace/span IDs, sidecar encodings, and LRS. Called between detach() and the - * return in the all-zero path; valid stays 0 (no attach) so no reader can see attrs_data. - * attrs_data_size is not reset here; the next non-zero setContext call will reset it - * before attach(). This is safe because valid remains 0 after clear, so no reader will - * observe the stale attrs_data_size. - * - *

External OTEP readers see valid=0 and skip the record — they cannot distinguish - * "cleared" from "being mutated". This is intentional: without also resetting - * attrs_data_size here, publishing valid=1 with a stale attrs_data_size would expose - * a partially-valid record. The cleared state is effectively invisible to external - * readers until the next non-zero setContext call publishes it. - */ - private void clearContextDirect() { - ctxBuffer.putLong(traceIdOffset, 0); - ctxBuffer.putLong(traceIdOffset + 8, 0); - ctxBuffer.putLong(spanIdOffset, 0); - writeLrsHex(0); - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - ctxBuffer.putLong(lrsOffset, 0); - } - - /** - * Overwrite the 16 hex value bytes of the fixed LRS entry in-place. - * The entry header (key_index=0, length=16) is pre-initialized and never touched. - * Called between detach() and attach(); no allocation. - */ - private void writeLrsHex(long val) { - int base = attrsDataOffset + 2; // skip key_index byte + length byte - for (int i = 15; i >= 0; i--) { - ctxBuffer.put(base + i, HEX_DIGITS[(int)(val & 0xF)]); - val >>>= 4; - } - } - - // ---- OTEP record helpers (called between detach/attach) ---- - - /** - * Invalidates the record (sets valid=0). Typically followed by attach(), but the - * clear path intentionally leaves the record invalid without calling attach(). - */ - private void detach() { - ctxBuffer.put(validOffset, (byte) 0); - BUFFER_WRITER.storeFence(); - } - - /** Validate record. */ - private void attach() { - // storeFence ensures all record writes are visible before valid=1. - // The TLS pointer (otel_thread_ctx_v1) is permanent and never - // written here; external profilers rely solely on the valid flag. - BUFFER_WRITER.storeFence(); - // Plain put is sufficient: signal handlers run on the same hardware thread, - // so they observe stores in program order — no volatile needed for same-thread - // visibility. The preceding storeFence() provides the release barrier. - ctxBuffer.put(validOffset, (byte) 1); - } - - /** - * Replace or insert an attribute in attrs_data. Record must be detached. - * Writes the pre-encoded UTF-8 bytes into the record. - * - *

Caller contract: {@code utf8.length <= MAX_VALUE_BYTES}, enforced at the public - * entry point in {@link #setContextAttributeDirect}. - */ - private boolean replaceOtepAttribute(int otepKeyIndex, byte[] utf8) { - int currentSize = compactOtepAttribute(otepKeyIndex); - int valueLen = utf8.length; - int entrySize = 2 + valueLen; - if (currentSize + entrySize <= maxAttrsDataSize) { - int base = attrsDataOffset + currentSize; - ctxBuffer.put(base, (byte) otepKeyIndex); - ctxBuffer.put(base + 1, (byte) valueLen); - for (int i = 0; i < valueLen; i++) { - ctxBuffer.put(base + 2 + i, utf8[i]); - } - currentSize += entrySize; - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return true; - } - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return false; - } - - /** Remove an attribute from attrs_data by compacting. Record must be detached. */ - private void removeOtepAttribute(int otepKeyIndex) { - int currentSize = compactOtepAttribute(otepKeyIndex); - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - } - - /** - * Scan attrs_data and compact out the entry with the given key_index. - * Returns the new attrs_data size after compaction. - * - *

{@code otepKeyIndex} is always {@code keyIndex + 1} for user attributes, - * so it is never 0. Index 0 is reserved for the fixed LRS entry. - */ - private int compactOtepAttribute(int otepKeyIndex) { - int currentSize = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int readPos = 0; - int writePos = 0; - boolean found = false; - while (readPos + 2 <= currentSize) { - int k = ctxBuffer.get(attrsDataOffset + readPos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + readPos + 1) & 0xFF; - if (readPos + 2 + len > currentSize) { currentSize = writePos; break; } - if (k == otepKeyIndex) { - found = true; - readPos += 2 + len; - } else { - if (found && writePos < readPos) { - for (int i = 0; i < 2 + len; i++) { - ctxBuffer.put(attrsDataOffset + writePos + i, - ctxBuffer.get(attrsDataOffset + readPos + i)); - } - } - writePos += 2 + len; - readPos += 2 + len; - } - } - return found ? writePos : currentSize; - } - - /** - * Reads a custom attribute value by key index by scanning {@code attrs_data}. - * - *

Test-only. The only caller is {@code TagContextTest}, which uses it via - * {@link JavaProfiler#getThreadContext()} to verify that writes to the OTEP record are - * observable after set / clear / span-reset cycles. No production path — neither the DD - * signal handler nor the OTEL eBPF reader — ever calls this method: the DD handler reads - * sidecar encoding IDs and the OTEL reader parses {@code attrs_data} directly from native - * memory. The per-call {@code byte[]} / {@code String} allocation is therefore acceptable; - * do not introduce a readback cache unless a real production consumer appears. - * - * @param keyIndex 0-based user key index (same as passed to setContextAttribute) - * @return the attribute value string, or null if not set - */ - public String readContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return null; - } - // valid=0 → record was detached or never published. No attrs_data to trust. - if (ctxBuffer.get(validOffset) == 0) { - return null; - } - int otepKeyIndex = keyIndex + 1; - int size = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int pos = 0; - while (pos + 2 <= size) { - int k = ctxBuffer.get(attrsDataOffset + pos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + pos + 1) & 0xFF; - if (pos + 2 + len > size) { - break; - } - if (k == otepKeyIndex) { - byte[] bytes = new byte[len]; - for (int i = 0; i < len; i++) { - bytes[i] = ctxBuffer.get(attrsDataOffset + pos + 2 + i); - } - return new String(bytes, StandardCharsets.UTF_8); - } - pos += 2 + len; - } - return null; - } - - /** - * Reads the trace ID from the OTEP record as a 32-char lowercase hex string. - * The trace ID is stored big-endian; this method returns it as-is. - * Intended for tests only. - */ - public String readTraceId() { - StringBuilder sb = new StringBuilder(32); - for (int i = 0; i < 16; i++) { - int b = ctxBuffer.get(traceIdOffset + i) & 0xFF; - sb.append((char) HEX_DIGITS[b >> 4]); - sb.append((char) HEX_DIGITS[b & 0xF]); - } - return sb.toString(); - } - - // Package-private (was private) so the all-native path's ContextValueCache can reuse the same - // process-global Dictionary registration. Relocates when ThreadContext is removed (phase 3). - static native int registerConstant0(String value); -} diff --git a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java b/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java deleted file mode 100644 index 9fcf77c4e5..0000000000 --- a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public final class BufferWriter9 implements BufferWriter.Impl { - private static final VarHandle LONG_VIEW_VH; - private static final VarHandle INT_VIEW_VH; - - static { - try { - // Create a view into ByteBuffer as if it were a long array - // The VarHandle coordinates are (ByteBuffer, int index) where index is in bytes - LONG_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - long[].class, ByteOrder.nativeOrder()); - INT_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - int[].class, ByteOrder.nativeOrder()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedLong) - LONG_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedInt) - INT_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void storeFence() { - VarHandle.storeStoreFence(); - } -} diff --git a/ddprof-stresstest/README.md b/ddprof-stresstest/README.md index f22caff0eb..c8b9db566f 100644 --- a/ddprof-stresstest/README.md +++ b/ddprof-stresstest/README.md @@ -92,18 +92,6 @@ Measure end-to-end profiling engine performance including signal handlers, stack **Documentation**: See `doc/architecture/CallTraceStorage.md` for detailed CallTraceStorage architecture, benchmark results analysis, and optimization recommendations. -### ThreadContext Benchmarks - -Compare performance of JNI-based native vs DirectByteBuffer-based Java implementations for thread context storage. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ThreadContextBenchmark -``` - -Tests various thread counts to measure both single-threaded overhead and multi-threaded contention. - ### ThreadFilter Benchmarks Measure thread filtering performance and overhead. @@ -264,7 +252,7 @@ ddprof-stresstest/ │ └── scenarios/ │ ├── throughput/ # Raw performance benchmarks │ │ ├── ProfilerThroughput* # End-to-end profiling engine suite -│ │ ├── ThreadContext* # ThreadContext benchmarks +│ │ ├── ContextCombinedBenchmark # All-native context write API perf guard │ │ └── ThreadFilter* # ThreadFilter benchmarks │ └── counters/ # Feature-specific benchmarks │ ├── TracedParallelWork # Distributed tracing overhead diff --git a/ddprof-stresstest/build.gradle.kts b/ddprof-stresstest/build.gradle.kts index 8119efb614..550baee43b 100644 --- a/ddprof-stresstest/build.gradle.kts +++ b/ddprof-stresstest/build.gradle.kts @@ -83,7 +83,7 @@ dependencies { // provides the (relocated) runtime classes and intercepts @Trace. "chaosCompileOnly"(libs.dd.trace.api) // ddprof-lib public API: compile-only; the patched dd-java-agent provides the - // classes at runtime for antagonists that call JavaProfiler/ThreadContext directly. + // classes at runtime for antagonists that call JavaProfiler directly. "chaosCompileOnly"(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug"))) } diff --git a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh b/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh deleted file mode 100755 index 86608bf7b6..0000000000 --- a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Run ThreadContext performance benchmark -# Compares JNI-based vs DirectByteBuffer-based implementations - -set -e - -JAVA_TEST_HOME="${JAVA_TEST_HOME:-$JAVA_HOME}" - -echo "Building JMH benchmark JAR..." -./gradlew :ddprof-stresstest:jmhJar - -echo "" -echo "Running ThreadContext benchmark..." -echo "This will take several minutes as it runs multiple configurations with 1, 2, 4, 8, and 16 threads" -echo "" - -# Run the benchmark with specific pattern to match our ThreadContext tests -"${JAVA_TEST_HOME}/bin/java" -jar ddprof-stresstest/build/libs/stresstests.jar \ - "ThreadContextBenchmark.*" \ - -rf json \ - -rff build/threadcontext-benchmark-results.json - -echo "" -echo "Benchmark complete!" -echo "Results saved to: build/threadcontext-benchmark-results.json" - diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md index 2c72a4eba0..74e5f19d4b 100644 --- a/ddprof-stresstest/src/chaos/README.md +++ b/ddprof-stresstest/src/chaos/README.md @@ -14,8 +14,8 @@ the runner script. | `vthread-churn` | virtual thread mount/unmount, carrier-thread context, `ProfiledThread` | | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | | `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | -| `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | -| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free | +| `trace-context` | `setTraceContext`/`clearTraceContext` racing signals, span ID propagation | +| `reapply-context-value` | concurrent per-slot `setContextValue`/`clearContextValue` churn interleaved with span activation, racing the all-native record's write/read window | ## Deferred diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 65b29c7c6e..7e846c02d1 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -74,8 +74,6 @@ private static Antagonist create(String name) { return new AllocStormAntagonist(); case "vthread-churn": return new VirtualThreadChurnAntagonist(); - case "vthread-context-cascade": - return new VirtualThreadContextCascadeAntagonist(); case "classloader-churn": return new ClassLoaderChurnAntagonist(); case "trace-context": @@ -94,8 +92,8 @@ private static Antagonist create(String name) { return new WeakRefWaveAntagonist(); case "dump-storm": return new DumpStormAntagonist(); - case "reapply-context": - return new ReapplyContextAntagonist(); + case "reapply-context-value": + return new ReapplyContextValueAntagonist(); // Deferred: dlopen-churn (needs per-arch dummy .so built in CI prep). default: throw new IllegalArgumentException("unknown antagonist: " + name); diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java deleted file mode 100644 index e853fa3c8b..0000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import com.datadoghq.profiler.ContextSetter; -import com.datadoghq.profiler.JavaProfiler; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Arrays; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -/** - * Drives the reapply-by-id-and-bytes hot path continuously from multiple threads while the - * profiler's wall-clock signal fires, racing the {@code detach}/{@code attach} window. - * - *

Each worker thread loops: span-activate (wiping slots) → reapply snapshot. This mirrors - * dd-trace-java's {@code reapplyAppContext} pattern and exercises the single-window invariant - * (no partial publish visible to a signal handler) under high thread count and signal pressure. - * - *

The only expected failure signal is a JVM crash. An unexpected {@code false} return from - * reapply (which should never happen since the record is always valid right after - * {@code setContext}) throws {@link IllegalStateException} to surface the bug immediately. - */ -public final class ReapplyContextAntagonist implements Antagonist { - - private static final String[] ATTR_NAMES = { - "http.route", "http.method", "http.status", "db.operation", "rpc.service" - }; - - private static final String[] ROUTES = { - "GET /api/users", - "POST /api/orders", - "GET /api/health", - "PUT /api/users/{id}", - "DELETE /api/sessions" - }; - - private final int workerCount; - private final ExecutorService pool; - private volatile boolean running; - - public ReapplyContextAntagonist() { - this(8); - } - - public ReapplyContextAntagonist(int workerCount) { - this.workerCount = workerCount; - this.pool = - Executors.newFixedThreadPool( - workerCount, - r -> { - Thread t = new Thread(r, "chaos-reapply-context"); - t.setDaemon(true); - return t; - }); - } - - @Override - public String name() { - return "reapply-context"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < workerCount; i++) { - pool.submit(this::workerLoop); - } - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - pool.shutdown(); - try { - pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void workerLoop() { - JavaProfiler profiler; - try { - profiler = JavaProfiler.getInstance(); - } catch (Exception e) { - System.err.println("[chaos] reapply-context: failed to get profiler: " + e); - return; - } - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); - - // Prime the per-thread encoding cache and capture a stable snapshot. - long spanId = Thread.currentThread().getId() + 1; - long localRootSpanId = spanId * 31L; - long traceIdLow = spanId * 6364136223846793005L + 1442695040888963407L; - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - contextSetter.setContextValue(i, ROUTES[i]); - } - int[] constantIds = contextSetter.snapshotTags(); - byte[][] utf8 = new byte[ROUTES.length][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - - while (running) { - // Span activation wipes all custom slots. - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - // Reapply restores them — this is the hot path under test. - if (!contextSetter.setContextValuesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException("reapply failed unexpectedly — record should be valid after setContext"); - } - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java new file mode 100644 index 0000000000..a8a84320ee --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextValueAntagonist.java @@ -0,0 +1,126 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import com.datadoghq.profiler.JavaProfiler; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +/** + * Drives repeated per-slot {@code setContextValue}/{@code clearContextValue} calls continuously + * from multiple threads, interleaved with {@code setTraceContext} span activations, while the + * profiler's wall-clock signal fires — racing the all-native record's write/read window. + * + *

Each worker thread loops: activate a span (resetting custom slots), churn a handful of + * attribute slots with set/clear calls, deactivate. Unlike the legacy DirectByteBuffer conduit + * this antagonist's predecessor exercised, there is no cached per-thread buffer to go stale — the + * only thing under test here is that concurrent single-slot native writes never produce a partial + * or torn record visible to a signal handler. + */ +public final class ReapplyContextValueAntagonist implements Antagonist { + + private static final String[] ROUTES = { + "GET /api/users", + "POST /api/orders", + "GET /api/health", + "PUT /api/users/{id}", + "DELETE /api/sessions" + }; + + private static final int SLOT_COUNT = 5; + + private final int workerCount; + private final ExecutorService pool; + private volatile boolean running; + + public ReapplyContextValueAntagonist() { + this(8); + } + + public ReapplyContextValueAntagonist(int workerCount) { + this.workerCount = workerCount; + this.pool = + Executors.newFixedThreadPool( + workerCount, + r -> { + Thread t = new Thread(r, "chaos-reapply-context-value"); + t.setDaemon(true); + return t; + }); + } + + @Override + public String name() { + return "reapply-context-value"; + } + + @Override + public void start() { + running = true; + for (int i = 0; i < workerCount; i++) { + pool.execute(this::workerLoop); + } + } + + @Override + public void stopGracefully(Duration timeout) { + running = false; + pool.shutdown(); + try { + pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void workerLoop() { + JavaProfiler profiler; + try { + profiler = JavaProfiler.getInstance(); + } catch (Exception e) { + System.err.println("[chaos] reapply-context-value: failed to get profiler: " + e); + return; + } + + long spanId = Thread.currentThread().getId() + 1; + long localRootSpanId = spanId * 31L; + long traceIdLow = spanId * 6364136223846793005L + 1442695040888963407L; + + while (running) { + // Span activation resets all custom slots. + profiler.setTraceContext(localRootSpanId, spanId, 0, traceIdLow, -1, null, -1, null); + for (int i = 0; i < SLOT_COUNT; i++) { + checkSetContextValue(profiler.setContextValue(i, ROUTES[i % ROUTES.length]), i); + } + for (int i = 0; i < SLOT_COUNT; i += 2) { + profiler.clearContextValue(i); + } + for (int i = 0; i < SLOT_COUNT; i += 2) { + int route = (i + ThreadLocalRandom.current().nextInt(ROUTES.length)) % ROUTES.length; + checkSetContextValue(profiler.setContextValue(i, ROUTES[route]), i); + } + profiler.clearTraceContext(); + } + } + + // Per Main's class javadoc, the harness's only failure signal is a non-zero process exit; + // an exception here would otherwise die silently on this daemon worker thread (and be + // swallowed entirely if the task were submitted via ExecutorService.submit and its Future + // never inspected). Halt immediately so an unexpected failure actually fails the CI job. + private static void checkSetContextValue(boolean succeeded, int slot) { + if (!succeeded) { + System.err.println("[chaos] reapply-context-value: setContextValue failed for slot " + slot); + Runtime.getRuntime().halt(1); + } + } +} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java index 8abee751b7..570c1275b0 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java @@ -21,7 +21,7 @@ /** * Drives nested {@link Trace}-annotated invocations from a pool of worker * threads. Under a {@code dd-java-agent}-instrumented JVM the tracer drives - * {@code JavaProfiler.setContext}/{@code clearContext} on every span + * {@code JavaProfiler.setTraceContext}/{@code clearTraceContext} on every span * activation, so a high enter/exit rate stresses that path against signal * delivery and stack walking. * diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java deleted file mode 100644 index ecc3c951e6..0000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ /dev/null @@ -1,560 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import datadog.trace.api.Trace; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.concurrent.Executor; -import java.util.concurrent.Semaphore; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and - * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. - * - *

Unlike a hand-rolled context simulation, every context write here is driven by the real - * Datadog tracer: {@link #runCascadeNode}/{@link #nestedOp}/{@link #staleRacerBefore}/{@link - * #staleRacerAfter} are {@link Trace}-annotated methods. Under a {@code dd-java-agent}- - * instrumented JVM the tracer intercepts each one, activating a real span on entry ({@code - * JavaProfiler.setContext}) and deactivating it on exit ({@code clearContext}) — exactly what a - * genuinely-instrumented application does. {@code dd-trace-api} is a {@code compileOnly} - * dependency (see {@code TraceContextAntagonist}): at runtime the patched {@code dd-java-agent} - * provides the (relocated) classes and intercepts the annotation, so this antagonist never touches - * {@code com.datadoghq.profiler.*} directly. - * - *

Each cascade node ({@link #runCascadeNode}) runs a handful of nested sub-operations, each - * activating a new child span and sleeping briefly ({@link #forceUnmount}) to force the virtual - * thread off its current carrier before the span's exit (and the next nested entry) write context - * again — the high context-change-rate dimension. It then fans out to child virtual threads while - * its own span is still active, handing the tracer's own async/virtual-thread instrumentation the - * job of propagating that span as parent context onto the next hop (the chain continues on a new - * thread/carrier). On exit, the span's close restores whatever was logically active underneath it. - * - *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived - * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) - * across a rotating set of monitors, forcing the JDK's virtual-thread scheduler to spin up extra - * compensating carrier platform threads while the pins are held. - * - *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} - * fallback (the default on JDK 21+ without {@code --add-exports - * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit - * for a virtual thread's OTEL context (which backs both the span id fields written by {@code - * setContext} and any custom attributes) is cached in a plain {@code ThreadLocal} the first time - * the thread writes context, bound to whichever carrier happened to be mounted at that moment; the - * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} - * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment - * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races - * this cheaply: a nested span's entry writes, {@link #forceUnmount} forces a short real unmount, - * then the span's exit (or the next nested entry) writes again — if the virtual thread was - * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. - * - *

Catching genuine memory reuse (not just misattribution) needs the original carrier - * to actually be torn down first. {@link #driverLoop} runs a duty cycle for the cascade/pin-churn - * side only (a short active burst followed by a quiet phase with no new work, long enough for the - * JDK's default {@code ForkJoinPool} scheduler — 30s keep-alive, see {@code - * java.lang.VirtualThread#createDefaultScheduler()} — to actually reap idle carriers). Stale- - * buffer racers ({@link #runStaleBufferRacer}) don't wait on that: every racer runs on a virtual - * thread bound to a purpose-built {@link #fastCarrierScheduler} — a {@link ThreadPoolExecutor} - * with a 200ms keep-alive — via the package-private {@code VirtualThread(Executor, String, int, - * Runnable)} constructor (JDK 21+; see {@code java.lang.VirtualThread}, requires {@code - * --add-opens java.base/java.lang=ALL-UNNAMED}). {@link #staleRacerLoop} keeps up to {@link - * #MAX_STALE_RACERS} of these racers in flight continuously, independent of the cascade/pin duty - * cycle above — sustained concurrent hammering rather than one batch per cycle, since more - * concurrent carrier churn means more chances for a freed {@code ProfiledThread} slot to actually - * get reused before a stale racer touches it. Each racer writes context once ({@link - * #staleRacerBefore}), sleeps a randomized interval straddling the scheduler's keep-alive ({@link - * #FAST_STALE_RACER_SLEEP_MIN_MILLIS}–{@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}), then writes - * again ({@link #staleRacerAfter}) — varying the wait means a run samples both "carrier likely - * still tearing down" and "carrier long gone, memory possibly already reused" timings, instead of - * only ever probing one fixed instant. - * - *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops the whole - * antagonist on older runtimes where it's absent. But on a JDK where it is present, the - * {@code VirtualThread} constructor above is required, not optional: it's this antagonist's only - * way to reach the stale-carrier UAF without a slow duty-cycle wait, so {@link #start} fails fast - * with an {@link IllegalStateException} (crashing the chaos harness process, which the CI runner - * already treats as a failure) rather than silently losing that coverage if a missing {@code - * --add-opens} flag or a JDK build that changed the constructor's signature makes it - * unreachable. Live virtual thread count is bounded by semaphore permit pools so cascades, - * pinning churn, and stale-buffer racers cannot runaway the JVM. - */ -public final class VirtualThreadContextCascadeAntagonist implements Antagonist { - - private static final Method OF_VIRTUAL = resolveOfVirtual(); - private static final Method BUILDER_START = resolveBuilderStart(); - - private static final int FAN_OUT = 3; - private static final int MAX_DEPTH = 4; - private static final int NESTED_OPS_PER_HOP = 3; - private static final int MAX_LIVE_VTHREADS = 512; - private static final int ROOT_BATCH = 16; - - // Carrier-pinning churn: rotating monitors held across a blocking sleep to force the - // scheduler to spin up (and later reap) compensating carrier platform threads. - private static final int PIN_LOCK_COUNT = 32; - private static final int PIN_BATCH = 24; - private static final int MAX_PIN_VTHREADS = 128; - private static final Object[] PIN_LOCKS = new Object[PIN_LOCK_COUNT]; - - static { - for (int i = 0; i < PIN_LOCK_COUNT; i++) { - PIN_LOCKS[i] = new Object(); - } - } - - // Duty cycle driving genuine carrier teardown: an active burst of cascade/pin-churn work, - // then a quiet phase long enough to clear the JDK's default 30s carrier keep-alive so idle - // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. - private static final long ACTIVE_PHASE_MILLIS = 8_000L; - private static final long QUIET_PHASE_MILLIS = 40_000L; - - // Stale-buffer racers run continuously on their own driver, decoupled from the cascade/pin - // duty cycle above — that cycle only exists to let the JDK default scheduler's 30s keep-alive - // reap cascade/pin carriers, which is irrelevant to racers bound to fastCarrierScheduler. - private static final int STALE_RACER_BATCH = 64; - private static final int MAX_STALE_RACERS = 1024; - - // A short keep-alive forces genuine carrier-thread exit (and thus a real ProfiledThread - // free()) in well under a second instead of 30s. The racer's sleep is randomized across a - // range straddling this keep-alive so runs cover both "carrier likely still exiting" and - // "carrier long gone, memory possibly already reused" timings instead of one fixed instant. - private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; - private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; - private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; - - // Caps the fast-carrier pool's platform-thread growth. MAX_STALE_RACERS is - // already the hard ceiling on concurrent virtual threads that could each - // need a carrier, so this doesn't reduce headroom — it just removes the - // unbounded-growth characteristic of Integer.MAX_VALUE. - private static final int MAX_FAST_CARRIER_THREADS = MAX_STALE_RACERS; - - private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); - - private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); - private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); - private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); - private final Executor fastCarrierScheduler = VTHREAD_CTOR != null ? newFastCarrierScheduler() : null; - private volatile boolean running; - private volatile boolean activePhase = true; - private Thread driver; - private Thread pinningDriver; - private Thread staleRacerDriver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "vthread-context-cascade"; - } - - @Override - public void start() { - if (OF_VIRTUAL != null && BUILDER_START != null && VTHREAD_CTOR == null) { - // We're on a VT-capable JDK, so the fast-carrier constructor should be reachable — - // its absence (missing --add-opens, or a JDK build that changed the constructor) - // would otherwise silently drop stale-racer coverage instead of failing the run. - throw new IllegalStateException( - "vthread-context-cascade: java.lang.VirtualThread(Executor, String, int, " - + "Runnable) is unreachable on a VT-capable JDK — pass --add-opens " - + "java.base/java.lang=ALL-UNNAMED, or this antagonist loses its core " - + "stale-carrier UAF coverage"); - } - running = true; - driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); - driver.setDaemon(true); - driver.start(); - pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); - pinningDriver.setDaemon(true); - pinningDriver.start(); - staleRacerDriver = new Thread(this::staleRacerLoop, "chaos-vthread-cascade-racer-driver"); - staleRacerDriver.setDaemon(true); - staleRacerDriver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long deadlineNanos = System.nanoTime() + timeout.toNanos(); - // Driver threads are only non-null once start() has actually spawned them; guard against - // stopGracefully being called when start() never ran or failed partway through. - if (driver != null) { - try { - driver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - if (pinningDriver != null) { - try { - pinningDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - if (staleRacerDriver != null) { - try { - staleRacerDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. - try { - liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is - // best-effort only — outstanding racers are daemon threads and die with the JVM. - try { - liveStaleRacers.tryAcquire(MAX_STALE_RACERS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - /** Milliseconds remaining until {@code deadlineNanos}, clamped to zero (never negative). */ - private static long remainingMillis(long deadlineNanos) { - long remainingNanos = deadlineNanos - System.nanoTime(); - return remainingNanos <= 0L ? 0L : TimeUnit.NANOSECONDS.toMillis(remainingNanos); - } - - /** - * Independently pins short-lived virtual threads to their carrier and releases them again, - * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes - * are racing carrier migration on the other driver. - */ - private void pinningChurnLoop() { - if (OF_VIRTUAL == null || BUILDER_START == null) { - return; - } - while (running) { - if (!activePhase) { - // Quiet phase: stay out of the way so idle carriers can actually be reaped. - try { - Thread.sleep(50L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - continue; - } - for (int i = 0; i < PIN_BATCH && running && activePhase; i++) { - if (!livePinVthreads.tryAcquire()) { - continue; - } - final Object lock = PIN_LOCKS[ThreadLocalRandom.current().nextInt(PIN_LOCK_COUNT)]; - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke( - builder, - (Runnable) - () -> { - try { - synchronized (lock) { - // Sleeping while holding a monitor pins the - // carrier: the scheduler must compensate with an - // extra platform thread for other runnable - // virtual threads, then reap it once idle. - Thread.sleep(2L); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - livePinVthreads.release(); - } - }); - } catch (Throwable t) { - livePinVthreads.release(); - } - } - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private void driverLoop() { - if (OF_VIRTUAL == null || BUILDER_START == null) { - System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); - return; - } - while (running) { - // Active phase: mint cascade work. - activePhase = true; - long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); - while (running && System.nanoTime() < activeDeadlineNanos) { - for (int i = 0; i < ROOT_BATCH && running; i++) { - // Mint one chain-of-operations: a fresh synthetic trace, its root span - // activated by the tracer the moment runCascadeNode is entered. - spawnCascadeNode(0); - } - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - if (!running) { - return; - } - // Quiet phase: mint nothing new so short-lived virtual threads drain and carriers - // can actually sit idle past the scheduler's 30s keep-alive and get reaped. - activePhase = false; - try { - Thread.sleep(QUIET_PHASE_MILLIS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - /** - * Independent driver that keeps the stale-buffer racer population saturated at {@link - * #MAX_STALE_RACERS} for as long as the antagonist runs, regardless of the cascade/pin duty - * cycle — the fast-carrier scheduler needs no quiet phase to reap its carriers, so there's no - * reason to gate racer spawning on one. This is the "many vthreads hammering" dimension: - * sustained concurrent pressure on the fast-carrier scheduler's pool, not periodic bursts. - */ - private void staleRacerLoop() { - while (running) { - boolean spawnedAny = spawnStaleBufferRacers(); - try { - // Once the pool is fully saturated, back off instead of re-polling 1000x/sec for - // permits that free up far slower (racers sleep up to FAST_STALE_RACER_SLEEP_MAX_MILLIS). - Thread.sleep(spawnedAny ? 1L : 20L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - /** Returns whether at least one racer was actually started this pass. */ - private boolean spawnStaleBufferRacers() { - boolean spawnedAny = false; - for (int i = 0; i < STALE_RACER_BATCH && running; i++) { - if (!liveStaleRacers.tryAcquire()) { - continue; - } - if (startFastCarrierVirtualThread(this::runStaleBufferRacer) == null) { - liveStaleRacers.release(); - } else { - spawnedAny = true; - } - } - return spawnedAny; - } - - /** - * Activates a span once, sleeps past the carrier's keep-alive, then activates another. Under - * {@code ContextStorageMode.THREAD} the second activation reuses whatever {@code - * DirectByteBuffer} was cached on the first — if the original carrier has since been reaped - * and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. - * - *

The sleep is randomized per racer across [{@link #FAST_STALE_RACER_SLEEP_MIN_MILLIS}, - * {@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}] rather than fixed, so a sustained run samples a - * spread of timings relative to {@link #FAST_SCHEDULER_KEEPALIVE_MILLIS} instead of only ever - * probing the same instant. - */ - private void runStaleBufferRacer() { - try { - staleRacerBefore(); - long sleepMillis = - ThreadLocalRandom.current() - .nextLong( - FAST_STALE_RACER_SLEEP_MIN_MILLIS, - FAST_STALE_RACER_SLEEP_MAX_MILLIS + 1); - Thread.sleep(sleepMillis); - staleRacerAfter(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - liveStaleRacers.release(); - } - } - - @Trace(operationName = "chaos.stale.before", resourceName = "chaos.stale.before") - private void staleRacerBefore() { - // Entry/exit alone is the "write" — the tracer activates/deactivates a span around this - // no-op body, exactly like TraceContextAntagonist's inner()/outer(). - } - - @Trace(operationName = "chaos.stale.after", resourceName = "chaos.stale.after") - private void staleRacerAfter() {} - - /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ - private void spawnCascadeNode(int depth) { - if (!running || !liveVthreads.tryAcquire()) { - return; - } - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke( - builder, - (Runnable) - () -> { - try { - runCascadeNode(depth); - } finally { - liveVthreads.release(); - } - }); - } catch (Throwable t) { - liveVthreads.release(); - } - } - - /** - * One hop of the cascade, run as its own {@link Trace}-annotated span on a fresh virtual - * thread. The tracer activates this span on entry and deactivates it on exit; while it's - * active, this hop fans out to child hops on new virtual threads, handing the tracer's own - * virtual-thread instrumentation the job of propagating it as parent context onto each child. - */ - @Trace(operationName = "chaos.cascade.hop", resourceName = "chaos.cascade.hop") - private void runCascadeNode(int depth) { - long r = ThreadLocalRandom.current().nextLong() ^ ((long) depth << 48); - for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { - r = nestedOp(r); - } - sink.addAndGet(r); - - // Propagate the chain to the next hop before this thread's own span is torn down. - if (depth < MAX_DEPTH && running) { - for (int i = 0; i < FAN_OUT; i++) { - spawnCascadeNode(depth + 1); - } - } - } - - /** - * A nested sub-operation within a hop: the tracer activates a child span on entry, {@link - * #forceUnmount} sleeps briefly to force the virtual thread off its current carrier, then the - * span's exit (or the next nested entry) writes context again — racing carrier migration - * against the cached context conduit. - */ - @Trace(operationName = "chaos.cascade.op", resourceName = "chaos.cascade.op") - private long nestedOp(long seed) { - return forceUnmount(seed); - } - - /** - * Burns some CPU (a stand-in for real work) then blocks briefly, forcing the virtual thread - * to genuinely unmount. On resumption it may be remounted on a different carrier than the - * one it was on when the context before this call was written. - */ - private static long forceUnmount(long seed) { - long r = seed; - for (int i = 0; i < 2_000; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - } - try { - Thread.sleep(1L + (r & 1)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return r; - } - - /** - * Resolves {@code java.lang.VirtualThread}'s package-private {@code (Executor, String, int, - * Runnable)} constructor, the only way to bind a virtual thread to a custom scheduler on - * mainline OpenJDK 21+ (there is no public {@code Thread.ofVirtual().scheduler(Executor)} - * API). Requires {@code --add-opens java.base/java.lang=ALL-UNNAMED}; returns {@code null} - * (caller falls back to the default-scheduler path) if that flag is absent, the JVM is - * pre-21, or the constructor's signature changes on some future JDK build. - */ - private static Constructor resolveVirtualThreadCtor() { - try { - Class vthreadClass = Class.forName("java.lang.VirtualThread"); - Constructor ctor = - vthreadClass.getDeclaredConstructor( - Executor.class, String.class, int.class, Runnable.class); - ctor.setAccessible(true); - return ctor; - } catch (Throwable t) { - return null; - } - } - - /** - * A carrier pool with a short keep-alive instead of the JDK default scheduler's fixed 30s, so - * an idle carrier's OS thread actually exits (and its {@code ProfiledThread} is freed) in - * well under a second. - */ - private static Executor newFastCarrierScheduler() { - return new ThreadPoolExecutor( - 0, - MAX_FAST_CARRIER_THREADS, - FAST_SCHEDULER_KEEPALIVE_MILLIS, - TimeUnit.MILLISECONDS, - new SynchronousQueue<>(), - r -> { - Thread carrier = new Thread(r, "chaos-vthread-cascade-fast-carrier"); - carrier.setDaemon(true); - return carrier; - }); - } - - /** - * Starts {@code task} on a virtual thread bound to {@link #fastCarrierScheduler}. Returns - * {@code null} (never starts anything) if {@link #VTHREAD_CTOR} wasn't resolved or the - * reflective construction fails; {@link #start} already fails fast when {@link #VTHREAD_CTOR} - * is unreachable on a VT-capable JDK, so callers here only need to handle the rarer - * construction-failure case (e.g. transient reflection errors). - * - *

{@code 0} for the characteristics argument mirrors {@code Thread.ofVirtual()}'s default - * (the only known bit, {@code Thread.NO_INHERIT_THREAD_LOCALS}, is unset). - */ - private Thread startFastCarrierVirtualThread(Runnable task) { - if (VTHREAD_CTOR == null) { - return null; - } - try { - Thread t = (Thread) VTHREAD_CTOR.newInstance(fastCarrierScheduler, null, 0, task); - t.start(); - return t; - } catch (Throwable t) { - return null; - } - } - - private static Method resolveOfVirtual() { - try { - return Thread.class.getMethod("ofVirtual"); - } catch (NoSuchMethodException e) { - return null; - } - } - - private static Method resolveBuilderStart() { - try { - Class builder = Class.forName("java.lang.Thread$Builder"); - return builder.getMethod("start", Runnable.class); - } catch (ClassNotFoundException | NoSuchMethodException e) { - return null; - } - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java index 9b538e3dbc..0b43dc50ce 100644 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java +++ b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java @@ -1,6 +1,5 @@ package com.datadoghq.profiler.stresstest.scenarios.counters; -import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; import com.datadoghq.profiler.context.ContextExecutor; import com.datadoghq.profiler.context.Tracing; @@ -9,7 +8,6 @@ import org.openjdk.jmh.infra.Blackhole; import java.io.IOException; -import java.util.Arrays; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -18,6 +16,9 @@ public class TracedParallelWork { + private static final int SLOT_TAG0 = 0; + private static final int SLOT_TAG1 = 1; + @State(Scope.Benchmark) public static class BenchmarkState extends Configuration { @@ -29,10 +30,6 @@ public static class BenchmarkState extends Configuration { int tagCardinality; ContextExecutor executor; JavaProfiler profiler; - ContextSetter contextSetter; - - int tag0; - int tag1; String[] tagValues; @@ -49,9 +46,6 @@ public String getTag(long id) { public void setup() throws IOException { profiler = JavaProfiler.getInstance(); executor = new ContextExecutor(200, profiler); - contextSetter = new ContextSetter(profiler, Arrays.asList("tag0", "tag1")); - tag0 = contextSetter.offsetOf("tag0"); - tag1 = contextSetter.offsetOf("tag1"); tagValues = IntStream.range(0, tagCardinality).mapToObj(i -> UUID.randomUUID().toString()) .toArray(String[]::new); } @@ -59,12 +53,10 @@ public void setup() throws IOException { @Benchmark @Threads(8) - @SuppressWarnings("deprecation") public Object work(BenchmarkState state, Blackhole bh) throws ExecutionException, InterruptedException { try (Tracing.Context context = Tracing.newContext(state::newTraceId, state.profiler)) { - state.profiler.setContext(context.getSpanId(), context.getRootSpanId()); - state.contextSetter.setContextValue(state.tag0, state.getTag(context.getSpanId())); - state.contextSetter.setContextValue(state.tag1, state.getTag(context.getSpanId() + 1)); + state.profiler.setContextValue(SLOT_TAG0, state.getTag(context.getSpanId())); + state.profiler.setContextValue(SLOT_TAG1, state.getTag(context.getSpanId() + 1)); Future f = state.executor.submit(() -> compute(state)); bh.consume(compute(state)); return f.get(); @@ -74,9 +66,9 @@ public Object work(BenchmarkState state, Blackhole bh) throws ExecutionException public long compute(BenchmarkState state) { long x = ThreadLocalRandom.current().nextLong(); for (int i = 0; i < 10_000; i++) { - state.contextSetter.setContextValue(state.tag0, state.getTag(x)); + state.profiler.setContextValue(SLOT_TAG0, state.getTag(x)); x ^= ThreadLocalRandom.current().nextLong(); - state.contextSetter.setContextValue(state.tag1, state.getTag(x)); + state.profiler.setContextValue(SLOT_TAG1, state.getTag(x)); } return x; } diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java index 2f3277faab..ebc1d7be26 100644 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java +++ b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java @@ -36,26 +36,19 @@ /** * Perf-guard for the all-native context write API: the full per-scope activate+deactivate cycle - * as dd-trace-java drives it, comparing the production combined native calls - * ({@code setTraceContext}/{@code clearTraceContext}) against the deprecated DirectByteBuffer - * sequence ({@code setContext} + 2x {@code setContextAttribute}, then {@code clearContext} + 2x - * {@code clearContextAttribute}). + * as dd-trace-java drives it, using the combined native calls ({@code setTraceContext}/ + * {@code clearTraceContext}). * *

One measured op = one full activate+deactivate cycle (@OperationsPerInvocation({@value #BATCH})). - * The all-native cycle is expected to be at parity or a modest win over the fine-grained DBB cycle, - * on both platform and mounted virtual threads and independent of storage mode (see the design - * note, doc/plans/2026-07-02-all-native-context-storage-design.md). Guards against a regression in - * the shipping API. + * Guards against a regression in the shipping API, on both platform and mounted virtual threads. * *

Run: {@code ./gradlew :ddprof-stresstest:jmh -PjmhInclude="ContextCombinedBenchmark"} */ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 3, warmups = 0, - jvmArgsAppend = {"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED"}) +@Fork(value = 3, warmups = 0) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 2) -@SuppressWarnings("deprecation") // intentionally exercises the deprecated DirectByteBuffer path as the baseline public class ContextCombinedBenchmark { static final int BATCH = 20_000; @@ -89,16 +82,11 @@ private static void runOnVirtualThread(Runnable r) throws InterruptedException { public static class ProfilerState { JavaProfiler profiler; - @Param({"carrier", "thread"}) - String mode; - @Setup(Level.Trial) public void setup() throws Exception { - System.setProperty("ddprof.debug.context.storage.mode", mode); profiler = JavaProfiler.getInstance(); Path jfr = Files.createTempFile("ctx-combined", ".jfr"); profiler.execute("start,cpu=10ms,attributes=op;res,jfr,file=" + jfr.toAbsolutePath()); - System.out.println("[bench] requested mode=" + mode + " actual=" + profiler.contextStorageMode()); } } @@ -121,17 +109,6 @@ private static void nativeCycle(JavaProfiler p, CtxState s) { p.clearTraceContext(); } - // ---- deprecated DirectByteBuffer cycle (6 calls; cache-hit attribute writes) ---- - - private static void dbbCycle(JavaProfiler p, CtxState s) { - p.setContext(s.lrs, s.span, 0, s.trLo); - p.setContextAttribute(SLOT_OP, OP_NAME); - p.setContextAttribute(SLOT_RES, RES_NAME); - p.setContext(0, 0, 0, 0); - p.clearContextAttribute(SLOT_OP); - p.clearContextAttribute(SLOT_RES); - } - @Benchmark @OperationsPerInvocation(BATCH) public void platform_native_cycle(ProfilerState ps, CtxState s) { @@ -140,14 +117,6 @@ public void platform_native_cycle(ProfilerState ps, CtxState s) { } } - @Benchmark - @OperationsPerInvocation(BATCH) - public void platform_dbb_cycle(ProfilerState ps, CtxState s) { - for (int i = 0; i < BATCH; i++) { - dbbCycle(ps.profiler, s); - } - } - @Benchmark @OperationsPerInvocation(BATCH) public void vthread_native_cycle(ProfilerState ps, CtxState s) throws InterruptedException { @@ -157,14 +126,4 @@ public void vthread_native_cycle(ProfilerState ps, CtxState s) throws Interrupte } }); } - - @Benchmark - @OperationsPerInvocation(BATCH) - public void vthread_dbb_cycle(ProfilerState ps, CtxState s) throws InterruptedException { - runOnVirtualThread(() -> { - for (int i = 0; i < BATCH; i++) { - dbbCycle(ps.profiler, s); - } - }); - } } diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java deleted file mode 100644 index dbd4eec4cc..0000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Benchmarks for ThreadContext operations — measures the per-call cost of - * context set/get/attribute operations on the span start/end hot path. - * - *

Run: ./gradlew :ddprof-stresstest:jmh -PjmhInclude="ThreadContextBenchmark" - * - *

Run with different JAVA_HOME to compare JNI (17+) vs ByteBuffer (<17) paths. - */ -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 2, warmups = 0) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 2) -public class ThreadContextBenchmark { - - private static final String[] ROUTES = { - "GET /api/users", "POST /api/orders", "GET /api/health", - "PUT /api/users/{id}", "DELETE /api/sessions" - }; - - @State(Scope.Benchmark) - public static class ProfilerState { - JavaProfiler profiler; - - @Setup(Level.Trial) - public void setup() throws Exception { - profiler = JavaProfiler.getInstance(); - Path jfr = Files.createTempFile("bench", ".jfr"); - profiler.execute("start,cpu=10ms,attributes=http.route,jfr,file=" + jfr.toAbsolutePath()); - } - } - - @State(Scope.Thread) - public static class ThreadState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int counter; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - } - } - - @State(Scope.Thread) - public static class ReapplyState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int[] constantIds; - byte[][] utf8; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - - // Prime the normal path to obtain constant IDs, then snapshot for reapply. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - ctx.setContextAttribute(i, ROUTES[i]); - } - constantIds = new int[10]; - ctx.copyCustoms(constantIds); - utf8 = new byte[10][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - } - - @Setup(Level.Iteration) - public void resetToSteadyState() { - // Re-establish a live span (valid=1) and pre-populate attrs_data with all slots - // before each measurement window. Without this, reapplyByIdAndBytes sees a - // different attrs_data state on the first invocation of each iteration (empty - // after put() in the previous iteration or after the trial setup), causing a - // bimodal distribution across forks due to JIT profile divergence. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - if (!ctx.setContextAttributesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException( - "resetToSteadyState: setContextAttributesByIdAndBytes failed; benchmark state invalid"); - } - } - } - - @Benchmark - public void setContextFull(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - public boolean setAttrCacheHit(ThreadState ts) { - return ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public void spanLifecycle(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(2) - public void setContextFull_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(4) - public void setContextFull_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(2) - public void spanLifecycle_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(4) - public void spanLifecycle_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public long getSpanId(ThreadState ts) { - return ts.ctx.getSpanId(); - } - - @Benchmark - public void clearContext(ThreadState ts) { - ts.ctx.put(0, 0, 0, 0); - } - - /** Bare reapply cost with constant IDs and bytes already in hand — no Dictionary lookup. */ - @Benchmark - public boolean reapplyByIdAndBytes(ReapplyState rs) { - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - /** Full reapply cycle: span activation wipes slots, then reapply restores them. */ - @Benchmark - public boolean reapplyCycle(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(2) - public boolean reapplyCycle_2t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(4) - public boolean reapplyCycle_4t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java index edded2ea90..97d0faaa7b 100644 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java +++ b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java @@ -24,12 +24,11 @@ protected RunnableFuture newTaskFor(Runnable runnable, T value) { @Override protected void beforeExecute(Thread t, Runnable r) { super.beforeExecute(t, r); + // Clear any context left over from this worker's previous task before wall-clock + // profiling is re-enabled below, so a signal in the window before ContextTask.run() + // activates the new context can't be attributed to the previous task's span. + profiler.clearTraceContext(); profiler.addThread(); - // Prime OTEL context TLS to avoid race condition with wall clock signals. - // TLS is lazily initialized on first setContext() call, which happens in - // ContextTask.run() after this method returns. If a wall clock signal - // arrives between now and then, the context would be uninitialized. - profiler.setContext(0, 0); } @Override diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java index 18a15a4719..17b2089feb 100644 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java +++ b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java @@ -82,7 +82,7 @@ public MigratingContext snapshot() { } private void notifyProfiler() { - profiler.setContext(spanId, rootSpanId); + profiler.setTraceContext(rootSpanId, spanId, 0, spanId, -1, null, -1, null); } public long getRootSpanId() { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index de75c2f068..1da7b59bf8 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -330,7 +330,6 @@ protected void runTests(Runnable... runnables) throws InterruptedException { public final void stopProfiler() { if (!stopped) { profiler.stop(); - profiler.resetThreadContext(); stopped = true; checkConfig(); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java deleted file mode 100644 index 853b06431d..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Comprehensive tests for {@link BufferWriter} class. - * - *

This test validates: - *

    - *
  • Correct implementation selection based on Java version
    • - *
  • Basic write and read operations with ordered (release) semantics
    • - *
  • Release (ordered) semantics for single-threaded signal handler safety
    • - *
  • Int and long write operations
    • - *
  • Various offsets and buffer positions
    • - *
  • Edge cases and boundary conditions
    • - *
- */ -public class BufferWriterTest { - - private BufferWriter bufferWriter; - - @BeforeEach - public void setUp() { - bufferWriter = new BufferWriter(); - } - - /** - * Helper method to create a direct ByteBuffer with native byte order. - * BufferWriter implementations use native byte order, so tests must read in the same order. - */ - private ByteBuffer createBuffer(int capacity) { - return ByteBuffer.allocateDirect(capacity).order(ByteOrder.nativeOrder()); - } - - /** - * Tests that the correct implementation is loaded based on Java version. - */ - @Test - public void testCorrectImplementationLoaded() { - assertNotNull(bufferWriter, "BufferWriter instance should not be null"); - - // Verify implementation can be used without throwing exceptions - ByteBuffer buffer = createBuffer(16); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 0, 42L)); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 8, 99L)); - } - - /** - * Tests basic ordered long write and read functionality at offset 0. - */ - @Test - public void testWriteOrderedLongAtOffsetZero() { - ByteBuffer buffer = createBuffer(16); - long expectedValue = 0x123456789ABCDEF0L; - - bufferWriter.writeOrderedLong(buffer, 0, expectedValue); - long actualValue = buffer.getLong(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written value"); - } - - /** - * Tests basic ordered int write and read functionality. - */ - @Test - public void testWriteOrderedInt() { - ByteBuffer buffer = createBuffer(16); - int expectedValue = 0x12345678; - - bufferWriter.writeOrderedInt(buffer, 0, expectedValue); - int actualValue = buffer.getInt(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written int value"); - } - - /** - * Tests ordered long write operations at various offsets within the buffer. - */ - @Test - public void testWriteOrderedLongAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 8-byte aligned offsets - int[] offsets = {0, 8, 16, 24, 32, 40, 48, 56}; - long[] expectedValues = { - 0x1111111111111111L, - 0x2222222222222222L, - 0x3333333333333333L, - 0x4444444444444444L, - 0x5555555555555555L, - 0x6666666666666666L, - 0x7777777777777777L, - 0x8888888888888888L - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedLong(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - long actualValue = buffer.getLong(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Buffer value at offset %d should match", offsets[i])); - } - } - - /** - * Tests int write operations at various offsets within the buffer. - */ - @Test - public void testWriteIntAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 4-byte aligned offsets - int[] offsets = {0, 4, 8, 12, 16, 20, 24, 28}; - int[] expectedValues = { - 0x11111111, - 0x22222222, - 0x33333333, - 0x44444444, - 0x55555555, - 0x66666666, - 0x77777777, - 0x88888888 - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedInt(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - int actualValue = buffer.getInt(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Int value at offset %d should match", offsets[i])); - } - } - - /** - * Tests writing special long values (min, max, zero, negative). - */ - @Test - public void testSpecialLongValues() { - ByteBuffer buffer = createBuffer(40); - long[] specialValues = { - 0L, // Zero - Long.MIN_VALUE, // Minimum long - Long.MAX_VALUE, // Maximum long - -1L, // All bits set - 0xDEADBEEFCAFEBABEL // Arbitrary pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 8; - bufferWriter.writeOrderedLong(buffer, offset, specialValues[i]); - long actualValue = buffer.getLong(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests writing special int values (min, max, zero, negative). - */ - @Test - public void testSpecialIntValues() { - ByteBuffer buffer = createBuffer(24); - int[] specialValues = { - 0, // Zero - Integer.MIN_VALUE, // Minimum int - Integer.MAX_VALUE, // Maximum int - -1, // All bits set - 0xDEADBEEF, // Arbitrary pattern - 0x12345678 // Another pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 4; - bufferWriter.writeOrderedInt(buffer, offset, specialValues[i]); - int actualValue = buffer.getInt(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special int value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests that multiple consecutive writes work correctly without interference. - */ - @Test - public void testConsecutiveWrites() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write multiple times to the same offset - bufferWriter.writeOrderedLong(buffer, offset, 100L); - assertEquals(100L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 200L); - assertEquals(200L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 300L); - assertEquals(300L, buffer.getLong(offset)); - } - - /** - * Tests that writes to adjacent locations don't interfere with each other. - */ - @Test - public void testNonInterference() { - ByteBuffer buffer = createBuffer(32); - - long value1 = 0x1111111111111111L; - long value2 = 0x2222222222222222L; - long value3 = 0x3333333333333333L; - - bufferWriter.writeOrderedLong(buffer, 0, value1); - bufferWriter.writeOrderedLong(buffer, 8, value2); - bufferWriter.writeOrderedLong(buffer, 16, value3); - - assertEquals(value1, buffer.getLong(0), "First value should not be affected"); - assertEquals(value2, buffer.getLong(8), "Second value should not be affected"); - assertEquals(value3, buffer.getLong(16), "Third value should not be affected"); - } - - /** - * Tests writing to a buffer at maximum valid offset for longs. - */ - @Test - public void testMaximumValidOffsetLong() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 8; // 8 bytes for a long - - long expectedValue = 0xFEDCBA9876543210L; - bufferWriter.writeOrderedLong(buffer, maxValidOffset, expectedValue); - - long actualValue = buffer.getLong(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Value at maximum offset should be written correctly"); - } - - /** - * Tests writing to a buffer at maximum valid offset for ints. - */ - @Test - public void testMaximumValidOffsetInt() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 4; // 4 bytes for an int - - int expectedValue = 0xFEDCBA98; - bufferWriter.writeOrderedInt(buffer, maxValidOffset, expectedValue); - - int actualValue = buffer.getInt(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Int value at maximum offset should be written correctly"); - } - - /** - * Tests that overwriting values works correctly. - */ - @Test - public void testOverwrite() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write initial pattern - bufferWriter.writeOrderedLong(buffer, offset, 0xAAAAAAAAAAAAAAAAL); - assertEquals(0xAAAAAAAAAAAAAAAAL, buffer.getLong(offset)); - - // Overwrite with different pattern - bufferWriter.writeOrderedLong(buffer, offset, 0x5555555555555555L); - assertEquals(0x5555555555555555L, buffer.getLong(offset)); - - // Overwrite with zeros - bufferWriter.writeOrderedLong(buffer, offset, 0L); - assertEquals(0L, buffer.getLong(offset)); - } - - /** - * Tests parallel writes to different offsets from multiple threads. - */ - @Test - public void testParallelWrites() throws InterruptedException { - ByteBuffer buffer = createBuffer(128); - int numThreads = 8; - Thread[] threads = new Thread[numThreads]; - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch doneLatch = new CountDownLatch(numThreads); - - for (int i = 0; i < numThreads; i++) { - final int threadIndex = i; - final int offset = threadIndex * 16; - final long expectedValue = (long) threadIndex * 0x1111111111111111L; - - threads[i] = new Thread(() -> { - try { - startLatch.await(5, TimeUnit.SECONDS); - bufferWriter.writeOrderedLong(buffer, offset, expectedValue); - doneLatch.countDown(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - threads[i].start(); - } - - // Start all threads simultaneously - startLatch.countDown(); - - // Wait for all threads to complete - assertTrue(doneLatch.await(5, TimeUnit.SECONDS), - "All threads should complete within timeout"); - - // Verify all values were written correctly - for (int i = 0; i < numThreads; i++) { - int offset = i * 16; - long expectedValue = (long) i * 0x1111111111111111L; - long actualValue = buffer.getLong(offset); - assertEquals(expectedValue, actualValue, - String.format("Thread %d's value at offset %d should be correct", i, offset)); - } - - for (Thread thread : threads) { - thread.join(1000); - } - } - - /** - * Tests that the buffer's original position and limit are not affected by writes. - */ - @Test - public void testBufferStatePreservation() { - ByteBuffer buffer = createBuffer(64); - buffer.position(10); - buffer.limit(50); - - int originalPosition = buffer.position(); - int originalLimit = buffer.limit(); - - bufferWriter.writeOrderedLong(buffer, 16, 0x123456789ABCDEF0L); - bufferWriter.writeOrderedLong(buffer, 24, 0xFEDCBA9876543210L); - bufferWriter.writeOrderedInt(buffer, 32, 0x12345678); - bufferWriter.writeOrderedInt(buffer, 36, 0x9ABCDEF0); - - assertEquals(originalPosition, buffer.position(), - "Buffer position should not be affected by writes"); - assertEquals(originalLimit, buffer.limit(), - "Buffer limit should not be affected by writes"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java new file mode 100644 index 0000000000..53e32ed5aa --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Unit tests for {@link ContextValueCache}, the process-wide value cache backing the all-native + * context write path. Package-private, so this test lives in {@code com.datadoghq.profiler}. + */ +public class ContextValueCacheTest { + + private final ContextValueCache cache = new ContextValueCache(); + + private static void loadLibrary() throws IOException { + JavaProfiler.getInstance(); // registerConstant0 is native + } + + @Test + public void nullResolvesToNull() throws IOException { + loadLibrary(); + assertNull(cache.resolve(null)); + } + + @Test + public void resolveHitsCacheOnSecondLookup() throws IOException { + loadLibrary(); + ContextValueCache.Entry first = cache.resolve("hello"); + assertNotNull(first); + ContextValueCache.Entry second = cache.resolve("hello"); + assertSame(first, second, "an unchanged value must hit the cache, not re-register"); + } + + @Test + public void oversizedValueResolvesToNull() throws IOException { + loadLibrary(); + char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES + 1]; + Arrays.fill(chars, 'x'); + assertNull(cache.resolve(new String(chars))); + } + + @Test + public void maxSizeValueResolves() throws IOException { + loadLibrary(); + char[] chars = new char[ContextValueCache.MAX_VALUE_BYTES]; + Arrays.fill(chars, 'x'); + String value = new String(chars); + ContextValueCache.Entry e = cache.resolve(value); + assertNotNull(e); + assertEquals(ContextValueCache.MAX_VALUE_BYTES, e.utf8.length); + } + + @Test + public void hashCollisionEvictsPreviousEntryButBothRemainResolvable() throws IOException { + loadLibrary(); + // ContextValueCache is direct-mapped by value.hashCode() & 0xFF (SIZE=256). Find two + // distinct strings that collide in the same slot to exercise the eviction path. + String a = "collision-a"; + int slotA = a.hashCode() & 0xFF; + String b = null; + for (int i = 0; i < 100_000; i++) { + String candidate = "collision-b-" + i; + if ((candidate.hashCode() & 0xFF) == slotA && !candidate.equals(a)) { + b = candidate; + break; + } + } + assertNotNull(b, "failed to find a colliding string for the test slot"); + + ContextValueCache.Entry entryA = cache.resolve(a); + assertNotNull(entryA); + ContextValueCache.Entry entryB = cache.resolve(b); + assertNotNull(entryB); + assertNotEquals(entryA.encoding, entryB.encoding); + + // Re-resolving "a" after it was evicted from its slot by "b" must still work — it's a + // cache miss, so it re-registers and gets a fresh (but valid) entry. + ContextValueCache.Entry entryAAgain = cache.resolve(a); + assertNotNull(entryAAgain); + assertEquals(a, entryAAgain.key); + } + + @Test + public void clearDropsCachedEntries() throws IOException { + loadLibrary(); + ContextValueCache.Entry before = cache.resolve("to-clear"); + assertNotNull(before); + cache.clear(); + ContextValueCache.Entry after = cache.resolve("to-clear"); + assertNotNull(after); + assertNotSame(before, after, "resolve after clear() must not return the pre-clear entry"); + } + + @Test + public void charSequenceHitNeverCallsToString() throws IOException { + loadLibrary(); + ContextValueCache.Entry primed = cache.resolve("shared-value"); + assertNotNull(primed); + + ToStringCountingCharSequence value = new ToStringCountingCharSequence("shared-value"); + ContextValueCache.Entry hit = cache.resolve((CharSequence) value); + assertSame(primed, hit, "a content-equal CharSequence must hit the String-keyed entry"); + assertEquals(0, value.toStringCalls, "a cache hit must not materialize a String"); + } + + @Test + public void charSequenceMissMaterializesStringExactlyOnce() throws IOException { + loadLibrary(); + ToStringCountingCharSequence value = new ToStringCountingCharSequence("not-yet-cached"); + ContextValueCache.Entry e = cache.resolve((CharSequence) value); + assertNotNull(e); + assertEquals("not-yet-cached", e.key); + assertEquals(1, value.toStringCalls, "a miss must materialize a String exactly once"); + } + + /** A {@link CharSequence} that is not a {@link String}, counting {@link #toString()} calls. */ + private static final class ToStringCountingCharSequence implements CharSequence { + private final String value; + int toStringCalls = 0; + + ToStringCountingCharSequence(String value) { + this.value = value; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return value.subSequence(start, end); + } + + @Override + public String toString() { + toStringCalls++; + return value; + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java index 002e4fba93..787275dcdf 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/MaxContextSlotsTest.java @@ -34,8 +34,5 @@ public void javaBoundMatchesNativeCapacity() throws IOException { JavaProfiler.getInstance(); // ensure the native library is loaded assertEquals(JavaProfiler.maxContextSlots0(), JavaProfiler.MAX_CONTEXT_SLOTS, "JavaProfiler.MAX_CONTEXT_SLOTS drifted from native DD_TAGS_CAPACITY"); - // ThreadContext mirrors the same native bound on the deprecated path; keep them aligned too. - assertEquals(JavaProfiler.MAX_CONTEXT_SLOTS, ThreadContext.MAX_CUSTOM_SLOTS, - "MAX_CONTEXT_SLOTS drifted from ThreadContext.MAX_CUSTOM_SLOTS"); } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java deleted file mode 100644 index ec45ce7cda..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/OtelContextStorageTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Unit tests for {@link OtelContextStorage} mode selection and the {@code thread} kill-switch. - * - *

{@link OtelContextStorage#create()} reads {@link OtelContextStorage#MODE_PROPERTY} on each - * call and returns an instance whose concrete type determines its {@link ContextStorageMode} - * — there is no shared mutable state — so these tests can drive it directly via the system - * property without disturbing the profiler's already-constructed storage. - */ -public class OtelContextStorageTest { - - private String saved; - - @AfterEach - public void restore() { - if (saved == null) { - System.clearProperty(OtelContextStorage.MODE_PROPERTY); - } else { - System.setProperty(OtelContextStorage.MODE_PROPERTY, saved); - } - } - - private void setMode(String mode) { - saved = System.getProperty(OtelContextStorage.MODE_PROPERTY); - System.setProperty(OtelContextStorage.MODE_PROPERTY, mode); - } - - /** The kill-switch: {@code mode=thread} always yields plain thread-scoped storage. */ - @Test - public void threadModeForcesPlainThreadLocal() { - setMode("thread"); - ThreadLocal storage = OtelContextStorage.create(); - assertNotNull(storage); - assertEquals(ContextStorageMode.THREAD, OtelContextStorage.modeOf(storage)); - // A forced-thread instance must be a plain ThreadLocal, never CarrierThreadLocal. - assertEquals(ThreadLocal.class, storage.getClass(), - "thread mode must not use jdk.internal.misc.CarrierThreadLocal"); - } - - /** - * On JDK 21+ with {@code jdk.internal.misc} exported (the build adds the flag), {@code auto} - * resolves to carrier scoping. Skipped otherwise, since the fallback is environment-driven. - */ - @Test - public void autoModeUsesCarrierWhenAvailable() { - assumeTrue(Platform.isJavaVersionAtLeast(21), "carrier scoping needs JDK 21+"); - // Confirm CarrierThreadLocal is actually reachable in this run; if not (export not - // granted), auto legitimately falls back to THREAD and there is nothing to assert. - assumeTrue(carrierThreadLocalAccessible(), - "jdk.internal.misc.CarrierThreadLocal not accessible; needs --add-exports"); - - setMode("auto"); - ThreadLocal storage = OtelContextStorage.create(); - assertEquals(ContextStorageMode.CARRIER, OtelContextStorage.modeOf(storage), - "auto must select carrier scoping when CarrierThreadLocal is accessible"); - } - - /** {@code carrier} succeeds and yields carrier-scoped storage when CarrierThreadLocal is accessible. */ - @Test - public void carrierModeUsesCarrierWhenAvailable() { - assumeTrue(carrierThreadLocalAccessible(), - "CarrierThreadLocal not accessible; the fail-fast path is covered by the throw test"); - setMode("carrier"); - ThreadLocal storage = OtelContextStorage.create(); - assertEquals(ContextStorageMode.CARRIER, OtelContextStorage.modeOf(storage)); - } - - /** - * {@code carrier} fails hard when CarrierThreadLocal is unavailable, rather than silently - * falling back to the virtual-thread-pinned storage the fix removes. - */ - @Test - public void carrierModeThrowsWhenUnavailable() { - assumeTrue(!carrierThreadLocalAccessible(), - "CarrierThreadLocal IS accessible here; the fail-fast path only applies when it is not (older JDKs / no export)"); - setMode("carrier"); - assertThrows(IllegalStateException.class, OtelContextStorage::create); - } - - private static boolean carrierThreadLocalAccessible() { - try { - Class c = Class.forName("jdk.internal.misc.CarrierThreadLocal"); - c.getConstructor().newInstance(); - return true; - } catch (Throwable t) { - return false; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java deleted file mode 100644 index 5078181df2..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java +++ /dev/null @@ -1,164 +0,0 @@ -package com.datadoghq.profiler; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; - -/** - * Pure-Java unit test for {@link ScopeStack}. Uses heap-backed {@link ByteBuffer}s so - * no native library is required. Exercises depth accounting, underflow, and round-trip - * preservation of trace/span IDs across fast-path and chunked-path depths. - */ -public class ScopeStackTest { - - // Offsets mirror OtelThreadContextRecord in otel_context.h and the sidecar layout - // built by initializeContextTLS0 in javaApi.cpp. These are spec-fixed; guarded by - // static_asserts in native code. All are absolute within the unified buffer. - private static final int TRACE_ID_OFFSET = 0; - private static final int SPAN_ID_OFFSET = 16; - private static final int VALID_OFFSET = 24; - private static final int ATTRS_DATA_SIZE_OFFSET = 26; - private static final int ATTRS_DATA_OFFSET = 28; - private static final int LRS_OFFSET = 640 + 40; // after 640-byte record + 10 * sizeof(u32) - - private static ThreadContext newContext() { - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - return new ThreadContext(buf, metadata); - } - - private static void assumeLittleEndian() { - Assumptions.assumeTrue( - ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN, - "ThreadContext only supports little-endian platforms"); - } - - @Test - public void depthBalance() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertEquals(0, stack.depth()); - stack.enter(ctx); - assertEquals(1, stack.depth()); - stack.enter(ctx); - assertEquals(2, stack.depth()); - stack.exit(ctx); - assertEquals(1, stack.depth()); - stack.exit(ctx); - assertEquals(0, stack.depth()); - } - - @Test - public void exitUnderflowThrows() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertThrows(IllegalStateException.class, () -> stack.exit(ctx)); - } - - @Test - public void fastPathRoundTrip() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(/*lrs*/ 100L, /*span*/ 200L, /*trHi*/ 0L, /*trLo*/ 300L); - assertEquals(200L, ctx.getSpanId()); - assertEquals(100L, ctx.getRootSpanId()); - - stack.enter(ctx); - ctx.put(500L, 600L, 0L, 700L); - assertEquals(600L, ctx.getSpanId()); - assertEquals(500L, ctx.getRootSpanId()); - - stack.exit(ctx); - assertEquals(200L, ctx.getSpanId(), "span must be restored"); - assertEquals(100L, ctx.getRootSpanId(), "root span must be restored"); - } - - @Test - public void chunkedPathRoundTrip() { - // Push past FAST_DEPTH (6) to exercise the lazy-chunk path and Arrays.copyOf growth. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - final int depth = 20; // FAST_DEPTH + one full 12-slot chunk + 2 into the next - for (int i = 0; i < depth; i++) { - ctx.put(1000L + i, 2000L + i, 0L, 3000L + i); - stack.enter(ctx); - } - assertEquals(depth, stack.depth()); - - // Scramble state so restore has something to correct. - ctx.put(99L, 99L, 0L, 99L); - - for (int i = depth - 1; i >= 0; i--) { - stack.exit(ctx); - assertEquals(2000L + i, ctx.getSpanId(), "span mismatch at depth " + i); - assertEquals(1000L + i, ctx.getRootSpanId(), "root mismatch at depth " + i); - } - assertEquals(0, stack.depth()); - } - - @Test - public void reusesStackAfterFullUnwind() { - // After the stack returns to depth 0, re-entering must not leak state from the prior run. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - stack.enter(ctx); - ctx.put(10L, 20L, 0L, 30L); - stack.exit(ctx); - assertEquals(2L, ctx.getSpanId()); - - ctx.put(4L, 5L, 0L, 6L); - stack.enter(ctx); - ctx.put(40L, 50L, 0L, 60L); - stack.exit(ctx); - assertEquals(5L, ctx.getSpanId()); - } - - @Test - public void snapshotOverClearedContextDoesNotRepublish() { - // Regression: snapshot() used to unconditionally re-attach, flipping valid back to 1 - // after a zero-put clear. The clear path leaves attrs_data_size / attrs_data stale and - // relies on valid=0 to keep external readers from seeing the stale bytes. Here we verify - // the valid byte directly since setContextAttribute is a native path unavailable to - // pure-Java tests. - assumeLittleEndian(); - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - ThreadContext ctx = new ThreadContext(buf, metadata); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - assertEquals(1, buf.get(VALID_OFFSET), "record must be published after non-zero put"); - - // Zero-put clear: leaves valid=0 (the all-zero early-return in setContextDirect). - ctx.put(0L, 0L, 0L, 0L); - assertEquals(0, buf.get(VALID_OFFSET), "record must be invalid after zero-put clear"); - - stack.enter(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "snapshot must preserve valid=0 — not republish a cleared record"); - - stack.exit(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "restore must replay valid=0 — not republish a cleared record"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java index 9f87801c2c..80b81c1b7b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java @@ -32,12 +32,11 @@ * *

The wall-clock sampler reads context through {@code ContextApi::get}, which returns nothing * until the thread is context-initialized (and only then is the {@code otel_thread_ctx_v1} discovery - * pointer published). Before the fix, that flag was set only by the deprecated DirectByteBuffer path - * ({@code getThreadContext()} / {@code initializeContextTLS0}); a thread that used only the - * all-native API ({@code setTraceContext}) wrote a record the sampler silently ignored. + * pointer published). {@code setTraceContext} must set that flag itself on first write — a thread + * that only ever calls the all-native API must still be visible to the sampler. * - *

This test deliberately never calls {@code getThreadContext()} on the sampled thread. - * If the native write does not initialize TLS, every sample carries spanId 0 and the assertion fails. + *

If the native write does not initialize TLS, every sample carries spanId 0 and the assertion + * fails. */ public class AllNativeContextSamplingTest extends AbstractProfilerTest { @@ -46,7 +45,7 @@ public class AllNativeContextSamplingTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - // filter=0 samples every thread, so no getThreadContext()/registration is needed. + // filter=0 samples every thread, so no extra registration is needed. return "wall=1ms,filter=0,loglevel=warn"; } @@ -54,8 +53,8 @@ protected String getProfilerCommand() { public void nativeOnlyContextIsVisibleToSampler() throws Exception { Assumptions.assumeTrue(!Platform.isJ9() && !Platform.isZing()); - // Register for wall-clock profiling (addThread(); does NOT touch getThreadContext, so the - // all-native-only nature of the test is preserved). + // Register for wall-clock profiling (addThread()); the sampled thread never touches + // anything but the all-native context API. registerCurrentThreadForWallClockProfiling(); // Keep the context live for the whole sampling window; only the all-native path is used. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java index 60f7680e16..70007fe03e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java @@ -17,7 +17,6 @@ import com.datadoghq.profiler.JavaProfiler; import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ThreadContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -38,13 +37,11 @@ * {@code clearTraceContext}, {@code setContextValue}, {@code clearContextValue}. * *

These write the current carrier's native OtelThreadContextRecord directly (no cached - * DirectByteBuffer). The DirectByteBuffer read methods on {@link ThreadContext} - * ({@code getSpanId}/{@code getRootSpanId}/{@code readTraceId}/{@code readContextAttribute}) are a - * view over that same record, so they serve as the read-back oracle here. - * - *

Ordering note: the first {@link JavaProfiler#getThreadContext()} on a thread runs the - * {@code ThreadContext} constructor, which resets the record. So every test resolves the read - * handle before writing, then reads through that handle after the native write. + * per-thread buffer). The read-back oracle is {@link JavaProfiler}'s package-private + * {@code test*} accessors ({@code testGetSpanId}/{@code testGetRootSpanId}/ + * {@code testReadTraceId}/{@code testReadContextAttribute}/{@code testIsContextValid}), which + * read the same record directly and are invoked here via reflection (this test lives in a + * different package than {@link JavaProfiler}). */ public class AllNativeContextTest { @@ -64,7 +61,6 @@ public static void setup() throws IOException { public void cleanup() { if (profilerStarted) { profiler.stop(); - profiler.resetThreadContext(); profilerStarted = false; } } @@ -88,15 +84,34 @@ private static boolean setContextValue0(int slot, int encoding, byte[] utf8) thr return (boolean) m.invoke(null, slot, encoding, utf8); } - /** Reads the record's {@code valid} byte through the ThreadContext buffer (test-only introspection). */ - private static int readValidByte(ThreadContext ctx) throws Exception { - java.lang.reflect.Field bufField = ThreadContext.class.getDeclaredField("ctxBuffer"); - bufField.setAccessible(true); - java.nio.ByteBuffer buf = (java.nio.ByteBuffer) bufField.get(ctx); - java.lang.reflect.Field offField = ThreadContext.class.getDeclaredField("validOffset"); - offField.setAccessible(true); - int off = (int) offField.get(ctx); - return buf.get(off) & 0xFF; + private static long testGetSpanId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testGetSpanId"); + m.setAccessible(true); + return (long) m.invoke(profiler); + } + + private static long testGetRootSpanId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testGetRootSpanId"); + m.setAccessible(true); + return (long) m.invoke(profiler); + } + + private static String testReadTraceId() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testReadTraceId"); + m.setAccessible(true); + return (String) m.invoke(profiler); + } + + private static String testReadContextAttribute(int slot) throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testReadContextAttribute", int.class); + m.setAccessible(true); + return (String) m.invoke(profiler, slot); + } + + private static boolean testIsContextValid() throws Exception { + Method m = JavaProfiler.class.getDeclaredMethod("testIsContextValid"); + m.setAccessible(true); + return (boolean) m.invoke(profiler); } /** {@code Thread.ofVirtual().start(task)} via reflection so this compiles with --release 8. */ @@ -111,7 +126,6 @@ private static Thread startVirtualThread(Runnable task) throws Exception { @Test public void setTraceContextRoundTrips() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); long lrs = 0x1111222233334444L; long span = 0xAAAABBBBCCCCDDDDL; @@ -119,95 +133,57 @@ public void setTraceContextRoundTrips() throws Exception { long trLo = 0x9999AAAABBBBCCCCL; profiler.setTraceContext(lrs, span, trHi, trLo, SLOT_OP, "servlet.request", SLOT_RES, "GET /users"); - assertEquals(span, ctx.getSpanId()); - assertEquals(lrs, ctx.getRootSpanId()); - assertEquals("55556666777788889999aaaabbbbcccc", ctx.readTraceId()); - assertEquals("servlet.request", ctx.readContextAttribute(SLOT_OP)); - assertEquals("GET /users", ctx.readContextAttribute(SLOT_RES)); - } - - /** The native write must produce the same observable record state as the DirectByteBuffer path. */ - @Test - public void nativeMatchesDirectByteBufferWrite() throws Exception { - start(); - ThreadContext ctx = profiler.getThreadContext(); - - long lrs = 0x0102030405060708L; - long span = 0x1122334455667788L; - long trHi = 0xAABBCCDDEEFF0011L; - long trLo = 0x2233445566778899L; - - // DirectByteBuffer path (deprecated). - profiler.setContext(lrs, span, trHi, trLo); - ctx.setContextAttribute(SLOT_OP, "servlet.request"); - ctx.setContextAttribute(SLOT_RES, "GET /users/{id}"); - long dbbSpan = ctx.getSpanId(); - long dbbRoot = ctx.getRootSpanId(); - String dbbTrace = ctx.readTraceId(); - String dbbOp = ctx.readContextAttribute(SLOT_OP); - String dbbRes = ctx.readContextAttribute(SLOT_RES); - - profiler.clearTraceContext(); - assertEquals(0, ctx.getSpanId(), "precondition: cleared before native write"); - - // All-native path — identical inputs. - profiler.setTraceContext(lrs, span, trHi, trLo, SLOT_OP, "servlet.request", SLOT_RES, "GET /users/{id}"); - - assertEquals(dbbSpan, ctx.getSpanId(), "spanId"); - assertEquals(dbbRoot, ctx.getRootSpanId(), "rootSpanId"); - assertEquals(dbbTrace, ctx.readTraceId(), "traceId"); - assertEquals(dbbOp, ctx.readContextAttribute(SLOT_OP), "op attribute"); - assertEquals(dbbRes, ctx.readContextAttribute(SLOT_RES), "resource attribute"); + assertEquals(span, testGetSpanId()); + assertEquals(lrs, testGetRootSpanId()); + assertEquals("55556666777788889999aaaabbbbcccc", testReadTraceId()); + assertEquals("servlet.request", testReadContextAttribute(SLOT_OP)); + assertEquals("GET /users", testReadContextAttribute(SLOT_RES)); } @Test public void clearTraceContextResetsRecord() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, SLOT_OP, "op", SLOT_RES, "res"); - assertEquals(0x1L, ctx.getSpanId()); + assertEquals(0x1L, testGetSpanId()); profiler.clearTraceContext(); - assertEquals(0, ctx.getSpanId(), "spanId cleared"); - assertEquals(0, ctx.getRootSpanId(), "rootSpanId cleared"); + assertEquals(0, testGetSpanId(), "spanId cleared"); + assertEquals(0, testGetRootSpanId(), "rootSpanId cleared"); // valid=0 after clear, so attributes are not observable. - assertNull(ctx.readContextAttribute(SLOT_OP), "attributes cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attributes cleared"); } /** setTraceContext resets custom slots, so a span-to-span transition must not leak attributes. */ @Test public void spanTransitionClearsAttributes() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x1L, 0x1L, 0L, 0x1L, SLOT_OP, "/api/spanA", -1, null); - assertEquals("/api/spanA", ctx.readContextAttribute(SLOT_OP)); + assertEquals("/api/spanA", testReadContextAttribute(SLOT_OP)); // Next span carries no activation attributes. profiler.setTraceContext(0x2L, 0x2L, 0L, 0x2L, -1, null, -1, null); - assertEquals(0x2L, ctx.getSpanId()); - assertNull(ctx.readContextAttribute(SLOT_OP), "previous span's attribute must be cleared"); + assertEquals(0x2L, testGetSpanId()); + assertNull(testReadContextAttribute(SLOT_OP), "previous span's attribute must be cleared"); } @Test public void singleAttributeSetAndClear() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span, no attrs assertTrue(profiler.setContextValue(SLOT_OP, "GET /api/users")); - assertEquals("GET /api/users", ctx.readContextAttribute(SLOT_OP)); + assertEquals("GET /api/users", testReadContextAttribute(SLOT_OP)); profiler.clearContextValue(SLOT_OP); - assertNull(ctx.readContextAttribute(SLOT_OP), "attribute cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attribute cleared"); } /** attrs_data overflow is reported (false), not crashed; unrelated slots keep working. */ @Test public void attributeOverflowReturnsFalse() throws Exception { start(); - profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x3L, -1, null, -1, null); StringBuilder sb = new StringBuilder(255); @@ -225,7 +201,6 @@ public void attributeOverflowReturnsFalse() throws Exception { @Test public void rejectedValuesReturnFalse() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); assertFalse(profiler.setContextValue(SLOT_OP, null), "null value rejected"); @@ -234,27 +209,7 @@ public void rejectedValuesReturnFalse() throws Exception { assertFalse(profiler.setContextValue(SLOT_OP, sb.toString()), "oversized (256B) value rejected"); // A subsequent valid write still works. assertTrue(profiler.setContextValue(SLOT_OP, "ok")); - assertEquals("ok", ctx.readContextAttribute(SLOT_OP)); - } - - /** The deprecated DBB path and the all-native path write the same record and interleave cleanly. */ - @Test - public void coexistenceOfDbbAndNativePaths() throws Exception { - start(); - ThreadContext ctx = profiler.getThreadContext(); - - profiler.setContext(0xA0L, 0xA1L, 0L, 0xA1L); // DBB - assertEquals(0xA1L, ctx.getSpanId()); - - profiler.setTraceContext(0xB0L, 0xB1L, 0L, 0xB1L, -1, null, -1, null); // native - assertEquals(0xB1L, ctx.getSpanId()); - - ctx.setContextAttribute(SLOT_OP, "dbb-value"); // DBB attribute - assertEquals("dbb-value", ctx.readContextAttribute(SLOT_OP)); - - assertTrue(profiler.setContextValue(SLOT_RES, "native-value")); // native attribute - assertEquals("native-value", ctx.readContextAttribute(SLOT_RES)); - assertEquals("dbb-value", ctx.readContextAttribute(SLOT_OP), "DBB-written attr still intact"); + assertEquals("ok", testReadContextAttribute(SLOT_OP)); } /** @@ -265,18 +220,17 @@ public void coexistenceOfDbbAndNativePaths() throws Exception { @Test public void setTraceContextRejectsZeroSpanId() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x9L, 0x7L, 0L, 0x7L, -1, null, -1, null); // active span - assertEquals(0x7L, ctx.getSpanId()); + assertEquals(0x7L, testGetSpanId()); assertThrows(IllegalArgumentException.class, () -> profiler.setTraceContext(0x2L, 0L, 0L, 0L, -1, null, -1, null), "spanId=0 must be rejected"); // The rejected call is a no-op: it neither clears nor corrupts the still-active span. - assertEquals(1, readValidByte(ctx), "record still valid after rejected call"); - assertEquals(0x7L, ctx.getSpanId(), "previously active span left intact"); + assertTrue(testIsContextValid(), "record still valid after rejected call"); + assertEquals(0x7L, testGetSpanId(), "previously active span left intact"); } /** @@ -287,17 +241,16 @@ public void setTraceContextRejectsZeroSpanId() throws Exception { @Test public void clearContextValuePreservesInvalidState() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, SLOT_OP, "op", -1, null); // active span - assertEquals(1, readValidByte(ctx), "precondition: active record is valid"); + assertTrue(testIsContextValid(), "precondition: active record is valid"); profiler.clearTraceContext(); - assertEquals(0, readValidByte(ctx), "clearTraceContext leaves the record deactivated"); + assertFalse(testIsContextValid(), "clearTraceContext leaves the record deactivated"); // Clearing an attribute on a deactivated record must not flip valid back to 1. profiler.clearContextValue(SLOT_OP); - assertEquals(0, readValidByte(ctx), + assertFalse(testIsContextValid(), "clearContextValue must preserve valid=0; a deactivated record must stay deactivated"); } @@ -311,27 +264,25 @@ public void clearContextValuePreservesInvalidState() throws Exception { @Test public void setContextValuePublishesAppContextWithoutSpan() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // active span - assertEquals(1, readValidByte(ctx), "precondition: active record is valid"); + assertTrue(testIsContextValid(), "precondition: active record is valid"); profiler.clearTraceContext(); - assertEquals(0, readValidByte(ctx), "clearTraceContext leaves the record deactivated"); + assertFalse(testIsContextValid(), "clearTraceContext leaves the record deactivated"); // Setting an attribute on a deactivated record republishes it so the attribute is visible // with no active span (span/trace stay zero, but the record is valid and carries the value). profiler.setContextValue(SLOT_OP, "late"); - assertEquals(1, readValidByte(ctx), + assertTrue(testIsContextValid(), "setContextValue must publish (valid=1) so app context is visible without a span"); - assertEquals(0, ctx.getSpanId(), "span stays zero — this is app context, not a span"); - assertEquals("late", ctx.readContextAttribute(SLOT_OP), "attribute observable without a span"); + assertEquals(0, testGetSpanId(), "span stays zero — this is app context, not a span"); + assertEquals("late", testReadContextAttribute(SLOT_OP), "attribute observable without a span"); } /** * copyContextTags reads the sidecar tag encodings written through the all-native path directly - * from the record — without going through ThreadContext (which would reset the record). The test - * never calls getThreadContext, proving the native read observes native writes. + * from the record. */ @Test public void copyContextTagsReadsNativeEncodings() throws Exception { @@ -360,13 +311,12 @@ public void copyContextTagsReadsNativeEncodings() throws Exception { @Test public void nativeNullValueByteArrayIsTreatedAsEmpty() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span // encoding 0 is benign; the point is that a null byte[] does not crash and yields an empty // attribute rather than a dereference. assertTrue(setContextValue0(SLOT_OP, 0, null), "null value byte[] accepted as empty"); - assertEquals("", ctx.readContextAttribute(SLOT_OP), "null value stored as empty attribute"); + assertEquals("", testReadContextAttribute(SLOT_OP), "null value stored as empty attribute"); } /** @@ -377,19 +327,18 @@ public void nativeNullValueByteArrayIsTreatedAsEmpty() throws Exception { @Test public void zeroLengthAttributeCompactsCleanly() throws Exception { start(); - ThreadContext ctx = profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span assertTrue(profiler.setContextValue(SLOT_OP, ""), "empty value written"); - assertEquals("", ctx.readContextAttribute(SLOT_OP), "empty attribute observable"); + assertEquals("", testReadContextAttribute(SLOT_OP), "empty attribute observable"); // Overwrite the zero-length entry: compaction must walk over the 2-byte entry correctly. assertTrue(profiler.setContextValue(SLOT_OP, "now-non-empty")); - assertEquals("now-non-empty", ctx.readContextAttribute(SLOT_OP)); + assertEquals("now-non-empty", testReadContextAttribute(SLOT_OP)); // Clearing it back out also compacts across the (now larger) entry without corruption. profiler.clearContextValue(SLOT_OP); - assertNull(ctx.readContextAttribute(SLOT_OP), "attribute cleared"); + assertNull(testReadContextAttribute(SLOT_OP), "attribute cleared"); } /** @@ -401,7 +350,6 @@ public void zeroLengthAttributeCompactsCleanly() throws Exception { @Test public void slotBoundaryIsRejected() throws Exception { start(); - profiler.getThreadContext(); profiler.setTraceContext(0x2L, 0x1L, 0L, 0x1L, -1, null, -1, null); // live span final int capacity = 10; // native DD_TAGS_CAPACITY; drift caught by MaxContextSlotsTest @@ -448,15 +396,14 @@ public void nativeWritesFromVirtualThreadsAreCoherent() throws Exception { final long root = 0x900000L + i; startVirtualThread(() -> { try { - // Resolve the read handle first (may init/reset the carrier record), then write - // and read back with no yield point in between. - ThreadContext ctx = profiler.getThreadContext(); + // Write and read back with no yield point in between, so this vthread stays + // mounted on one carrier for the whole write+read. profiler.setTraceContext(root, span, 0L, span, SLOT_OP, "vt", -1, null); - long got = ctx.getSpanId(); + long got = testGetSpanId(); if (got != span) { failures.add("expected span " + span + " but read " + got); } - if (!"vt".equals(ctx.readContextAttribute(SLOT_OP))) { + if (!"vt".equals(testReadContextAttribute(SLOT_OP))) { failures.add("attribute mismatch for span " + span); } } catch (Throwable t) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java deleted file mode 100644 index f0ac047d8f..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CarrierContextStorageTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.ContextStorageMode; -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ThreadContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Verifies that {@link ThreadContext} storage is scoped to the carrier thread when - * carrier scoping is active ({@link com.datadoghq.profiler.ContextStorageMode#CARRIER}). - * - *

The OTEP record a {@code ThreadContext} writes to is embedded in the carrier's native - * {@code ProfiledThread} and is what the (carrier-bound) sampler reads. Under carrier scoping, - * every virtual thread mounted on a given carrier must resolve to the same - * {@code ThreadContext} — the one whose buffer targets that carrier's live record — regardless - * of how many virtual threads time-share the carrier. Under the legacy plain-{@code ThreadLocal} - * behavior each virtual thread gets its own, pinned to whatever carrier it first ran on. - * - *

Requires JDK 21+ (virtual threads) and {@code jdk.internal.misc.CarrierThreadLocal} being - * accessible (the build adds {@code --add-exports java.base/jdk.internal.misc=ALL-UNNAMED} on - * 21+). When carrier scoping is not active the test is skipped rather than failing, so it never - * gives false confidence on JDKs/configs where the fix cannot engage. - */ -public class CarrierContextStorageTest { - - private static JavaProfiler profiler; - - @BeforeAll - public static void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - /** {@code Thread.ofVirtual().start(task)} via reflection so this compiles with --release 8. */ - private static Thread startVirtualThread(Runnable task) throws Exception { - Method ofVirtual = Thread.class.getMethod("ofVirtual"); - Object builder = ofVirtual.invoke(null); - Class builderInterface = Class.forName("java.lang.Thread$Builder"); - Method start = builderInterface.getMethod("start", Runnable.class); - return (Thread) start.invoke(builder, task); - } - - /** Extracts the carrier name from a mounted VirtualThread's toString, e.g. {@code ...@ForkJoinPool-1-worker-2}. */ - private static String carrierOf(Thread current) { - String s = current.toString(); - int at = s.lastIndexOf('@'); - return at >= 0 ? s.substring(at + 1) : ""; - } - - @Test - public void contextIsSharedPerCarrierAcrossVirtualThreads() throws Exception { - assumeTrue(Platform.isJavaVersionAtLeast(21), "virtual threads require JDK 21+"); - assumeTrue(ContextStorageMode.CARRIER == profiler.contextStorageMode(), - "carrier-scoped storage not active (mode=" + profiler.contextStorageMode() - + "); needs JDK 21+ and --add-exports java.base/jdk.internal.misc=ALL-UNNAMED"); - - // Map each observed carrier to the set of distinct ThreadContext identities seen on it. - final Map> carrierToContexts = new ConcurrentHashMap<>(); - final int nThreads = 2000; - final CountDownLatch done = new CountDownLatch(nThreads); - - for (int i = 0; i < nThreads; i++) { - startVirtualThread(() -> { - try { - // Resolve the context on this vthread and record its identity against the - // carrier it is currently mounted on. With far more vthreads (2000) than - // carriers, many vthreads time-share each carrier, so keying by the virtual - // thread would produce ~one context per vthread while carrier scoping produces - // ~one per carrier. - ThreadContext c1 = profiler.getThreadContext(); - String carrier = carrierOf(Thread.currentThread()); - carrierToContexts - .computeIfAbsent(carrier, k -> ConcurrentHashMap.newKeySet()) - .add(System.identityHashCode(c1)); - } finally { - done.countDown(); - } - }); - } - - assertTrue(done.await(60, TimeUnit.SECONDS), "virtual threads did not finish in time"); - - int carriers = carrierToContexts.size(); - Set allContexts = ConcurrentHashMap.newKeySet(); - for (Map.Entry> e : carrierToContexts.entrySet()) { - // Normally every vthread that ran on this carrier saw the SAME ThreadContext, so the - // expected count is 1. We assert >= 1 rather than == 1 because the key is the carrier - // *name* (e.g. ForkJoinPool-1-worker-3): if the pool retires a worker and creates a - // replacement for the same slot mid-run, a second (equally valid) context can appear - // under one name. The did-NOT-key-by-vthread guarantee is enforced by the aggregate - // bound below, which does not depend on name stability. - assertTrue(e.getValue().size() >= 1, - "carrier " + e.getKey() + " must expose at least one ThreadContext"); - allContexts.addAll(e.getValue()); - } - - // The crux: far fewer distinct contexts than virtual threads — i.e. storage did NOT key by - // the virtual thread. Roughly one context per carrier (allowing for occasional carrier-name - // reuse, so allContexts may slightly exceed the carrier count). - assertTrue(allContexts.size() >= carriers, - "expected at least one ThreadContext per carrier"); - assertTrue(carriers > 0 && carriers < nThreads && allContexts.size() < nThreads, - "expected carrier count (" + carriers + ") and context count (" + allContexts.size() - + ") to be well below vthread count (" + nThreads + ")"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java new file mode 100644 index 0000000000..0a5b43c8f8 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/context/CustomContextAttributeSamplingTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler.context; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.ContextSetter; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junitpioneer.jupiter.RetryingTest; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end coverage for the custom-attribute {@code setContextValue}/{@code ContextSetter} -> + * JFR field pipeline, on the all-native context write path. + * + *

{@link com.datadoghq.profiler.ContextValueCacheTest} covers the value-cache unit in + * isolation, and {@link AllNativeContextSamplingTest} covers span/root-span propagation into JFR, + * but neither confirms that a value written via {@code setContextValue} actually surfaces as a + * correctly-named, correctly-weighted JFR field. This closes that gap. + */ +public class CustomContextAttributeSamplingTest extends AbstractProfilerTest { + + @BeforeEach + void assumeNotJ9() { + // On J9, ProfiledThread (and thus the OTEP TLS buffer) is not allocated until the thread + // is registered for wall-clock profiling, so a fresh context write throws. + Assumptions.assumeTrue(!Platform.isJ9()); + } + + @Override + protected String getProfilerCommand() { + return "wall=1ms,filter=0,attributes=tag1;tag2;tag3"; + } + + @RetryingTest(10) + public void customAttributeValueSurfacesInJfr() throws InterruptedException { + registerCurrentThreadForWallClockProfiling(); + ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag3")); + int slot = contextSetter.offsetOf("tag1"); + + // Session-unique prefix so each @RetryingTest attempt registers fresh values in the + // native Dictionary, matching dictionary_context_keys to this run's writes only. + String pfx = Long.toHexString(System.nanoTime()) + "_"; + String[] values = IntStream.range(0, 10).mapToObj(i -> pfx + i).toArray(String[]::new); + for (int i = 0; i < values.length * 10; i++) { + String value = values[i % values.length]; + assertTrue(profiler.setContextValue(slot, value)); + Thread.sleep(10); + profiler.clearContextValue(slot); + } + stopProfiler(); + + IItemCollection events = verifyEvents("datadog.MethodSample"); + Map weightsByTagValue = new HashMap<>(); + for (IItemIterable samples : events) { + IMemberAccessor weightAccessor = WEIGHT.getAccessor(samples.getType()); + IMemberAccessor tag1Accessor = TAG_1.getAccessor(samples.getType()); + IMemberAccessor tag2Accessor = TAG_2.getAccessor(samples.getType()); + if (tag1Accessor == null || tag2Accessor == null) { + continue; + } + IMemberAccessor stacktraceAccessor = + JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); + for (IItem sample : samples) { + String stacktrace = stacktraceAccessor.getMember(sample); + if (!stacktrace.contains("sleep") || stacktrace.contains("")) { + // Only count samples taken while the context was definitely active. + continue; + } + String tag = tag1Accessor.getMember(sample); + if (tag == null) { + continue; + } + weightsByTagValue.computeIfAbsent(tag, v -> new AtomicLong()) + .addAndGet(weightAccessor.getMember(sample).longValue()); + assertNull(tag2Accessor.getMember(sample), "tag2 was never set for this thread"); + } + } + for (String value : values) { + assertNotNull(weightsByTagValue.get(value), + "no sample carried tag1=" + value + ". Found: " + weightsByTagValue.keySet()); + } + + // jdk.ActiveSetting must enumerate the configured attribute names, unbundling the + // dynamic-column config into the recording. + IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); + Set recordedContextAttributes = new HashSet<>(); + for (IItemIterable activeSetting : activeSettings) { + IMemberAccessor nameAccessor = + JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); + IMemberAccessor valueAccessor = + JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); + for (IItem item : activeSetting) { + if ("contextattribute".equals(nameAccessor.getMember(item))) { + recordedContextAttributes.add(valueAccessor.getMember(item)); + } + } + } + assertEquals(3, recordedContextAttributes.size()); + assertTrue(recordedContextAttributes.contains("tag1")); + assertTrue(recordedContextAttributes.contains("tag2")); + assertTrue(recordedContextAttributes.contains("tag3")); + + // dictionary_context_keys must match the number of distinct values registered above. + Map jfrCounters = new HashMap<>(); + for (IItemIterable counterEvent : verifyEvents("datadog.ProfilerCounter")) { + IMemberAccessor nameAccessor = NAME.getAccessor(counterEvent.getType()); + IMemberAccessor countAccessor = COUNT.getAccessor(counterEvent.getType()); + for (IItem item : counterEvent) { + jfrCounters.put(nameAccessor.getMember(item), countAccessor.getMember(item).longValue()); + } + } + assertFalse(jfrCounters.isEmpty()); + assertEquals(values.length, jfrCounters.get("dictionary_context_keys")); + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java deleted file mode 100644 index 8e5f4f6ac8..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Tests for OTEL-compatible context storage (OTEP #4947). - */ -public class OtelContextStorageModeTest { - - private static JavaProfiler profiler; - private boolean profilerStarted = false; - - @BeforeAll - public static void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - @AfterEach - public void cleanup() { - if (profilerStarted) { - profiler.stop(); - profiler.resetThreadContext(); - profilerStarted = false; - } - } - - /** - * Tests that context round-trips correctly. - */ - @Test - public void testOtelStorageModeContext() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-otel", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=tag1;tag2;tag3,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - long traceIdHigh = 0x5555666677778888L; - long traceIdLow = 0x9999AAAABBBBCCCCL; - profiler.setContext(localRootSpanId, spanId, traceIdHigh, traceIdLow); - - ThreadContext ctx = profiler.getThreadContext(); - assertEquals(spanId, ctx.getSpanId(), "SpanId should match"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match"); - // Verify the 128-bit trace ID round-trips through the OTEP record (big-endian) - assertEquals("55556666777788889999aaaabbbbcccc", ctx.readTraceId(), "TraceId should match"); - } - - - /** - * Tests that custom attributes are correctly written to and read back from - * the OTEP record's attrs_data (via DirectByteBuffer). Verifies the - * sidecar encoding is set and the UTF-8 value appears in attrs_data. - */ - @Test - public void testOtelModeCustomAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-attrs", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=http.route;db.system,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - profiler.setContext(localRootSpanId, spanId, 0L, 0x9999L); - - ThreadContext ctx = profiler.getThreadContext(); - boolean result = ctx.setContextAttribute(0, "GET /api/users"); - assertTrue(result, "setContextAttribute should succeed"); - - result = ctx.setContextAttribute(1, "postgresql"); - assertTrue(result, "setContextAttribute for second key should succeed"); - - // Verify attribute values round-trip correctly through attrs_data - assertEquals("GET /api/users", ctx.readContextAttribute(0), "http.route should round-trip"); - assertEquals("postgresql", ctx.readContextAttribute(1), "db.system should round-trip"); - - // Verify trace context is still intact after attribute writes - assertEquals(spanId, ctx.getSpanId(), "SpanId should match after setAttribute"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match after setAttribute"); - } - - /** - * Tests that attrs_data overflow is handled gracefully (returns false, no crash). - */ - @Test - public void testOtelModeAttributeOverflow() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-overflow", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=k0;k1;k2;k3;k4,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - profiler.setContext(0x2L, 0x1L, 0L, 0x3L); - - ThreadContext ctx = profiler.getThreadContext(); - - // LRS is a fixed 18-byte entry (key=0, len=16, 16 hex value bytes). - // Available for custom attrs: 612 - 18 = 594 bytes. - // Each 255-char attr = 257 bytes. Two fit (514 ≤ 594); third overflows (771 > 594). - StringBuilder sb = new StringBuilder(255); - for (int i = 0; i < 255; i++) sb.append('x'); - String longValue = sb.toString(); - assertTrue(ctx.setContextAttribute(0, longValue), "First long attr should fit"); - assertTrue(ctx.setContextAttribute(1, longValue), "Second long attr should fit"); - assertFalse(ctx.setContextAttribute(2, longValue), "Third long attr should overflow"); - - // Short values should still work for remaining slots - assertTrue(ctx.setContextAttribute(3, "short"), "Short attr after overflow should work"); - } - - /** - * Tests sequential context updates including boundary values and clearing. - * Verifies that each setContext overwrites the previous values, that MAX_VALUE - * round-trips correctly, and that clearContext resets both IDs to zero. - */ - @Test - public void testSequentialContextUpdates() { - profiler.setContext(2L, 1L, 0, 1L); - assertEquals(1L, profiler.getThreadContext().getSpanId()); - assertEquals(2L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(20L, 10L, 0, 10L); - assertEquals(10L, profiler.getThreadContext().getSpanId()); - assertEquals(20L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(200L, 100L, 0, 100L); - assertEquals(100L, profiler.getThreadContext().getSpanId()); - assertEquals(200L, profiler.getThreadContext().getRootSpanId()); - - long maxValue = Long.MAX_VALUE; - profiler.setContext(maxValue, maxValue, 0, maxValue); - assertEquals(maxValue, profiler.getThreadContext().getSpanId(), "SpanId should be MAX_VALUE"); - assertEquals(maxValue, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be MAX_VALUE"); - - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "SpanId should be zero after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be zero after clear"); - } - - @Test - public void testThreadIsolation() throws InterruptedException { - long threadASpanId = 1000L; - long threadARootSpanId = 1001L; - profiler.setContext(threadARootSpanId, threadASpanId, 0, threadASpanId); - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - - final long threadBSpanId = 2000L; - final long threadBRootSpanId = 2001L; - final AssertionError[] threadBError = {null}; - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(threadBRootSpanId, threadBSpanId, 0, threadBSpanId); - assertEquals(threadBSpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadBRootSpanId, profiler.getThreadContext().getRootSpanId()); - } catch (AssertionError e) { - threadBError[0] = e; - } - }, "TestThread-B"); - - threadB.start(); - threadB.join(); - - if (threadBError[0] != null) throw threadBError[0]; - - // Thread A's context must be unaffected - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - } - - /** - * Tests that a direct span-to-span transition (no clearContext in between) - * does not leak custom attributes from the previous span. - */ - @Test - public void testSpanTransitionClearsAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-transition", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=http.route,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - // Span A: set a custom attribute - profiler.setContext(0x1L, 0x1L, 0L, 0x1L); - ThreadContext ctx = profiler.getThreadContext(); - ctx.setContextAttribute(0, "/api/spanA"); - - // Transition directly to span B without clearing - profiler.setContext(0x2L, 0x2L, 0L, 0x2L); - - // Span A's attribute must not be visible in span B's context - assertNull(ctx.readContextAttribute(0), "Custom attribute must be cleared on span transition"); - } - - /** - * Stress-tests the OTEP context path with many sequential writes to catch - * buffer corruption or stale-value leaks over repeated updates. - */ - @Test - public void testRepeatedContextWrites() { - for (int i = 1; i <= 1000; i++) { - long spanId = (long) i; - long rootSpanId = (long) (i + 10000); - profiler.setContext(rootSpanId, spanId, 0L, spanId); - assertEquals(spanId, profiler.getThreadContext().getSpanId(), - "spanId mismatch at iteration " + i); - assertEquals(rootSpanId, profiler.getThreadContext().getRootSpanId(), - "rootSpanId mismatch at iteration " + i); - } - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "spanId should be 0 after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "rootSpanId should be 0 after clear"); - } - - /** - * Tests that the per-thread attribute cache isolates threads correctly. - * "FB" and "Ea" have equal hashCode() (both 2236), so they map to the same - * cache slot. With per-thread caches each thread owns its slot independently. - */ - @Test - public void testAttributeCacheIsolation() throws Exception { - Path jfrFile = Files.createTempFile("otel-attr-cache-iso", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=attr0,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - final String valueA = "FB"; // hashCode = 2236, slot 188 - final String valueB = "Ea"; // hashCode = 2236, same slot - final AssertionError[] errors = {null, null}; - final CountDownLatch bothWritten = new CountDownLatch(2); - - Thread threadA = new Thread(() -> { - try { - profiler.setContext(1L, 1L, 0L, 1L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueA)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - // After thread B has written "Ea" to its own slot, A must still read "FB" - assertEquals(valueA, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[0] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheA"); - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(2L, 2L, 0L, 2L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueB)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - assertEquals(valueB, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[1] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheB"); - - threadA.start(); - threadB.start(); - threadA.join(10_000); - threadB.join(10_000); - - if (errors[0] != null) throw errors[0]; - if (errors[1] != null) throw errors[1]; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java deleted file mode 100644 index 95c8fecd50..0000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java +++ /dev/null @@ -1,641 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.IntStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.ContextSetter; -import static com.datadoghq.profiler.MoreAssertions.DICTIONARY_PAGE_SIZE; -import static com.datadoghq.profiler.MoreAssertions.assertBoundedBy; -import com.datadoghq.profiler.Platform; - -public class TagContextTest extends AbstractProfilerTest { - - @BeforeEach - void assumeNotJ9() { - // On J9, ProfiledThread (and thus the OTEP TLS buffer) is not allocated until the thread - // is registered for wall-clock profiling, so initializeContextTLS0() returns null and - // ThreadContext creation throws. These tests require a live ThreadContext from the start. - Assumptions.assumeTrue(!Platform.isJ9()); - } - - @RetryingTest(10) - public void test() throws InterruptedException { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag1")); - - // Use session-unique prefix so each @RetryingTest attempt registers fresh values in the - // native Dictionary. Without this, on musl (no JVM fork) the per-thread attrCacheKeys - // persists across retries: cache hits skip registerConstant0(), leaving - // dictionary_context_keys=0 on every retry after the first. - String pfx = Long.toHexString(System.nanoTime()) + "_"; - String[] strings = IntStream.range(0, 10).mapToObj(i -> pfx + i).toArray(String[]::new); - for (int i = 0; i < strings.length * 10; i++) { - work(contextSetter, "tag1", strings[i % strings.length]); - } - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - Map weightsByTagValue = new HashMap<>(); - long droppedSamplesCount = 0; - long droppedSamplesWeight = 0; - long totalSamplesCount = 0; - long totalSamplesWeight = 0; - try { - for (IItemIterable wallclockSamples : events) { - IMemberAccessor weightAccessor = WEIGHT.getAccessor(wallclockSamples.getType()); - // this will become more generic in the future - IMemberAccessor tag1Accessor = TAG_1.getAccessor(wallclockSamples.getType()); - assertNotNull(tag1Accessor); - IMemberAccessor tag2Accessor = TAG_2.getAccessor(wallclockSamples.getType()); - assertNotNull(tag2Accessor); - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String stacktrace = stacktraceAccessor.getMember(sample); - if (!stacktrace.contains("sleep")) { - // we don't know the context has been set for sure until the sleep has started - continue; - } - - long weight = weightAccessor.getMember(sample).longValue(); - totalSamplesCount++; - totalSamplesWeight += weight; - - if (stacktrace.contains("")) { - // track dropped samples statistics but skip for weight distribution calculation - droppedSamplesCount++; - droppedSamplesWeight += weight; - continue; - } - - String tag = tag1Accessor.getMember(sample); - weightsByTagValue.computeIfAbsent(tag, v -> new AtomicLong()) - .addAndGet(weight); - assertNull(tag2Accessor.getMember(sample)); - } - } - long sum = 0; - long[] weights = new long[strings.length]; - System.out.println("Found tag values: " + weightsByTagValue.keySet()); - for (int i = 0; i < strings.length; i++) { - AtomicLong weight = weightsByTagValue.get(strings[i]); - assertNotNull(weight, "Weight for " + strings[i] + " not found. Found: " + weightsByTagValue.keySet()); - weights[i] = weightsByTagValue.get(strings[i]).get(); - sum += weights[i]; - } - double avg = (double) sum / weights.length; - for (int i = 0; i < weights.length; i++) { - assertTrue(Math.abs(weights[i] - avg) < 0.15 * weights[i], strings[i] - + " more than 15% from mean"); - } - - // now check we have settings to unbundle the dynamic columns - IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); - Set recordedContextAttributes = new HashSet<>(); - for (IItemIterable activeSetting : activeSettings) { - IMemberAccessor nameAccessor = JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); - IMemberAccessor valueAccessor = JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); - for (IItem item : activeSetting) { - String name = nameAccessor.getMember(item); - if ("contextattribute".equals(name)) { - recordedContextAttributes.add(valueAccessor.getMember(item)); - } - } - } - assertEquals(3, recordedContextAttributes.size()); - assertTrue(recordedContextAttributes.contains("tag1")); - assertTrue(recordedContextAttributes.contains("tag2")); - assertTrue(recordedContextAttributes.contains("tag3")); - - // Verify counters from JFR serialized data (not live process counters which are reset) - Map jfrCounters = new HashMap<>(); - for (IItemIterable counterEvent : verifyEvents("datadog.ProfilerCounter")) { - IMemberAccessor nameAccessor = NAME.getAccessor(counterEvent.getType()); - IMemberAccessor countAccessor = COUNT.getAccessor(counterEvent.getType()); - for (IItem item : counterEvent) { - String name = nameAccessor.getMember(item); - jfrCounters.put(name, countAccessor.getMember(item).longValue()); - } - } - - assertFalse(jfrCounters.isEmpty()); - assertEquals(strings.length, jfrCounters.get("dictionary_context_keys")); - } finally { - // Print statistics about dropped samples for debugging - double dropRate = totalSamplesCount > 0 ? (100.0 * droppedSamplesCount / totalSamplesCount) : 0.0; - double dropWeightRate = totalSamplesWeight > 0 ? (100.0 * droppedSamplesWeight / totalSamplesWeight) : 0.0; - System.out.printf("Sample statistics: %d total (%d dropped, %.2f%%), weight %d total (%d dropped, %.2f%%)%n", - totalSamplesCount, droppedSamplesCount, dropRate, - totalSamplesWeight, droppedSamplesWeight, dropWeightRate); - } - } - - /** - * Reads the current value of {@code tag} via {@link ThreadContext#readContextAttribute} - * — the only readback path retained on the Java side (test-only). - */ - private String readTag(ContextSetter contextSetter, String tag) { - return profiler.getThreadContext().readContextAttribute(contextSetter.offsetOf(tag)); - } - - @Test - public void testSnapshotRestore() throws Exception { - // J9 does not initialize ThreadContext for non-profiled threads; skip. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - // Initially both slots are empty - assertNull(readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - - // Set a value and read it back - assertTrue(contextSetter.setContextValue("tag1", "before")); - assertEquals("before", readTag(contextSetter, "tag1")); - - // Snapshot the string, overwrite, then restore - String saved = readTag(contextSetter, "tag1"); - assertTrue(contextSetter.setContextValue("tag1", "inside")); - assertEquals("inside", readTag(contextSetter, "tag1")); - - // Restore via setContextValue - assertTrue(contextSetter.setContextValue("tag1", saved)); - assertEquals("before", readTag(contextSetter, "tag1")); - - // put/clear/put cycle: verify offset stability across state transitions - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertTrue(contextSetter.setContextValue("tag1", "after")); - assertEquals("after", readTag(contextSetter, "tag1")); - - // tag2 was never set; readContextAttribute returns null - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testAttrsDataOverflow() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - char[] chars = new char[255]; - java.util.Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - int overflowIndex = -1; - for (int i = 1; i <= 10; i++) { - if (!contextSetter.setContextValue("tag" + i, bigValue)) { - overflowIndex = i; - break; - } - } - assertTrue(overflowIndex >= 0, "Expected at least one write to overflow attrs_data"); - assertNull(readTag(contextSetter, "tag" + overflowIndex), - "Overflowed slot must read null — the entry never landed in attrs_data"); - } - - @Test - public void testPutClearsCustomSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "before-put")); - assertEquals("before-put", readTag(contextSetter, "tag1")); - - // setContext() triggers setContextDirect which resets attrs_data_size to the LRS entry only, - // dropping all user attribute entries — so scanning attrs_data for tag1 returns null. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "tag1 must be null after setContext resets attrs_data"); - } - - @Test - public void testCrossSlotIsolation() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - assertTrue(contextSetter.clearContextValue("tag2")); - assertEquals("v1", readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testReapplyByIdAndBytes() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - String value = "app-managed"; - - // Set the attribute the normal way, then capture both the constant ID (sidecar) and the - // UTF-8 bytes — exactly what dd-trace-java retains for the reapply hot path. - assertTrue(contextSetter.setContextValue("tag1", value)); - int[] ids = contextSetter.snapshotTags(); - int savedId = ids[slot]; - assertNotEquals(0, savedId); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = value.getBytes(StandardCharsets.UTF_8); - - // setContext (span activation) wipes both views. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "attrs_data must be wiped by setContext"); - assertEquals(0, contextSetter.snapshotTags()[slot], "sidecar must be wiped by setContext"); - - // Reapply by ID + bytes restores BOTH views. - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals(value, readTag(contextSetter, "tag1"), "attrs_data must be restored"); - assertEquals(savedId, contextSetter.snapshotTags()[slot], "sidecar must be restored"); - } - - @Test - public void testReapplyByIdAndBytesRejectsBadArgs() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "v")); - int id = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id); - - // Null arrays and length mismatch must throw. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(null, new byte[1][])); - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[1], null)); - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[2], new byte[3][])); - - // A slot with constantId > 0 requires non-null bytes within the size limit. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {null, null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after NPE on active utf8[i]"); - - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {new byte[256], null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after IAE on oversized utf8[i]"); - - // 255 bytes is the boundary and must be accepted. - // Register the 255-byte value so its constant ID matches the bytes we pass. - byte[] ok255 = new byte[255]; - Arrays.fill(ok255, (byte) 'x'); - assertTrue(contextSetter.setContextValue("tag1", new String(ok255, StandardCharsets.UTF_8))); - int id255 = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id255); - assertTrue(contextSetter.setContextValuesByIdAndBytes( - new int[] {id255, 0}, new byte[][] {ok255, null})); - } - - @Test - public void testReapplyByIdAndBytesReplacesExistingValue() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Capture the ID + bytes for "first". - assertTrue(contextSetter.setContextValue("tag1", "first")); - int[] idsFirst = contextSetter.snapshotTags(); - byte[][] bytesFirst = new byte[idsFirst.length][]; - bytesFirst[slot] = "first".getBytes(StandardCharsets.UTF_8); - - // Overwrite the live slot with a different value. - assertTrue(contextSetter.setContextValue("tag1", "second")); - assertEquals("second", readTag(contextSetter, "tag1")); - - // Reapply "first" by ID + bytes over the live "second" — exercises the - // compact-then-insert path in replaceOtepAttribute. - assertTrue(contextSetter.setContextValuesByIdAndBytes(idsFirst, bytesFirst)); - assertEquals("first", readTag(contextSetter, "tag1")); - assertEquals(idsFirst[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesAfterClear() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "live")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "live".getBytes(StandardCharsets.UTF_8); - - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertEquals(0, contextSetter.snapshotTags()[slot]); - - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals("live", readTag(contextSetter, "tag1")); - assertEquals(ids[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesClearedRecord() throws Exception { - // Verifies that setContextValuesByIdAndBytes never resurrects a cleared (span-less) record. - // A cleared record has valid=0 and no trace/span context; re-publishing it would expose - // attribute values with no associated trace, which is meaningless to the signal handler. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a live record with tag1 set. - profiler.setContext(1L, 42L, 0L, 43L); - assertTrue(contextSetter.setContextValue("tag1", "will-be-cleared")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "will-be-cleared".getBytes(StandardCharsets.UTF_8); - - // Drive valid=0 via the all-zero clear path (clearContext → put(0,0,0,0) → no attach()). - profiler.clearContext(); - // readContextAttribute respects valid=0 and returns null, confirming the record is dark. - assertNull(readTag(contextSetter, "tag1")); - - // Reapply must return false and must not resurrect the cleared record. - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "setContextValuesByIdAndBytes must return false when the record is cleared (valid=0)"); - assertNull(readTag(contextSetter, "tag1"), - "cleared record must not be resurrected by setContextValuesByIdAndBytes"); - } - - @Test - public void testReapplyByIdAndBytesOverflowRollback() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Register one 255-byte value to obtain a valid constant ID. - char[] chars = new char[255]; - Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - assertTrue(contextSetter.setContextValue("tag1", bigValue)); - int bigId = contextSetter.snapshotTags()[contextSetter.offsetOf("tag1")]; - assertNotEquals(0, bigId); - byte[] bigBytes = bigValue.getBytes(StandardCharsets.UTF_8); - - // Reapply the same 255-byte value to all 10 slots — attrs_data cannot hold them all. - int[] ids = new int[10]; - byte[][] bytes = new byte[10][]; - Arrays.fill(ids, bigId); - Arrays.fill(bytes, bigBytes); - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "10 x 255-byte values must overflow attrs_data"); - - // The last slot certainly overflowed: its sidecar must be zeroed and attrs_data empty. - int lastSlot = contextSetter.offsetOf("tag10"); - assertEquals(0, contextSetter.snapshotTags()[lastSlot], - "overflowed slot's sidecar must be zeroed"); - assertNull(readTag(contextSetter, "tag10"), - "overflowed slot must read null — the entry never landed in attrs_data"); - - // Slots processed before the overflow are durably written — false does not mean - // the record is unchanged. At least tag1 (slot 0) must retain the new value. - int firstSlot = contextSetter.offsetOf("tag1"); - assertEquals(bigId, contextSetter.snapshotTags()[firstSlot], - "slot 0 processed before overflow must have its sidecar durably written"); - assertEquals(bigValue, readTag(contextSetter, "tag1"), - "slot 0 processed before overflow must be readable via attrs_data"); - } - - // ----------------------------------------------------------------------- - // Acceptance tests for the MAX_CUSTOM_SLOTS guard fixes - // ----------------------------------------------------------------------- - - /** - * Test 1: setContextValuesByIdAndBytes must throw IllegalArgumentException immediately when - * the arrays are longer than MAX_CUSTOM_SLOTS (10), and must not perform - * any partial write before the rejection. - */ - @Test - public void testSetContextValuesByIdAndBytesRejectsArraysLongerThanMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a known value and capture its constant ID. - assertTrue(contextSetter.setContextValue("tag1", "original")); - int savedId = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, savedId); - - // Build arrays of length 11 (> MAX_CUSTOM_SLOTS = 10). - int[] ids = new int[11]; - byte[][] utf8 = new byte[11][]; - ids[0] = savedId; - utf8[0] = "original".getBytes(StandardCharsets.UTF_8); - // All other entries remain 0 / null. - - // The call must be rejected with an exception before any write. - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(ids, utf8), - "setContextValuesByIdAndBytes must throw when array length > MAX_CUSTOM_SLOTS"); - - // No partial write: the sidecar for slot 0 must be unchanged. - assertEquals(savedId, contextSetter.snapshotTags()[slot], - "sidecar must not be modified before the length guard fires"); - } - - /** - * Test 2: setContextValuesByIdAndBytes must accept arrays of exactly - * MAX_CUSTOM_SLOTS (10) and return true, restoring all sidecar values. - */ - @Test - public void testSetContextValuesByIdAndBytesAcceptsExactlyMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Set all 10 attributes to distinct values and capture constant IDs + bytes. - int[] savedIds = new int[10]; - byte[][] savedBytes = new byte[10][]; - for (int i = 0; i < 10; i++) { - String value = "val" + i; - assertTrue(contextSetter.setContextValue("tag" + (i + 1), value)); - savedBytes[i] = value.getBytes(StandardCharsets.UTF_8); - } - int[] snapshot = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - savedIds[i] = snapshot[i]; - assertNotEquals(0, savedIds[i], "tag" + (i + 1) + " must have a non-zero sidecar ID"); - } - - // Wipe all slots via setContext (span activation). - profiler.setContext(1L, 42L, 0L, 43L); - - // Reapply with exactly-10-element arrays — must succeed. - assertTrue(contextSetter.setContextValuesByIdAndBytes(savedIds, savedBytes), - "setContextValuesByIdAndBytes must return true for arrays of length == MAX_CUSTOM_SLOTS"); - - // All 10 sidecar IDs must be restored. - int[] restored = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - assertEquals(savedIds[i], restored[i], - "sidecar for tag" + (i + 1) + " must be restored after reapply"); - } - } - - /** - * Test 3: snapshotTags(int[]) with an oversized buffer (length > attributes.size()) - * must write the managed indices [0, attributes.size()) with the current sidecar values, - * and zero out the extra indices [attributes.size(), snapshot.length). - */ - @Test - public void testSnapshotTagsOversizedBufferCopiesAndZerosExtras() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - - // Verify no-arg overload returns valid IDs. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Oversized buffer: length 5 > attributes.size() == 2. - int[] oversized = new int[5]; - Arrays.fill(oversized, -1); - contextSetter.snapshotTags(oversized); - - // Managed indices [0, attributes.size()) must contain the current sidecar values. - assertEquals(canonical[0], oversized[0], - "oversized buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], oversized[1], - "oversized buffer[1] must match no-arg snapshotTags()[1]"); - - // Extra indices [attributes.size(), snapshot.length) must be zeroed. - for (int i = 2; i < oversized.length; i++) { - assertEquals(0, oversized[i], - "oversized buffer element [" + i + "] must be zeroed by snapshotTags"); - } - - // No-arg overload must still work correctly. - int[] check = contextSetter.snapshotTags(); - assertEquals(canonical[0], check[0]); - assertEquals(canonical[1], check[1]); - } - - /** - * Test 4: snapshotTags(int[]) with an undersized buffer (length < attributes.size()) - * must be a no-op — existing no-op semantics must be preserved. - */ - @Test - public void testSnapshotTagsUndersizedBufferIsNoOp() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag3")); - - assertTrue(contextSetter.setContextValue("tag1", "a")); - assertTrue(contextSetter.setContextValue("tag2", "b")); - assertTrue(contextSetter.setContextValue("tag3", "c")); - - // Undersized buffer: length 1 < attributes.size() == 3. - int[] undersized = new int[1]; - undersized[0] = -1; - contextSetter.snapshotTags(undersized); - - assertEquals(-1, undersized[0], - "undersized buffer must not be written by snapshotTags"); - } - - /** - * Test 5: snapshotTags(int[]) with an exact-size buffer (length == attributes.size()) - * must copy the current sidecar values correctly. - */ - @Test - public void testSnapshotTagsExactSizeBufferCopiesCorrectly() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "x")); - assertTrue(contextSetter.setContextValue("tag2", "y")); - - // No-arg overload to obtain expected values. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Exact-size buffer: length 2 == attributes.size() == 2. - int[] exact = new int[2]; - contextSetter.snapshotTags(exact); - - assertEquals(canonical[0], exact[0], - "exact-size buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], exact[1], - "exact-size buffer[1] must match no-arg snapshotTags()[1]"); - } - - private void work(ContextSetter contextSetter, String contextAttribute, String contextValue) - throws InterruptedException { - assertTrue(contextSetter.setContextValue(contextAttribute, contextValue)); - checkTagValues(contextSetter, contextAttribute); - Thread.sleep(10); - assertTrue(contextSetter.clearContextValue(contextAttribute)); - } - - private void checkTagValues(ContextSetter contextSetter, String contextAttribute) { - int[] tags = contextSetter.snapshotTags(); - // expects tag1/tag2/tag3 - change this if the tested tags change - int offset = Integer.parseInt(contextAttribute.substring(3)) - 1; - for (int i = 0; i < tags.length; i++) { - if (i == offset) { - assertNotEquals(0, tags[i]); - } else { - assertEquals(0, tags[i]); - } - } - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter=0,attributes=tag1;tag2;tag3"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java index 2595ed0c6e..a5c7bc1928 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java @@ -32,14 +32,13 @@ public void smokeTest() throws Exception { doThreadFiltering(); } - @SuppressWarnings("deprecation") private void doThreadFiltering() throws Exception { Future[] futures = new Future[1000]; for (int i = 0; i < futures.length; i++) { int id = i; futures[i] = executorService.submit(() -> { profiler.addThread(); - profiler.setContext(id, 42); + profiler.setTraceContext(42, id + 1, 0, id + 1, -1, null, -1, null); try { Thread.sleep(2); } catch(InterruptedException e) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java index 2201f7cac0..845311ca2c 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java @@ -118,11 +118,7 @@ public void testNoSigsegvInWriteStackTracesAfterClassUnload() throws Exception { assertTrue(Files.size(dumpFile) > 0, "Profiler produced no output — SIGSEGV during writeStackTraces is suspected"); } finally { - try { - profiler.stop(); - } finally { - profiler.resetThreadContext(); - } + profiler.stop(); } } finally { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java index 88cc6a472f..03d4a6505f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java @@ -41,14 +41,13 @@ private Task(JavaProfiler profiler) { } @Override - @SuppressWarnings("deprecation") public void run() { - profiler.setContext(1, 2); + profiler.setTraceContext(2, 1, 0, 1, -1, null, -1, null); long now = profiler.getCurrentTicks(); if (profiler.isThresholdExceeded(9, start, now)) { profiler.recordQueueTime(start, now, getClass(), QueueTimeTest.class, ArrayBlockingQueue.class, 10, origin); } - profiler.clearContext(); + profiler.clearTraceContext(); } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java index 43d840b4a4..e646762ecd 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java @@ -134,11 +134,11 @@ public void tracedSleepingThreadIsSampled() throws InterruptedException { Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); registerCurrentThreadForWallClockProfiling(); - profiler.setContext(0x5100L, 0x5101L, 0L, 0x5101L); + profiler.setTraceContext(0x5100L, 0x5101L, 0L, 0x5101L, -1, null, -1, null); try { Thread.sleep(300); } finally { - profiler.clearContext(); + profiler.clearTraceContext(); } stopProfiler(); @@ -186,14 +186,13 @@ public void suppressionCounterIsZeroWhenPrecheckDisabled() throws Exception { } /** - * Recreates the steady state left after a previous test initialized and then removed the Java - * ThreadContext: the native ProfiledThread still owns an initialized OTEP record, but the - * record is cleared and invalid. + * Recreates the steady state left after a previous test activated and then cleared the trace + * context: the native ProfiledThread still owns an initialized OTEP record, but the record is + * cleared and invalid. */ private void leaveClearedInitializedContext() { - profiler.setContext(0x7700L, 0x7701L, 0L, 0x7701L); - profiler.clearContext(); - profiler.resetThreadContext(); + profiler.setTraceContext(0x7700L, 0x7701L, 0L, 0x7701L, -1, null, -1, null); + profiler.clearTraceContext(); } @Override diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java index a2caa1b8a5..7ae94290db 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java @@ -62,7 +62,7 @@ public void precheckAndParkSuppressionWorkTogether() throws Exception { registerCurrentThreadForWallClockProfiling(); long spanId = 0x1111L; long rootSpanId = 0x2222L; - profiler.setContext(rootSpanId, spanId, 0, 0); + profiler.setTraceContext(rootSpanId, spanId, 0, 0, -1, null, -1, null); ready.countDown(); ProfilerOwnedBlockHooks.parkEnter(profiler); long parkedUntil = System.nanoTime() + 280_000_000L; @@ -71,7 +71,7 @@ public void precheckAndParkSuppressionWorkTogether() throws Exception { } ProfilerOwnedBlockHooks.parkExit( profiler, System.identityHashCode(this), 0L); - profiler.clearContext(); + profiler.clearTraceContext(); }, "combined-parked"); diff --git a/doc/architecture/TLSContext.md b/doc/architecture/TLSContext.md index 0a17829dce..caf6988f06 100644 --- a/doc/architecture/TLSContext.md +++ b/doc/architecture/TLSContext.md @@ -10,9 +10,12 @@ with active traces. The system uses OTEL profiling signal conventions ([OTEP #4947](https://github.com/open-telemetry/oteps/pull/4947)) as its -sole context storage format. Java code writes tracing context into -thread-local `DirectByteBuffer`s mapped to native structs. Two consumer -paths read this context concurrently: +sole context storage format. Java code writes tracing context through a +small set of native JNI primitives (`setTraceContext`/`clearTraceContext`/ +`setContextValue`/`clearContextValue`) that resolve the current carrier's +`OtelThreadContextRecord` and mutate it directly — there is no cached +per-thread Java-side buffer. Two consumer paths read this context +concurrently: 1. **DD signal handler (SIGPROF)** — reads integer tag encodings and root-span ID from the sidecar buffer, and span ID from the OTEL record @@ -21,33 +24,27 @@ paths read this context concurrently: `otel_thread_ctx_v1` TLS symbol via ELF dynsym and read the `OtelThreadContextRecord` directly. -All writes from Java are zero-JNI on the hot path (cache-hit case), -using `DirectByteBuffer` with explicit memory ordering. A detach/attach -publication protocol ensures readers see either a complete old record or -a complete new record, never a torn intermediate state. - -For benchmark data, see -[ThreadContext Benchmark Report](../performance/reports/thread-context-benchmark-2026-03-21.md). +Each write resolves the current carrier's record inside a single JNI call — +race-free by construction under virtual-thread migration, since a JNI native +frame pins a mounted virtual thread to its carrier for the call's duration. +A detach/attach publication protocol ensures readers see either a complete +old record or a complete new record, never a torn intermediate state. ## Core Design Principles -1. **Zero-Copy Shared Memory** — Java writes to `DirectByteBuffer`s - mapped to native `ProfiledThread` fields; no data copying between - Java and native heaps. -2. **Signal Handler Safety** — all signal-handler reads use lock-free +1. **Signal Handler Safety** — all signal-handler reads use lock-free atomic loads with acquire semantics; no allocation, no locks, no syscalls. -3. **Detach/Attach Publication Protocol** — the `valid` flag is cleared - before mutation and set after, with `storeFence` barriers between - steps. The TLS pointer is set permanently at thread init. -4. **Two-Phase Attribute Registration** — string attribute values are - registered in the native Dictionary once via JNI; subsequent uses - are zero-JNI ByteBuffer writes from a per-thread encoding cache. -5. **Platform Independence** — correct on both strong (x86/TSO) and - weak (ARM) memory models via explicit `storeFence` / volatile write - barriers. -6. **Low Overhead** — typical span lifecycle write ~30 ns, sidecar - encoding read ~2 ns (no syscalls, no locks). +2. **Detach/Attach Publication Protocol** — the `valid` flag is cleared + before mutation and set after, with release fences between steps. The + TLS pointer is set permanently at thread init. +3. **Two-Phase Attribute Registration** — string attribute values are + registered in the native Dictionary once via JNI (`ContextValueCache`); + subsequent uses of the same value reuse the cached encoding. +4. **Platform Independence** — correct on both strong (x86/TSO) and + weak (ARM) memory models via explicit release fences. +5. **Low Overhead** — one JNI call per context operation; no cached buffer + to invalidate or reattach across thread migration. ## Architecture @@ -58,30 +55,29 @@ For benchmark data, see │ Application Thread │ ├─────────────────────────────────────────────────────────────────────┤ │ │ -│ Tracer calls ThreadContext.put(lrs, spanId, trHi, trLo) │ +│ Tracer calls JavaProfiler.setTraceContext(lrs, spanId, trHi, trLo, │ +│ slot0, v0, slot1, v1) │ │ │ │ │ ▼ │ │ ┌───────────────────────────────────────────────────────────────┐ │ -│ │ setContextDirect() │ │ -│ │ 1. detach() — valid ← 0, storeFence │ │ -│ │ 2. ctxBuffer.putLong(traceIdOffset, reverseBytes(trHi)) │ │ -│ │ ctxBuffer.putLong(traceIdOffset+8, reverseBytes(trLo)) │ │ -│ │ ctxBuffer.putLong(spanIdOffset, reverseBytes(spanId)) │ │ +│ │ setTraceContext0() (single JNI call) │ │ +│ │ 1. detach() — valid ← 0, release fence │ │ +│ │ 2. record->trace_id/span_id ← big-endian encode │ │ │ │ 3. tag_encodings[0..9] ← 0 │ │ │ │ attrs_data_size ← LRS_ENTRY_SIZE (keeps fixed LRS at [0]) │ │ -│ │ 4. ctxBuffer.putLong(lrsOffset, lrs) │ │ -│ │ writeLrsHex(lrs) — update fixed LRS entry in attrs_data │ │ -│ │ 5. attach() — storeFence, valid ← 1 │ │ +│ │ 4. sidecar local_root_span_id ← lrs; write LRS hex entry │ │ +│ │ 5. write slot0/slot1 activation attributes (if any) │ │ +│ │ 6. attach() — release fence, valid ← 1 │ │ │ └───────────────────────────────────────────────────────────────┘ │ │ │ │ │ ▼ │ │ ┌──────────────────────────────────────────────────────────────┐ │ -│ │ Unified ctxBuffer (688B, single DirectByteBuffer) │ │ +│ │ ProfiledThread (native, per-thread heap allocation) │ │ │ │ ┌──────────────────────┐ ┌───────────────────────────┐ │ │ │ │ │ OtelThreadContextRec │ │ tag_encodings[10] (u32) │ │ │ │ │ │ trace_id[16] (BE) │ │ local_root_span_id (u64) │ │ │ │ │ │ span_id[8] (BE) │ └───────────────────────────┘ │ │ -│ │ │ valid (u8)│ offsets 640..688 in ctxBuffer │ │ +│ │ │ valid (u8)│ contiguous sidecar region │ │ │ │ │ reserved (u8)│ │ │ │ │ │ attrs_data_size(u16)│ ┌──────────────────────────────┐ │ │ │ │ │ attrs_data[612] │ │ TLS pointer (8B) │ │ │ @@ -105,25 +101,16 @@ For benchmark data, see ├──────────────────────────────────────────────────────────────────────┤ │ │ │ JavaProfiler │ -│ ├─ ThreadLocal tlsContextStorage │ -│ ├─ initializeContextTLS0(long[] metadata) → ByteBuffer (688B) │ -│ └─ registerConstant0(String value) → int encoding │ -│ │ -│ ThreadContext (per thread) │ -│ ├─ ctxBuffer (688B DirectByteBuffer — record + sidecar contiguous)│ -│ ├─ put(lrs, spanId, trHi, trLo) → setContextDirect() │ -│ ├─ setContextAttribute(keyIdx, value) → setContextAttributeDirect │ -│ ├─ snapshot(byte[], int) / restore(byte[], int) ← nested scopes │ -│ └─ Per-thread caches: │ -│ └─ attrCache[CACHE_SIZE]: String → {int encoding, byte[] utf8}│ +│ ├─ setTraceContext(lrs, spanId, trHi, trLo, slot0, v0, slot1, v1) │ +│ ├─ clearTraceContext() │ +│ ├─ setContextValue(slot, value) / clearContextValue(slot) │ +│ └─ contextValueCache: ContextValueCache │ │ │ -│ BufferWriter (memory ordering abstraction) │ -│ ├─ BufferWriter8 (Java 8: Unsafe) │ -│ │ ├─ putOrderedLong / putOrderedInt │ -│ │ └─ storeFence → Unsafe.storeFence() │ -│ └─ BufferWriter9 (Java 9+: VarHandle) │ -│ ├─ setRelease │ -│ └─ storeFence → VarHandle.storeStoreFence() │ +│ ContextValueCache (process-wide, shared across every carrier / │ +│ virtual thread — no per-thread instance) │ +│ ├─ resolve(value) → {encoding, utf8} │ +│ ├─ direct-mapped table keyed by value.hashCode() & 0xFF │ +│ └─ registerConstant0(String value) → int encoding (JNI) │ └──────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────┐ @@ -179,7 +166,6 @@ Offset Size Field Description Total: 48 bytes ``` -The sidecar fields are exposed to Java as a single `DirectByteBuffer`. Tag encodings are integer IDs from the profiler's `Dictionary` constant pool — the signal handler writes them directly into JFR events without any string lookup. @@ -202,7 +188,7 @@ Each entry in `attrs_data` is encoded as: ### Problem Two concurrent readers may observe the record at any point during a -Java-side mutation: +native-side mutation: 1. **SIGPROF signal handler** — interrupts the writing thread mid-sequence, runs on the same thread. @@ -215,28 +201,31 @@ a partially-written record. ### Protocol ``` -Java writer timeline: +Native writer timeline (setTraceContext0 / setContextValue0): ────────────────────────────────────────────────────────────────── Time 0: detach() - ctxBuffer.put(validOffset, 0) ← mark invalid - storeFence() ← drain store buffer + record->valid ← 0 ← mark invalid + release fence ← drain store buffer Time 1: Mutate record fields - ctxBuffer.putLong(traceIdOffset, ...) - ctxBuffer.putLong(spanIdOffset, ...) - tag_encodings[0..9] ← 0 ← zero tag encodings (offsets 640..680) - attrs_data_size ← LRS_ENTRY_SIZE ← keep only fixed LRS entry at attrs_data[0] - ctxBuffer.putLong(lrsOffset, lrs) ← update LRS at offset 680 - writeLrsHex(lrs) ← update LRS hex entry in attrs_data + record->trace_id / span_id ← big-endian encode + tag_encodings[0..9] ← 0 ← zero tag encodings (setTraceContext0 only) + attrs_data_size ← LRS_ENTRY_SIZE ← keep only fixed LRS entry at attrs_data[0] + sidecar local_root_span_id ← lrs ← update LRS sidecar field + writeLrsHex(lrs) ← update LRS hex entry in attrs_data ⚡ SIGPROF may arrive here — handler sees valid=0, skips record Time 2: attach() - storeFence() ← ensure writes visible - ctxBuffer.put(validOffset, 1) ← mark valid + release fence ← ensure writes visible + record->valid ← 1 ← mark valid ────────────────────────────────────────────────────────────────── ``` +`clearContextValue0` is the one exception: it preserves whatever `valid` +value it found (removing a single attribute must not resurrect a record +that a preceding `clearTraceContext0()` intentionally left detached). + ### Reader: DD Signal Handler ```cpp @@ -267,7 +256,7 @@ for (int i = 0; i < 8; i++) { val = (val << 8) | record->span_id[i]; } span_id = val; ``` -The acquire fence pairs with the writer's `storeFence` + `valid=1` +The acquire fence pairs with the writer's release fence + `valid=1` sequence, ensuring all record field writes are visible if `valid` reads as 1. @@ -292,74 +281,41 @@ different reasons: - The **signal handler** runs on the same thread as the writer. The CPU presents its own stores in program order, so CPU store-buffer reordering is not a - concern. The JIT compiler can still reorder stores arbitrarily, so a compiler + concern. The compiler can still reorder stores arbitrarily, so a compiler barrier is required. - The **external OTEP profiler** (e.g. eBPF using scheduler events) attaches a `sched_switch` tracepoint that fires on the same CPU that was executing the thread. The Linux scheduler acquires `rq_lock` before the tracepoint fires, which includes a full hardware memory barrier (`smp_mb__before_spinlock` on ARM). By the time the eBPF probe runs, all prior user-space stores from that - thread are globally visible — including any stores ordered by `DMB ISHST`. - -In both cases `storeFence` serves as a compiler barrier that prevents the JIT -from sinking record field writes past the `valid=1` store. On ARM it also emits -`DMB ISHST`, which is required to order field writes before `valid=1` at the -hardware level — this is not a mere side effect. - -### Barrier Taxonomy - -| Operation | Java 8 | Java 9+ | ARM | x86 | -|-----------|--------|---------|-----|-----| -| `storeFence` | `Unsafe.storeFence` | `VarHandle.storeStoreFence` | DMB ISHST (~2 ns) | compiler barrier (free) | - -On x86, `storeFence` is a compiler-only barrier (TSO guarantees hardware -store ordering for free; Java 9+ `VarHandle.storeStoreFence` emits no -hardware instruction on x86). On ARM it compiles to `DMB ISHST` (~2 ns). - -### Why storeFence, Not fullFence + thread are globally visible. -The detach/attach protocol only requires store-store ordering — all -operations on the hot path are writes. There are no load-dependent -ordering requirements on the writer side. `storeFence` (~2 ns on ARM) -is sufficient; a full fence (~50 ns on ARM) would be wasteful. +In both cases the release fence prevents the compiler from sinking record +field writes past the `valid=1` store, and on ARM emits the hardware barrier +required to order field writes before `valid=1`. ## Initialization -### Per-Thread TLS Setup +### Self-Initializing on First Write -When a thread first accesses its `ThreadContext` via the `ThreadLocal`: - -```java -// JavaProfiler.initializeThreadContext() -long[] metadata = new long[6]; -ByteBuffer buffer = initializeContextTLS0(metadata); -return new ThreadContext(buffer, metadata); -``` - -The native `initializeContextTLS0` (in `javaApi.cpp`): +There is no separate TLS-initialization call. The first `setTraceContext0` +or `setContextValue0` on a thread that hasn't written context before: 1. Gets the calling thread's `ProfiledThread` (creates one if needed). 2. Sets `otel_thread_ctx_v1` permanently to the thread's `OtelThreadContextRecord` (triggering TLS slot init on musl). -3. Fills the `metadata` array with absolute offsets into the unified - buffer (computed via `offsetof` for record fields; `OTEL_MAX_RECORD_SIZE - + DD_TAGS_CAPACITY*sizeof(u32) = 680` for the LRS offset), so Java code - writes to the correct positions regardless of struct packing changes. -4. Creates a single `DirectByteBuffer` spanning the contiguous 688-byte - region: `_otel_ctx_record` (640 B) followed immediately by - `_otel_tag_encodings` (40 B) and `_otel_local_root_span_id` (8 B). - Contiguity is enforced by `alignas(8)` on `_otel_ctx_record` plus - `sizeof(OtelThreadContextRecord)` being a multiple of 8. -5. Returns the single buffer. - -This is the only JNI call in the initialization path. After this, all -hot-path operations are pure Java ByteBuffer writes into offset regions -of the one buffer. +3. Marks the thread's context initialized, so `ContextApi::get()` and the + wall-clock sampler's precheck start considering the record. + +A thread that only ever calls the all-native write API is therefore fully +visible to both the DD signal handler and external OTEP readers from its +very first write — no priming call is needed. ### Signal-Safe TLS Access -Signal handlers cannot call `initializeContextTLS0` (it may allocate). The -read path uses a pre-initialized pointer: +Signal handlers never call the write primitives (they may allocate on the +Dictionary-registration path). The read path uses a pre-initialized +pointer: ```cpp // ProfiledThread::currentSignalSafe() — no allocation, no TLS lazy init @@ -371,51 +327,32 @@ if (thrd == nullptr || !thrd->isContextInitialized()) { ## Two-Phase Attribute Registration -String attributes are set via `ThreadContext.setContextAttribute(keyIndex, value)`. -The hot path avoids JNI by splitting the work into two phases: +String attributes are set via `JavaProfiler.setContextValue(slot, value)` +(or as activation attributes via `setTraceContext`). Registration is split +into two phases so that repeated values avoid a JNI call: ### Phase 1: Registration (cache miss) -On the first call with a new string value: - -```java -encoding = registerConstant0(value); // JNI → Dictionary lookup -utf8 = value.getBytes(UTF_8); // one allocation, cached -attrCacheEncodings[slot] = encoding; -attrCacheBytes[slot] = utf8; -attrCacheKeys[slot] = value; // cache is per-thread; no fence needed -``` - -`registerConstant0` crosses JNI once to register the value in the -native `Dictionary` and returns an integer encoding. +On the first call with a new string value, `ContextValueCache.resolve` +crosses JNI once via `registerConstant0` to register the value in the +native Dictionary and cache the resulting `{encoding, utf8}` pair, +keyed by `value.hashCode() & 0xFF`. -### Phase 2: Cached Write (cache hit, zero JNI) +### Phase 2: Cached Resolution (cache hit) -On subsequent calls with the same string: - -```java -if (value.equals(attrCacheKeys[slot])) { - encoding = attrCacheEncodings[slot]; // int read - utf8 = attrCacheBytes[slot]; // ref read -} -// Both sidecar and OTEP attrs_data are written inside the detach/attach window -// so a signal handler never sees a new sidecar encoding alongside old attrs_data. -detach(); -ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * 4, encoding); -replaceOtepAttribute(otepKeyIndex, utf8); -attach(); -``` - -The cache is a 256-slot direct-mapped structure keyed by -`value.hashCode() & 0xFF`. Collisions evict the old entry (benign — -causes a redundant `registerConstant0` call). In production web -applications with 5–50 unique attribute values, the hit rate is -effectively 100%. +On subsequent calls with the same string, `ContextValueCache.resolve` +returns the cached `{encoding, utf8}` pair with no JNI call; the caller +still crosses JNI once to write the value into the current record (there +is no per-thread cached buffer to write into directly). Collisions in the +256-slot direct-mapped cache evict the old entry (benign — causes a +redundant `registerConstant0` call on the next use of the evicted value). +In production web applications with 5–50 unique attribute values, the hit +rate is effectively 100%. ## Signal Handler Read Path `Recording::writeCurrentContext()` executes in the SIGPROF handler and -reads context in bounded time (~15 ns) with no allocation: +reads context in bounded time with no allocation: 1. `ContextApi::get(spanId, rootSpanId)`: - `ProfiledThread::currentSignalSafe()` — cached pointer, no TLS @@ -432,73 +369,43 @@ reads context in bounded time (~15 ns) with no allocation: No dictionary lookup, no string comparison, no allocation. The encodings written to JFR events are resolved later during JFR parsing. -## Performance - -### Write Path Costs (arm64, Java 25) - -| Operation | ns/op | Path | -|-----------|------:|------| -| `clearContext` | 5.0 | detach + zero fields | -| `setContextFull` | 11.1 | detach + 3 putLong + attach | -| `setAttrCacheHit` | 10.7 | cache lookup + sidecar write + detach/attach | -| `spanLifecycle` | 30.4 | `setContextFull` + `setAttrCacheHit` | - -### Multi-Threaded Scaling - -| Benchmark | 1 thread | 2 threads | 4 threads | -|-----------|----------|-----------|-----------| -| `setContextFull` | 11.1 ns | 11.1 ns | 11.7 ns | -| `spanLifecycle` | 30.4 ns | 30.7 ns | 32.2 ns | - -No false sharing: each thread's `OtelThreadContextRecord` and sidecar -are embedded in its own heap-allocated `ProfiledThread`. - -### Instrumentation Budget - -At ~35 ns per span (`spanLifecycle` 30.4 ns + `clearContext` 5.0 ns), a single thread -can sustain ~28 million span transitions per second. For a web -application at 100K requests/second, this is <0.004% of CPU time. - -Full benchmark data and analysis: -[thread-context-benchmark-2026-03-21.md](../performance/reports/thread-context-benchmark-2026-03-21.md) - ## Testing ### Integration Tests (Java) -`ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java`: +`ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextTest.java`: + +- Verifies `setTraceContext`/`clearTraceContext`/`setContextValue`/ + `clearContextValue` round-trip correctly, using package-private + `JavaProfiler.test*` accessors (read via reflection) as the read oracle — + each wraps a native primitive that reads the current thread's + `OtelThreadContextRecord` directly, with no cached buffer involved. +- Covers span-transition attribute reset, attribute TLV encoding/overflow, + and thread isolation. + +`ddprof-test/src/test/java/com/datadoghq/profiler/context/AllNativeContextSamplingTest.java`: -- **testOtelStorageModeContext** — context round-trips correctly with JFR running. -- **testOtelModeCustomAttributes** — verifies attribute TLV encoding in - `attrs_data` via `setContextAttribute`. -- **testOtelModeAttributeOverflow** — overflow of `attrs_data` is handled - gracefully (returns false, no crash). -- **testSequentialContextUpdates** — repeated writes with varying values, - `Long.MAX_VALUE` round-trip, and `clearContext` resetting both IDs to zero. -- **testThreadIsolation** — concurrent writes from multiple threads, - validating thread-local isolation. -- **testSpanTransitionClearsAttributes** — direct span-to-span transition - without `clearContext` does not leak custom attributes from the previous span. +- Regression test that the all-native write path self-initializes OTEL TLS + on first write — a thread that never calls anything but the all-native + API must still be visible to the wall-clock sampler. `ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java`: -- **test** — validates context propagation through wall-clock profiling +- Validates context propagation through wall-clock profiling samples and JFR event correlation across cstack modes. -`ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java`: +`ddprof-test/src/test/java/com/datadoghq/profiler/ContextValueCacheTest.java`: -- **test** — validates integer tag/attribute context propagation through - profiling samples. +- Validates `ContextValueCache` resolution, hash-collision eviction, + oversized-value rejection, and `clear()` semantics directly. ### JMH Benchmarks -`ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java`: +`ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ContextCombinedBenchmark.java`: -- Single-threaded: `setContextFull`, `setAttrCacheHit`, `spanLifecycle`, - `clearContext`, `getContext`. -- Multi-threaded: `setContextFull_2t/4t`, `spanLifecycle_2t/4t` — - `@Threads(2)` and `@Threads(4)` variants to verify linear scaling - (absence of false sharing). +- Perf-guard for the full per-scope activate+deactivate cycle + (`setTraceContext`+`clearTraceContext`) as dd-trace-java drives it, on + both platform and mounted virtual threads. ## OTEP References diff --git a/doc/plans/2026-07-02-all-native-context-storage-design.md b/doc/plans/2026-07-02-all-native-context-storage-design.md deleted file mode 100644 index 7b299b94b7..0000000000 --- a/doc/plans/2026-07-02-all-native-context-storage-design.md +++ /dev/null @@ -1,316 +0,0 @@ -# Design note: all-native OTEL context storage (eliminating the carrier-owned DirectByteBuffer) - -- **Status:** Proposal (for review by java-profiler + dd-trace-java owners) -- **Date:** 2026-07-02 -- **Related:** PROF-15271 (carrier-scoping fix, commit `94686da`), dd-trace-java #11646 (reapply-on-mount) -- **Scope:** cross-repo — `java-profiler` (this repo) and `dd-trace-java` - -## TL;DR - -The OTEL trace-context write path assumes native OS threads map 1:1 to Java threads. Virtual -threads break that assumption and cause a use-after-free that can crash the JVM. The shipped -carrier-scoping fix (`94686da`) shrinks the crash window but does not close it. - -This note proposes making every context write go through a **native (JNI) call that resolves the -current carrier inside the call** — where the mounted virtual thread is *pinned to its carrier for -the duration by the native frame*, making the write **race-free by construction**. This deletes the -`DirectByteBuffer` / `CarrierThreadLocal` / reflection machinery entirely rather than adding more to -it. - -A JMH campaign (six benchmark classes, JDK 21, glibc x64) establishes the key result: the all-native -path is a **net win on the real per-scope-activation cycle** — but only when the per-activation -context is written in **one combined call**. With the current fine-grained call sequence it would -regress ~1.6×. The recommendation is therefore all-native **plus a combined per-activation API**, -rolled out via expand → migrate → contract. - -The size of the win depends on which DBB baseline it is measured against (see §4 for the full -breakdown and the two figures): - -- Against the **prototype microbenchmark** baseline (§4), which caches the `ThreadContext` handle and - so pays the per-call `currentContext()` / `CarrierThreadLocal` lookup only once, the combined - native cycle is **~7–10% faster** (128 vs 138 ns). -- Against the **production API** baseline — the shipped `ContextCombinedBenchmark`, where the DBB - cycle is the six real `setContext` / `setContextAttribute` / `clearContextAttribute` calls - dd-trace-java actually makes, each re-resolving `currentContext()` — the combined native cycle - (two JNI calls, no per-call lookup) is **~28% faster** (~136 vs ~189 ns). - -Both are the same effect from opposite ends: the fewer JNI transitions the combined call needs, the -more of the DBB path's repeated per-call lookup cost it removes. The ~28% figure is the one that -reflects real tracer usage; the ~7–10% is the conservative lower bound with the lookup amortized away. - -Consumer audit: the entire public context API has **exactly one production consumer** (the -dd-trace-java profiler bridge) and no in-repo external consumers, so the migration risk is internal -coordination only. - ---- - -## 1. Background & problem - -The OTEL context record (`OtelThreadContextRecord`, 640 B + tag-encoding sidecar + LRS = 688 B) is -embedded in the **carrier's** native `ProfiledThread`. It is written by Java (through a -`DirectByteBuffer` "window" over that native memory) and read by the sampler (an async-signal -handler bound to the carrier OS thread, via `ProfiledThread::getOtelContextRecord()` — the sampler -never touches the Java buffer). - -The write path was designed for platform threads, where a Java thread == a native thread for life. -Under Loom this breaks: - -- A `DirectByteBuffer` cached per *virtual* thread is bound to whatever carrier was mounted at first - use. When the vthread migrates, writes land on the wrong carrier (misattribution — the - "split-context" problem). -- Worse: when the original carrier's OS thread exits, its `ProfiledThread` is `delete`d - (`thread.cpp` `freeKey`), freeing the record. The dangling buffer keeps being written — a - **use-after-free** into memory the JVM has reused, observed as a SIGSEGV in - `ThreadsSMRSupport::free_list`. - -**What shipped (`94686da`, PROF-15271):** context storage was moved to -`jdk.internal.misc.CarrierThreadLocal`, so a mounted vthread resolves to its *current* carrier's -record per call. This eliminates the steady-state misattribution and shrinks the UAF window. - -**Why it's incomplete:** there is still a TOCTOU window between resolving the buffer and the write -completing. Today that window is effectively closed on the write path (the writes are plain -`ByteBuffer` ops with no yield point, and JDK 21 does not preemptively unmount vthreads at arbitrary -safepoints). But it reopens if/when Loom adds general preemptive scheduling — which is on the -roadmap. We want a fix that is correct by construction, not one that depends on current scheduler -behavior. - ---- - -## 2. Core insight - -Two observations reframe the problem: - -1. **Crash-safety is about record *lifetime*, not write *timing*.** A write landing on the "wrong" - carrier is only misattribution (non-fatal). The *only* fatal case is writing into a record whose - backing `ProfiledThread` has been freed. So the fix must ensure the write never targets freed - memory — either by never freeing it (arena) or by resolving a live target at write time (native). - -2. **A native frame pins a mounted virtual thread to its carrier.** JDK's freeze logic returns - `freeze_pinned_native` the moment it walks onto a native/JNI frame - (`continuationFreezeThaw.cpp`). So *while a JNI call executes, the vthread cannot unmount* — not - by voluntary yield, not by forced preemption. A native method that resolves - `ProfiledThread::current()` and writes the record therefore targets the **live current carrier**, - with no cached pointer and no migration window. Race-free by construction. - -This is why "all-native" is not just a performance question — it is the *most correct* design: it -removes the cached-buffer concept that caused the bug. - ---- - -## 3. Options considered - -| Option | Idea | Verdict | -|---|---|---| -| **A. Arena records** | Allocate records from a slab never returned to the allocator; `freeKey` recycles a slot instead of freeing memory. Stale writes then hit valid profiler memory (UAF → bounded misattribution). | Viable fallback. Keeps the fast zero-JNI DBB writes and is crash-safe with **no tracer change** — but adds an allocator (the "buffer management" we want to avoid) and keeps `CarrierThreadLocal`. | -| **B. All-native writes** | Every context write is a JNI call that resolves the current carrier (pinned by the native frame) and writes in C. | **Recommended.** Race-free by construction; deletes the DBB/CTL/reflection subsystem. Net perf win with a combined API (§4–5). | -| **C. JVMTI mount/unmount events** | Use `VirtualThreadMount/Unmount` extension events to refresh a cached buffer on mount. | Rejected. Cannot close the TOCTOU (the event can't reach a value already loaded into a Java frame), and enabling `can_support_virtual_threads` imposes a process-wide slow-path tax on *all* vthread transitions. | -| **D. In-buffer handshake** | A lock/generation word in the record; writer acquires it, `freeKey` waits for release before freeing. | Rejected. The lock lives in the memory being freed (bootstrap UAF: you can't safely touch the lock to learn the object is dead), and making `freeKey` wait converts the safety bug into a liveness stall under preemption. Collapses into "A + seqlock." | -| **E. Reflective pinning** (`disableSuspendAndPreempt`) | Borrow the JVM's own `notifyJvmtiDisableSuspend` + `Continuation.pin` to make the write critical-section unmount-proof. | Rejected. `notifyJvmtiDisableSuspend` is a private, boolean-**toggle**-with-assert (not a nesting counter) — driving it from outside `java.lang` risks corrupting per-thread VM state; unstable internals; per-call overhead on a hot path. | - -Option B is chosen because it is simultaneously the most-correct (native-frame pinning), the -lowest-maintenance (deletes code rather than adding an allocator), and — per the benchmarks — at -least perf-neutral on the real production surface. - ---- - -## 4. Benchmark evidence - -Full campaign: JDK 21 (`21.0.10`), glibc linux-x64, JMH `AverageTime` ns/op, 3 forks × 3+3 iters, -carrier vs thread storage mode confirmed per fork (no silent fallback). Prototypes are -benchmark-only (`*Native0` methods, not wired into the production API). Numbers are carrier mode -unless noted; treat as order-of-magnitude (per-fork JIT variance is ~1–6 ns on the sub-50 ns paths). - -**Component costs (corrected from initial estimates):** - -- `CarrierThreadLocal.get()` ≈ **3.9 ns** (plain `ThreadLocal` ≈ 3.3 ns) — the lookup is cheap, not - the ~10–25 ns first guessed. Carrier-scoping added ~no per-call cost. -- A Java→native JNI transition ≈ **~15 ns** (backed out from the scalar native path). So a - transition is *more* expensive than the TL lookup, not less. -- The DBB `setContext` *write* ≈ **18.8 ns** — ~30 bounds-checked `ByteBuffer` ops + 2 fences. This - (not the lookup) is where the DBB scalar cost lives. - -**Path-by-path (DBB vs all-native):** - -| Path | DBB | all-native | Verdict | -|---|---|---|---| -| scalar `setContext` (per span) | ~30 | ~27 | native wins ~11% | -| single `setContextAttribute` (cache hit) | **28.5** (zero-JNI) | 59.0 | native **loses ~2.1×** | -| single `clearContextAttribute` (fair) | **9.4** (zero-JNI) | 21.0 | native **loses ~2.2×** | -| batch reapply, 5 attrs, per-slot | 337 | 480 | native loses (object-array JNI) | -| batch reapply, 5 attrs, **flattened** | 337 | **81** precomp / 109 rebuild | native wins **3–4×** | -| `snapshot`+`restore` (688 B pair) | **45** (inlined memcpy) | 137 | native **loses ~3×** | -| **combined activate+deactivate cycle** | **138.5** | **128.3** | **native wins ~7% (carrier), ~10% (thread)** | - -Sub-results for the combined cycle: native *activate* alone loses (105.8 vs 73.5 — one combined call -vs 3 zero-JNI writes), but native *deactivate* wins decisively (~22 ns for one `clearFullContext` vs -~65 ns for `setContext(0,0,0,0)` + 2× `clearContextAttribute`); the full cycle nets in native's -favor. vthread mirrors platform (native 131.3 vs DBB 140.8), with a ~1–3 ns mount penalty; native is -mount- and mode-independent throughout. - -> **Prototype vs production baseline (why the PR quotes ~28%, not ~7–10%).** The `138.5` DBB figure -> above is the *prototype* microbenchmark: it caches the `ThreadContext` handle, so the DBB cycle -> pays the `currentContext()` / `CarrierThreadLocal` lookup only once. The shipped perf-guard, -> `ContextCombinedBenchmark`, instead drives the *production* API — the six real -> `setContext`/`setContextAttribute`/`clearContextAttribute` calls dd-trace-java makes, each -> re-resolving `currentContext()` — so its DBB baseline is higher (~189 ns) and the combined native -> cycle (two JNI calls, no repeated lookup) comes out **~28% faster** (~136 ns; ~24% in thread mode). -> Same effect, two baselines: ~7–10% is the conservative lower bound with the lookup amortized away, -> ~28% is the figure against real tracer usage. `ContextCombinedBenchmark` is the one to trust going -> forward; the prototype rows are retained for the path-by-path decomposition only. - -**The unifying principle (holds across all six classes):** - -> All-native **wins** when one JNI call replaces *multiple* DBB operations; it **loses** when it adds -> a JNI transition to a *single* operation the DBB already does zero-JNI. - -This is why batching matters: the per-activation context (4 longs + a couple of named attributes) is -a natural batch, so one combined native call amortizes the transition and beats the fine-grained DBB -sequence. Conversely, `snapshot/restore` is already a single inlined `memcpy` on the DBB side, so -native can only add cost there — but (see §6) that path is unused in production. - ---- - -## 5. Proposed design - -**Make context writes all-native, and expose a combined per-activation API.** - -Production per scope activation/deactivation (verified in §6) is: - -- activate: `setContext(rootSpan, span, trHi, trLo)` + `setContextValue(spanName)` + - `setContextValue(resourceName)` -- deactivate: `setContext(0,0,0,0)` + 2× `clearContextValue` - -Replace this fine-grained sequence with two batched native calls. The activation attributes are a -*variable, config-derived* set, so the API must not hardcode which/how many attributes there are: -dd-trace-java's attribute list is configurable (`orderedContextAttributes` = the `attributes=` -config plus appended `operation`/`resource`), and the operation/resource **slot offsets come from -`offsetOf(...)`, not fixed indices**. Today activation writes exactly two span-derived attributes -(operation + resource — the only values available from the `ProfilerContext`; other configured -attributes like `http.route` are set outside activation by instrumentations and are wiped per -activation so they don't leak between spans). So the combined call carries the scalar context plus a -small set of `(offset, value)` pairs: - -``` -// activate: one JNI call writes scalar context + N span-derived (offset,value) attributes, -// under one detach/attach. Explicit-arity overload for the common case avoids per-call allocation: -void setTraceContext(long rootSpanId, long spanId, long traceIdHigh, long traceIdLow, - int off0, CharSequence val0, int off1, CharSequence val1); -// deactivate: one JNI call clears -void clearTraceContext(); -``` - -- **Do not** hardcode `spanName`/`resourceName` or their slots — pass explicit offsets (the tracer - gets them from `operationNameOffset()`/`resourceNameOffset()`), so the API tracks the configurable - attribute model and survives a future third span-derived attribute (service, span kind). -- **Avoid per-activation allocation.** Prefer allocation-free explicit-arity overloads (the 2-pair - form above matches today exactly) over a naive `int[]`/`Object[]` variant, which would add GC - pressure on the activation hot path. If the span-derived attribute count ever grows past what - overloads cover, add a general array form backed by a tracer-reused scratch buffer. -- The `CharSequence → (encoding, utf8)` value cache (today inside `ThreadContext`) moves to a small - Java-side helper in `java-profiler` that resolves each value and invokes the native call — keeping - the OTEP record layout and the cache encapsulated; the tracer passes offsets + values. -- App-driven, sporadic attributes (e.g. `http.route`, set by instrumentations after activation) - keep a single-attribute native setter. This is the one path where native is ~2× the zero-JNI DBB - write — accepted because it is *not* per-activation (see §7). -- Reads (`getSpanId`, etc.) become JNI calls (~20 ns) — fine, they are cold/test-only (§6). -- The sampler is unchanged: it already reads the native record directly, not via the DBB. -- The OTEP record layout and the `otel_thread_ctx_v1` discovery symbol are unchanged, so external / - eBPF readers are unaffected. - -Why combined rather than keeping the fine-grained API all-native: the benchmarks show the -fine-grained all-native cycle regresses ~1.6× (~6 transitions/cycle), while the combined cycle wins -~7–10% (2 transitions/cycle). Batching is what makes all-native a win rather than a regression. - ---- - -## 6. Consumer audit (de-risking removal) - -Callers of the public context API were enumerated across `java-profiler` (all modules) and -`dd-trace-java`. Result: - -- **Exactly one production consumer:** the dd-trace-java bridge - (`DatadogProfilingIntegration` / `DatadogProfiler` / `ContextSetter`), which calls only - `setContext(4-arg)` and single `setContextValue`/`clearContextValue`. -- **No other production consumers** in either repo. No in-repo non-Datadog consumers. Classes live - in `com.datadoghq.profiler`; the jar is bundled into `dd-java-agent` (no `module-info` exports - found). -- **Verified unused by production** (contradicting an earlier automated pass): the batch setter - `setContextAttributesByIdAndBytes` (grep-empty outside tests), `DatadogProfiler.snapshot()` (the - `int[]` tag snapshot — no production callers), and `ThreadContext.snapshot/restore` + java-profiler's - `ScopeStack` (dd-trace-java has its own `ScopeStack` and re-derives parent context by re-calling - `setContext`; it never calls the profiler's snapshot/restore). -- **Test/benchmark-only:** `getThreadContext`, `resetThreadContext`, `contextStorageMode`, - `readContextAttribute`, `readTraceId`, `getSpanId`, `getRootSpanId`. - -**Removal risk:** entirely internal Datadog coordination (java-profiler ↔ dd-trace-java). No -external breakage identified. Caveat: if `ddprof-lib` is published as a standalone Maven artifact -that external users could compile against, add a deprecation window as insurance (§7). - -Consequence for the design: the `snapshot/restore` ~3× regression and the per-slot batch regression -are **moot** — neither path is on the production tracer surface. The only production paths that -matter are the ones the combined API covers, plus the sporadic single app-attribute. - ---- - -## 7. Migration plan (expand → migrate → contract) - -1. **Expand (java-profiler).** Add the all-native combined API (`setTraceContext` / - `clearTraceContext`), the single-attribute native setter for app tags, and native reads, - alongside the existing DBB APIs. Mark the DBB context methods `@Deprecated` pointing at the new - API. Release. Additive and backward-compatible — bumping to this version changes nothing - operationally. -2. **Migrate (dd-trace-java).** Switch `DatadogProfilingIntegration` to the combined API. Requires - java-profiler ≥ the phase-1 version. **This is where the crash fix and the perf win land.** -3. **Contract (java-profiler).** Once all consumers are on the new API, delete the - DBB / `CarrierThreadLocal` / `OtelContextStorage` subsystem (and, if desired, the unused - `ScopeStack` / `snapshot` / `restore` / batch-setter). Breaking release, gated on a - consumers-migrated confirmation. - -**Sequencing notes / caveats:** - -- **The crash fix lands in phase 2, not phase 1.** Phase 1 is purely additive; the UAF persists on - the old DBB path until the tracer switches. Until then, carrier-scoping (`94686da`) remains the - mitigation. If crash urgency is high, phase 2 is the milestone. -- **Coexistence is safe.** Old (DBB) and new (native) paths write the *same* underlying record; the - native path ignores the DBB; the tracer switches wholesale, so no thread interleaves both. No - dual-write coherence problem. -- **aarch64 memory model.** The native detach/attach uses a release fence + relaxed `valid` store, - mirroring `Unsafe.storeFence` + plain put. Trivially equivalent on x86; verify on aarch64 before - shipping the native path there. -- **Accepted minor regression.** Sporadic app-driven single attributes are ~2× the zero-JNI DBB - write on native. They are not per-activation, so the absolute impact is small; not designed around. - ---- - -## 8. Alternatives explicitly rejected (and why) - -- **Keep DBB + arena (Option A)** — perf-optimal and crash-safe with no tracer change, but adds an - allocator and keeps `CarrierThreadLocal`; contradicts the "no buffer management / most-correct" - goal. Retain as the fallback if the tracer API cannot change. -- **Reflective JVM pinning (Option E)** — corrupts VM state risk (toggle-with-assert), unstable - internals, hot-path overhead. -- **JVMTI mount events (Option C)** — cannot close the TOCTOU; global transition tax. -- **Fine-grained all-native (no combined call)** — measured ~1.6× per-cycle regression; batching is - required. - -## 9. Open questions / follow-ups - -- Confirm `ddprof-lib` Maven publication policy (external-consumer risk for phase 3). -- Decide whether to also remove the unused `ScopeStack` / `snapshot` / `restore` / batch setter in - phase 3, or retain for a possible future reapply-on-mount design (#11646). -- Re-run the combined-cycle benchmark on aarch64 and on a real dd-trace-java workload (chaos harness - under a patched `dd-java-agent`) to validate the ~7–10% win outside the microbenchmark. -- Split-context coherence (#11646) remains a separate tracer-side concern; this change makes it safe - but does not address it. - -## Appendix: benchmark experiment (reproducibility) - -The full JMH campaign — the six benchmark classes -(`ContextWritePathBenchmark`, `ContextAttrBenchmark`, `ContextVThreadBenchmark`, -`ContextScopeBenchmark`, `ContextSingleAttrBenchmark`, `ContextCombinedBenchmark`) and the -benchmark-only native prototypes they exercise (`*Native0` in `javaApi.cpp` with wrappers in -`JavaProfiler.java`) — is preserved on branch **`context-storage-benchmark-experiment`** -(`origin`). That scaffolding is intentionally *not* carried into the productionization branch; the -production API is the pared-down subset (combined per-activation write + single-attribute write + -value cache). Run any benchmark from that branch with -`./gradlew :ddprof-stresstest:jmh -PjmhInclude=""`. diff --git a/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md b/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md deleted file mode 100644 index fdc7f9d4fe..0000000000 --- a/doc/plans/2026-07-09-phase2-dd-trace-java-migration.md +++ /dev/null @@ -1,89 +0,0 @@ -# Phase 2 — migrate dd-trace-java onto the all-native context API - -- **Date:** 2026-07-09 -- **Status:** Plan (execution in progress) -- **Related:** PR #631 (phase 1, java-profiler), PROF-15271, PROF-15361 (follow-up: batch reapply), - design note `2026-07-02-all-native-context-storage-design.md` (§5/§7) -- **Scope:** cross-repo — `java-profiler` (small read-API addition) and `dd-trace-java` (the migration) - -## Goal - -Switch dd-trace-java's profiler bridge off the deprecated DirectByteBuffer (DBB) context APIs onto -the all-native API added in phase 1 (`setTraceContext` / `clearTraceContext` / `setContextValue` / -`clearContextValue`). This is where the virtual-thread use-after-free fix and the per-activation -perf win land (3 JNI calls → 1 on activation). After this, no production code uses the DBB path. - -## Consumption loop (verified) - -- java-profiler is `com.datadoghq:ddprof:1.47.0-SNAPSHOT`; dd-trace-java pins `ddprof = "1.46.0"` and - `-PddprofUseSnapshot=true` derives next-minor → `1.47.0-SNAPSHOT` (exact match). `mavenLocal()` is - already first in dd-trace-java's repositories. -- Dev loop: java-profiler `./gradlew publishToMavenLocal`; dd-trace-java builds/tests with - `-PddprofUseSnapshot=true`. dd-trace-java uses **dependency locking**, so a full build needs - `--write-locks`. - -## Decisions locked (this session) - -1. **`setContextValue` publishes (`valid=1`).** Setting a value makes it visible even with no active - span — app context is visible independent of spans (dd-trace-java PR #11646). `clearContextValue` - preserves the prior valid (removing a value must not resurrect a deactivated record). Mirrors the - DBB single-setter vs bulk-setter asymmetry the tracer's reapply was built on. (Done: `5beb6987c`.) -2. **`setTraceContext` rejects `spanId == 0`** (IllegalArgumentException, thin Java wrapper; native - asserts). Clearing is `clearTraceContext`. Out-of-range slots throw; data conditions (null / - oversized / Dictionary full) stay soft. -3. **Reapply stays per-slot (#1), no batch API.** A per-slot native `setContextValue` loop preserves - the current shape and is adequate for typical app-attr cardinality (0–2). A native batch is - deferred to PROF-15361 and only if measurement warrants (JNI accessors are VM transitions too, so - a `byte[][]` batch would not clearly win; a flattened primitive-array batch would, but is premature). -4. **Design note §5 "wipe and let instrumentation re-set" is superseded** by #11646's reapply model; - we preserve reapply-on-activation/deactivation. - -## API mapping (old DBB → new all-native) - -| dd-trace-java use (DBB) | replacement | -| --- | --- | -| `profiler.setContext(root,span,hi,lo)` + `setContextValue(op)` + `setContextValue(res)` | one `profiler.setTraceContext(root,span,hi,lo, opOff,opVal, resOff,resVal)` | -| `profiler.setContext(0,0,0,0)` (clear) | `profiler.clearTraceContext()` | -| `ContextSetter.setContextValue` / `clearContextValue` (→ DBB `setContextAttribute`) | `profiler.setContextValue(slot,v)` / `clearContextValue(slot)` (native) | -| `ContextSetter.setContextValuesByIdAndBytes` (batch reapply) | per-slot native `setContextValue` loop | -| `ContextSetter.snapshotTags` / `JavaProfiler.copyTags` (DBB read; resets via `ThreadContext` ctor) | new native `copyContextTags` read (option A) | -| `ContextSetter.offsetOf` | **kept** — pure-Java `attributes.indexOf`, no DBB | - -## java-profiler change (option A — needed for the test oracle) - -`snapshot()` in dd-trace-java (test-only per the consumer audit) reads tag encodings via -`copyTags` → `currentContext().copyCustoms` — a DBB read whose `ThreadContext` ctor **resets** the -record, clobbering native writes. Add a small **native** read instead: - -- `JavaProfiler.copyContextTags(int[] out)` + `native copyContextTags0(int[])` — copies the current - thread's `enc[]` sidecar tag encodings directly from `ProfiledThread::current()` (no `ThreadContext`, - no reset). Test/introspection only. Add a java-profiler test. - -## dd-trace-java edits (per file) - -- `DatadogProfiler`: - - add `setTraceContext(...)` / `clearTraceContext()` (each: native call + `reapplyAppContext()`). - - `setContextValue(int,String)` / `clearContextValue(int)` → native `profiler.*`. - - `reapplyAppContext()` → per-slot `profiler.setContextValue(i, snapshot.stringAt(i))` loop (drop the - batch + valid=0 fallback; native `setContextValue` publishes uniformly). - - `syncNativeAppContext()` → native `setContextValue`/`clearContextValue`. - - `recordAppContextValue` → drop `snapshotTags`/`contextScratch` (no encoding capture); simplify - `AppContextSnapshot` to strings-only (`nonZeroCount` by string presence; drop `ids`/`utf8`). - - `snapshot()` → `profiler.copyContextTags(scratch)` (native read). - - keep `ContextSetter` only for `offsetOf`. -- `DatadogProfilingIntegration`: activate → one `setTraceContext(...)` folding op+resource; close / - `clearContext()` → `clearTraceContext()`. -- `DatadogProfilingScope`: unchanged in shape (save/restore app context); its writes now land native. - -## Verification - -- java-profiler: `publishToMavenLocal` (1.47.0-SNAPSHOT, incl. the new native read); its own tests green. -- dd-trace-java: `:dd-java-agent:agent-profiling:profiling-ddprof` build + `DatadogProfilerTest`, - `DatadogProfilerConfigTest`, `DatadogProfilerRecordingTest` green with `-PddprofUseSnapshot=true` - (`--write-locks` as needed). Smoke: `dd-smoke-tests/profiling-integration-tests`. - -## Follow-ups - -- PROF-15361: batch reapply (flattened primitive-array), only if reapply shows up in profiling. -- Phase 3 (java-profiler): remove the DBB / `ThreadContext` / `ContextSetter` / `OtelContextStorage` - subsystem once dd-trace-java is on the new API. dd-trace-java then needs its own `offsetOf` map. diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh index ba8f695208..742d3f36a7 100755 --- a/utils/run-chaos-harness.sh +++ b/utils/run-chaos-harness.sh @@ -136,14 +136,12 @@ fi case $CONFIG in profiler) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context and - # vthread-context-cascade (which is driven entirely by @Trace-annotated - # methods) are excluded here. - DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + # @Trace is a no-op without the tracer, so trace-context is excluded here. + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm,reapply-context-value" ;; profiler+tracer) ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm,reapply-context-value" ;; *) echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 @@ -163,7 +161,7 @@ case $ALLOCATOR in # Logged so a glibc-detected corruption abort can be reproduced with the # same perturb byte (the value is otherwise random per run). echo "MALLOC_PERTURB_=${MALLOC_PERTURB_}" - # thread-churn/dump-storm/vthread-context-cascade cycle many short-lived + # thread-churn/dump-storm/vthread-churn cycle many short-lived # threads; glibc's per-thread arenas are slow to trim back to the OS, # which was inflating container RSS past the OOM limit on aarch64 # (mirrors the tcmalloc/jemalloc tuning below). @@ -251,7 +249,6 @@ CHAOS_START=$(date +%s) timeout "$((RUNTIME + 300))" \ java -javaagent:${PATCHED_AGENT} \ --add-opens java.base/java.lang=ALL-UNNAMED \ - --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ ${ENABLEMENT} \ -Ddd.profiling.upload.period=10 \ -Ddd.profiling.start-force-first=true \