From b8bbd269e0a4fa8950d864bd3ab25deed870fd91 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:55:46 -0400 Subject: [PATCH 01/26] Define Feature Flagging configuration source contract --- .../OpenFeatureProviderSmokeTest.groovy | 1 + .../datadog/trace/api/ConfigDefaults.java | 4 + .../api/config/FeatureFlaggingConfig.java | 11 +++ .../main/java/datadog/trace/api/Config.java | 78 +++++++++++++++++++ .../datadog/trace/api/ConfigTest.groovy | 32 ++++++++ metadata/supported-configurations.json | 32 ++++++++ .../featureflag/FeatureFlaggingSystem.java | 2 +- .../FeatureFlaggingSystemTest.java | 4 + ...e.java => ConfigurationSourceService.java} | 2 +- .../featureflag/RemoteConfigServiceImpl.java | 2 +- 10 files changed, 165 insertions(+), 3 deletions(-) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{RemoteConfigService.java => ConfigurationSourceService.java} (60%) diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy index cb4c641d667..ca80ca47f8b 100644 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy @@ -42,6 +42,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true') + builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') return builder } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 1aba2cb276b..532e9f47d71 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,6 +49,10 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..05d53a0c2ad 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,15 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = + "feature.flags.configuration.source"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = + "feature.flags.configuration.source.agentless.base.url"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = + "feature.flags.configuration.source.agentless.poll.interval.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = + "feature.flags.configuration.source.agentless.request.timeout.seconds"; + + private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 70ff04932e7..87f4866f5a5 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -86,6 +86,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_SERVER_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_HEALTH_METRICS_ENABLED; @@ -361,6 +364,10 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_EXCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_QUEUE_SIZE; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL; @@ -1213,6 +1220,11 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; + private final String featureFlaggingConfigurationSource; + private final String featureFlaggingConfigurationSourceAgentlessBaseUrl; + private final int featureFlaggingConfigurationSourcePollIntervalSeconds; + private final int featureFlaggingConfigurationSourceRequestTimeoutSeconds; + private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; private final boolean dbmTracePreparedStatements; @@ -2837,6 +2849,40 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); + featureFlaggingConfigurationSource = + normalizeFeatureFlaggingConfigurationSource( + configProvider.getString( + FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + featureFlaggingConfigurationSourceAgentlessBaseUrl = + configProvider.getStringNotEmpty( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); + int configuredFeatureFlaggingPollIntervalSeconds = + configProvider.getInteger( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); + if (configuredFeatureFlaggingPollIntervalSeconds <= 0) { + log.warn( + "Invalid Feature Flagging agentless poll interval: {}. The value must be positive", + configuredFeatureFlaggingPollIntervalSeconds); + configuredFeatureFlaggingPollIntervalSeconds = + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; + } + featureFlaggingConfigurationSourcePollIntervalSeconds = + configuredFeatureFlaggingPollIntervalSeconds; + int configuredFeatureFlaggingRequestTimeoutSeconds = + configProvider.getInteger( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); + if (configuredFeatureFlaggingRequestTimeoutSeconds <= 0) { + log.warn( + "Invalid Feature Flagging agentless request timeout: {}. The value must be positive", + configuredFeatureFlaggingRequestTimeoutSeconds); + configuredFeatureFlaggingRequestTimeoutSeconds = + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; + } + featureFlaggingConfigurationSourceRequestTimeoutSeconds = + configuredFeatureFlaggingRequestTimeoutSeconds; + dynamicInstrumentationEnabled = configProvider.getBoolean( DYNAMIC_INSTRUMENTATION_ENABLED, DEFAULT_DYNAMIC_INSTRUMENTATION_ENABLED); @@ -3749,6 +3795,14 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } + private static String normalizeFeatureFlaggingConfigurationSource(final String source) { + if (source == null) { + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; + } + final String normalized = source.trim().toLowerCase(Locale.ROOT); + return normalized.isEmpty() ? DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE : normalized; + } + public boolean isBaggageExtract() { return tracePropagationStylesToExtract.contains(TracePropagationStyle.BAGGAGE); } @@ -4656,6 +4710,22 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } + public String getFeatureFlaggingConfigurationSource() { + return featureFlaggingConfigurationSource; + } + + public String getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() { + return featureFlaggingConfigurationSourceAgentlessBaseUrl; + } + + public int getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { + return featureFlaggingConfigurationSourcePollIntervalSeconds; + } + + public int getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { + return featureFlaggingConfigurationSourceRequestTimeoutSeconds; + } + public boolean isDynamicInstrumentationEnabled() { return dynamicInstrumentationEnabled; } @@ -6488,6 +6558,14 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled + + ", featureFlaggingConfigurationSource=" + + featureFlaggingConfigurationSource + + ", featureFlaggingConfigurationSourceAgentlessBaseUrl=" + + featureFlaggingConfigurationSourceAgentlessBaseUrl + + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + + featureFlaggingConfigurationSourcePollIntervalSeconds + + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" + + featureFlaggingConfigurationSourceRequestTimeoutSeconds + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 7db955e49ae..953c5ced8f3 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -2,6 +2,8 @@ package datadog.trace.api import static datadog.trace.api.ConfigDefaults.DEFAULT_HTTP_CLIENT_ERROR_STATUSES import static datadog.trace.api.ConfigDefaults.DEFAULT_HTTP_SERVER_ERROR_STATUSES +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.ConfigDefaults.DEFAULT_PARTIAL_FLUSH_MIN_SPANS import static datadog.trace.api.ConfigDefaults.DEFAULT_SERVICE_NAME import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL @@ -55,6 +57,8 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_METRICS_CONFIGS @@ -3478,4 +3482,32 @@ class ConfigTest extends DDSpecification { "1" | true "0" | false } + + def "agentless feature flag timing uses positive configured values"() { + setup: + Properties properties = new Properties() + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "60") + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "4") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSourcePollIntervalSeconds == 60 + config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == 4 + } + + def "agentless feature flag timing falls back for non-positive values"() { + setup: + Properties properties = new Properties() + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "0") + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "-1") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSourcePollIntervalSeconds == DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS + config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS + } } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..74b6ba24df8 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1497,6 +1497,38 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ + { + "version": "A", + "type": "string", + "default": "agentless", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ + { + "version": "A", + "type": "int", + "default": "30", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ + { + "version": "A", + "type": "int", + "default": "2", + "aliases": [] + } + ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..2011d31e83d 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -9,7 +9,7 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - private static volatile RemoteConfigService CONFIG_SERVICE; + private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; private FeatureFlaggingSystem() {} diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..5b088d99b9d 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -23,6 +24,8 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -48,6 +51,7 @@ void testFeatureFlagSystemInitialization() { } @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java similarity index 60% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java index 5f84a78f7d4..83495545717 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java @@ -2,7 +2,7 @@ import java.io.Closeable; -public interface RemoteConfigService extends Closeable { +public interface ConfigurationSourceService extends Closeable { void init(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index ec28f684a96..a5a9b2d7f36 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -32,7 +32,7 @@ import okio.Okio; public class RemoteConfigServiceImpl - implements RemoteConfigService, ConfigurationChangesTypedListener { + implements ConfigurationSourceService, ConfigurationChangesTypedListener { private final ConfigurationPoller configurationPoller; From 4baefdbf6ec8f4e690e10d07f7ace15475ae6043 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:57:34 -0400 Subject: [PATCH 02/26] Add Datadog-managed agentless UFC polling --- .../trace/util/AgentThreadFactory.java | 3 +- .../feature-flagging-lib/build.gradle.kts | 1 + .../AgentlessConfigurationSource.java | 358 ++++++++ .../AgentlessConfigurationSourceTest.java | 805 ++++++++++++++++++ 4 files changed, 1166 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 752adb8899d..e9ef282cc7d 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -66,7 +66,8 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + FEATURE_FLAG_CONFIGURATION_POLLER("dd-feature-flagging-http-poller"); public final String threadName; diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3291e239d40..548f5e26d2b 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { api(libs.moshi) api(libs.jctools) api(project(":communication")) + implementation(project(":internal-api")) api(project(":products:feature-flagging:feature-flagging-bootstrap")) api(project(":utils:queue-utils")) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java new file mode 100644 index 00000000000..b3c58631d3a --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -0,0 +1,358 @@ +package com.datadog.featureflag; + +import static datadog.communication.http.OkHttpUtils.prepareRequest; +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import datadog.trace.util.AgentThreadFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.DoubleSupplier; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class AgentlessConfigurationSource implements ConfigurationSourceService { + private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessConfigurationSource.class); + + // TODO before merge: confirm the final backend route with the server-distribution API owners. + private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = + "/api/v2/feature-flagging/config/server-distribution"; + private static final int MAX_ATTEMPTS = 3; + private static final long FIRST_RETRY_MIN_MILLIS = 2_000; + private static final long FIRST_RETRY_MAX_MILLIS = 10_000; + private static final long SECOND_RETRY_MIN_MILLIS = 5_000; + private static final long SECOND_RETRY_MAX_MILLIS = 30_000; + private static final double RETRY_JITTER = 0.2; + + private final HttpUrl endpoint; + private final Config config; + private final long pollIntervalMillis; + private final UfcHttpClient client; + private final ScheduledExecutorService executor; + private final RetrySleeper retrySleeper; + private final DoubleSupplier jitter; + private final Object lifecycleLock = new Object(); + private final AtomicBoolean polling = new AtomicBoolean(); + private volatile boolean closed; + private volatile ScheduledFuture scheduledPoll; + private volatile String etag; + + AgentlessConfigurationSource(final Config config) { + this(config, endpoint(config)); + } + + private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint) { + this( + endpoint, + config, + millis(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()), + new OkHttpUfcHttpClient( + OkHttpUtils.buildHttpClient( + endpoint, + millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()))), + Executors.newSingleThreadScheduledExecutor( + new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), + TimeUnit.MILLISECONDS::sleep, + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor) { + this( + endpoint, + config, + pollIntervalMillis, + client, + executor, + TimeUnit.MILLISECONDS::sleep, + () -> 1.0); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter) { + this.endpoint = endpoint; + this.config = config; + this.pollIntervalMillis = pollIntervalMillis; + this.client = client; + this.executor = executor; + this.retrySleeper = retrySleeper; + this.jitter = jitter; + } + + @Override + public void init() { + synchronized (lifecycleLock) { + if (closed || scheduledPoll != null) { + return; + } + scheduledPoll = + executor.scheduleWithFixedDelay( + this::pollOnceSafely, 0, pollIntervalMillis, TimeUnit.MILLISECONDS); + } + } + + boolean pollOnce() { + if (closed || !polling.compareAndSet(false, true)) { + return false; + } + try { + return fetchAndApply(); + } finally { + polling.set(false); + } + } + + @Override + public void close() { + final ScheduledFuture poll; + synchronized (lifecycleLock) { + if (closed) { + return; + } + closed = true; + poll = scheduledPoll; + scheduledPoll = null; + } + if (poll != null) { + poll.cancel(true); + } + client.cancel(); + executor.shutdownNow(); + } + + private void pollOnceSafely() { + try { + pollOnce(); + } catch (final RuntimeException e) { + LOGGER.debug("Unexpected error while polling Feature Flagging HTTP configuration source", e); + } + } + + private boolean fetchAndApply() { + for (int attempt = 1; ; attempt++) { + try { + final UfcHttpResponse response = client.fetch(endpoint, config, etag); + if (closed) { + return false; + } + if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { + if (!waitBeforeRetry(attempt)) { + return false; + } + continue; + } + synchronized (lifecycleLock) { + return !closed && apply(response); + } + } catch (final IOException e) { + if (closed) { + return false; + } + if (attempt == MAX_ATTEMPTS) { + LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); + return false; + } + if (!waitBeforeRetry(attempt)) { + return false; + } + } + } + } + + private boolean waitBeforeRetry(final int attempt) { + if (closed) { + return false; + } + try { + retrySleeper.sleep(retryDelayMillis(pollIntervalMillis, attempt, jitter.getAsDouble())); + return !closed; + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + private boolean apply(final UfcHttpResponse response) { + if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { + return true; + } + if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED + || response.status == HttpURLConnection.HTTP_FORBIDDEN + || response.status != HttpURLConnection.HTTP_OK + || response.body == null) { + return false; + } + final ServerConfiguration configuration; + try { + configuration = + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( + response.body); + } catch (final IOException | RuntimeException e) { + LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); + return false; + } + if (configuration == null) { + return false; + } + FeatureFlaggingGateway.dispatch(configuration); + updateEtag(response.etag); + return true; + } + + private static boolean isRetryableStatus(final int status) { + return status == HttpURLConnection.HTTP_CLIENT_TIMEOUT + || status == 429 + || (status >= 500 && status <= 599); + } + + private void updateEtag(final String nextEtag) { + if (nextEtag != null && !nextEtag.trim().isEmpty()) { + etag = nextEtag; + } + } + + static HttpUrl endpoint(final Config config) { + final String endpoint = datadogApiServerDistributionEndpoint(config); + final HttpUrl parsed = HttpUrl.parse(endpoint); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); + } + return parsed; + } + + private static String datadogApiServerDistributionEndpoint(final Config config) { + final StringBuilder endpoint = + new StringBuilder("https://api.") + .append(config.getSite().toLowerCase(Locale.ROOT)) + .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); + final String env = config.getEnv(); + if (env != null && !env.isEmpty()) { + endpoint.append("?dd_env=").append(urlEncode(env)); + } + return endpoint.toString(); + } + + private static String urlEncode(final String value) { + try { + return URLEncoder.encode(value, "UTF-8"); + } catch (final IOException e) { + throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + } + } + + static long millis(final int seconds) { + return TimeUnit.SECONDS.toMillis(seconds); + } + + static long retryDelayMillis( + final long pollIntervalMillis, final int attempt, final double jitter) { + final long baseDelay; + if (attempt == 1) { + baseDelay = clamp(pollIntervalMillis / 6, FIRST_RETRY_MIN_MILLIS, FIRST_RETRY_MAX_MILLIS); + } else if (attempt == 2) { + baseDelay = clamp(pollIntervalMillis / 3, SECOND_RETRY_MIN_MILLIS, SECOND_RETRY_MAX_MILLIS); + } else { + throw new IllegalArgumentException("Unsupported Feature Flagging retry attempt: " + attempt); + } + return Math.max(1, Math.round(baseDelay * jitter)); + } + + private static long clamp(final long value, final long minimum, final long maximum) { + return Math.max(minimum, Math.min(maximum, value)); + } + + interface UfcHttpClient { + UfcHttpResponse fetch(HttpUrl endpoint, Config config, String etag) throws IOException; + + void cancel(); + } + + interface RetrySleeper { + void sleep(long delayMillis) throws InterruptedException; + } + + static final class UfcHttpResponse { + final int status; + final String etag; + final byte[] body; + + UfcHttpResponse(final int status, final String etag, final byte[] body) { + this.status = status; + this.etag = etag; + this.body = body; + } + } + + static final class OkHttpUfcHttpClient implements UfcHttpClient { + private final OkHttpClient httpClient; + private final AtomicReference activeCall = new AtomicReference<>(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + + OkHttpUfcHttpClient(final OkHttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final String etag) + throws IOException { + final Map headers = new HashMap<>(); + if (etag != null) { + headers.put("If-None-Match", etag); + } + final Request request = prepareRequest(endpoint, headers, config, true).get().build(); + final Call call = httpClient.newCall(request); + if (!activeCall.compareAndSet(null, call)) { + throw new IllegalStateException("Feature Flagging HTTP request already in flight"); + } + if (cancelled.get()) { + call.cancel(); + } + try (Response response = call.execute()) { + final ResponseBody responseBody = response.body(); + return new UfcHttpResponse(response.code(), response.header("ETag"), responseBody.bytes()); + } finally { + activeCall.compareAndSet(call, null); + } + } + + @Override + public void cancel() { + cancelled.set(true); + final Call call = activeCall.get(); + if (call != null) { + call.cancel(); + } + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java new file mode 100644 index 00000000000..4d5e0651c8e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -0,0 +1,805 @@ +package com.datadog.featureflag; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AgentlessConfigurationSourceTest { + private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/server-distribution"; + + @Mock private FeatureFlaggingGateway.ConfigListener listener; + + @AfterEach + void cleanup() { + FeatureFlaggingGateway.removeConfigListener(listener); + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + @Test + void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { + final Config config = config("datad0g.com", "staging env"); + + assertEquals( + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void derivesDatadogApiServerDistributionEndpointWithoutEnv() { + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", "")).toString()); + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); + } + + @Test + void rejectsInvalidDatadogApiServerDistributionEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> AgentlessConfigurationSource.endpoint(config("datadoghq.com:bad", ""))); + } + + @Test + void defaultConstructorBuildsHttpClientFromConfig() { + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource(config("datad0g.com", "staging")); + + service.close(); + } + + @Test + void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .addHeader("ETag", "etag-b") + .send(emptyConfig()))))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), "etag-a"); + + assertEquals(HttpURLConnection.HTTP_OK, response.status); + assertEquals("etag-b", response.etag); + assertEquals(emptyConfig(), new String(response.body, UTF_8)); + assertEquals("test-api-key", server.getLastRequest().getHeader("DD-API-KEY")); + assertEquals("etag-a", server.getLastRequest().getHeader("If-None-Match")); + assertEquals("java", server.getLastRequest().getHeader("Datadog-Meta-Lang")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .status(HttpURLConnection.HTTP_NO_CONTENT) + .send())))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null); + + assertEquals(HttpURLConnection.HTTP_NO_CONTENT, response.status); + assertNull(response.etag); + assertEquals(0, response.body.length); + assertNull(server.getLastRequest().getHeader("If-None-Match")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientCancellationInterruptsInFlightRequest() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> { + requestStarted.countDown(); + assertTrue(releaseRequest.await(1, TimeUnit.SECONDS)); + api.getResponse().send(emptyConfig()); + })))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + + try { + final Future response = + runner.submit( + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + assertThrows( + IllegalStateException.class, + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + + client.cancel(); + + final ExecutionException failure = + assertThrows(ExecutionException.class, () -> response.get(1, TimeUnit.SECONDS)); + assertInstanceOf(IOException.class, failure.getCause()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientCancellationBeforeFetchPreventsRequest() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> h.get(CONFIG_PATH, api -> api.getResponse().send(emptyConfig()))))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + client.cancel(); + + assertThrows( + IOException.class, + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientTimesOutDelayedResponse() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> { + TimeUnit.MILLISECONDS.sleep(500); + api.getResponse().send(emptyConfig()); + })))) { + final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); + final OkHttpClient httpClient = OkHttpUtils.buildHttpClient(endpoint, 50); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + assertThrows(IOException.class, () -> client.fetch(endpoint, config(), null)); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("test-api-key", client.requests.get(0).apiKey); + assertNull(client.requests.get(0).etag); + } + + @Test + void ignoresBlankEtag() throws Exception { + final FakeClient client = + new FakeClient(response(200, " ", emptyConfig()), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { + final FakeClient client = + new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + } + + @Test + void coldNotModifiedDoesNotEstablishEtag() throws Exception { + final FakeClient client = + new FakeClient(response(304, "etag-cold", null), response(200, "etag-warm", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void failedGatewayDispatchDoesNotAdvanceEtag() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + final FeatureFlaggingGateway.ConfigListener failingListener = + configuration -> { + throw new IllegalStateException("listener rejected configuration"); + }; + FeatureFlaggingGateway.addConfigListener(failingListener); + + try { + assertThrows(IllegalStateException.class, service::pollOnce); + FeatureFlaggingGateway.removeConfigListener(failingListener); + + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + } finally { + FeatureFlaggingGateway.removeConfigListener(failingListener); + } + } + + @Test + void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-good", emptyConfig()), + response(401, null, null), + response(200, null, "{not-json}"), + response(200, null, "{\"flags\":[]}")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-good", client.requests.get(1).etag); + assertEquals("etag-good", client.requests.get(2).etag); + assertEquals("etag-good", client.requests.get(3).etag); + } + + @Test + void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { + final FakeClient client = + new FakeClient( + response(403, null, null), + response(404, null, null), + response(600, null, null), + response(200, null, null), + response(200, null, "null")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verifyNoInteractions(listener); + } + + @Test + void retriesTimeoutBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesClientTimeoutAndRateLimitStatusBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + response(408, null, null), + response(200, "etag-a", emptyConfig()), + response(429, null, null), + response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertEquals(4, client.calls.get()); + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { + final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(2, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void givesUpAfterRetryableFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + response(503, null, null), response(503, null, null), response(503, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void givesUpAfterIoFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void usesIntervalAwareRetryBackoff() throws Exception { + final List delays = new ArrayList<>(); + final FakeClient client = + new FakeClient( + response(503, null, null), + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client, delays::add, () -> 1.0); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(java.util.Arrays.asList(5_000L, 10_000L), delays); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void clampsAndJittersRetryBackoff() { + assertEquals(2_000, AgentlessConfigurationSource.retryDelayMillis(1_000, 1, 1.0)); + assertEquals(5_000, AgentlessConfigurationSource.retryDelayMillis(1_000, 2, 1.0)); + assertEquals(10_000, AgentlessConfigurationSource.retryDelayMillis(600_000, 1, 1.0)); + assertEquals(30_000, AgentlessConfigurationSource.retryDelayMillis(600_000, 2, 1.0)); + assertEquals(6_000, AgentlessConfigurationSource.retryDelayMillis(30_000, 1, 1.2)); + assertThrows( + IllegalArgumentException.class, + () -> AgentlessConfigurationSource.retryDelayMillis(30_000, 3, 1.0)); + } + + @Test + void rejectsOverlappingPolls() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newFixedThreadPool(2); + + try { + final Future first = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + final Future second = runner.submit(service::pollOnce); + + assertFalse(second.get(1, TimeUnit.SECONDS)); + releaseRequest.countDown(); + assertTrue(first.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void initSchedulesPollAndCloseCancelsFuture() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 60_000, + client, + Executors.newSingleThreadScheduledExecutor()); + FeatureFlaggingGateway.addConfigListener(listener); + + service.init(); + awaitCalls(client, 1); + service.close(); + + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void repeatedInitStartsOnlyOnePoller() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.init(); + service.init(); + awaitCalls(client, 1); + service.close(); + + assertEquals(1, client.calls.get()); + } + + @Test + void closeCancelsInFlightRequestAndIgnoresLateSuccess() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + final Future poll = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertFalse(poll.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.cancelCalls.get()); + assertEquals(1, client.calls.get()); + verifyNoInteractions(listener); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closeDuringIoFailurePreventsRetry() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = + new FakeClient(new SocketTimeoutException("slow HTTP configuration source")); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + + try { + final Future poll = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertFalse(poll.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closeInterruptsRetryBackoff() throws Exception { + final CountDownLatch backoffStarted = new CountDownLatch(1); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> { + backoffStarted.countDown(); + TimeUnit.MINUTES.sleep(1); + }, + () -> 1.0); + + service.init(); + assertTrue(backoffStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } + + @Test + void closePreventsFurtherPolls() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.close(); + + assertFalse(service.pollOnce()); + assertEquals(0, client.calls.get()); + } + + @Test + void initAfterCloseDoesNotSchedulePoll() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.close(); + service.init(); + + assertEquals(0, client.calls.get()); + } + + @Nested + class SystemTestParity { + @Test + void preservesSystemTestSourceTransitionsAndLastKnownGoodState() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), + response(304, "etag-must-not-replace-a", null), + response(509, null, null), + response(200, "etag-b", emptyConfig()), + response(200, "etag-c", "{not-json}"), + response(401, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + assertEquals("etag-a", client.requests.get(2).etag); + assertEquals("etag-a", client.requests.get(3).etag); + assertEquals("etag-b", client.requests.get(4).etag); + assertEquals("etag-b", client.requests.get(5).etag); + } + } + + private static AgentlessConfigurationSource service(final FakeClient client) { + return service(client, delay -> {}, () -> 1.0); + } + + private static AgentlessConfigurationSource service( + final FakeClient client, + final AgentlessConfigurationSource.RetrySleeper retrySleeper, + final java.util.function.DoubleSupplier jitter) { + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + return new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + retrySleeper, + jitter); + } + + private static Config config() { + return config("datadoghq.com", ""); + } + + private static Config config(final String site, final String env) { + final Config config = mock(Config.class); + lenient() + .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) + .thenReturn(30); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) + .thenReturn(2); + lenient().when(config.getApiKey()).thenReturn("test-api-key"); + lenient().when(config.getSite()).thenReturn(site); + lenient().when(config.getEnv()).thenReturn(env); + return config; + } + + private static AgentlessConfigurationSource.UfcHttpResponse response( + final int status, final String etag, final String body) { + return new AgentlessConfigurationSource.UfcHttpResponse( + status, etag, body == null ? null : body.getBytes(UTF_8)); + } + + private static String emptyConfig() { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + + private static void awaitCalls(final FakeClient client, final int count) throws Exception { + for (int i = 0; i < 100; i++) { + if (client.calls.get() >= count) { + return; + } + TimeUnit.MILLISECONDS.sleep(10); + } + assertEquals(count, client.calls.get()); + } + + private static final class FakeClient implements AgentlessConfigurationSource.UfcHttpClient { + private final AtomicInteger calls = new AtomicInteger(); + private final AtomicInteger cancelCalls = new AtomicInteger(); + private final List requests = new ArrayList<>(); + private final BlockingQueue responses = new LinkedBlockingQueue<>(); + private CountDownLatch requestStarted; + private CountDownLatch releaseRequest; + + private FakeClient(final Object... responses) { + for (final Object response : responses) { + this.responses.add(response); + } + } + + private void block(final CountDownLatch requestStarted, final CountDownLatch releaseRequest) { + this.requestStarted = requestStarted; + this.releaseRequest = releaseRequest; + } + + @Override + public AgentlessConfigurationSource.UfcHttpResponse fetch( + final HttpUrl endpoint, final Config config, final String etag) throws IOException { + calls.incrementAndGet(); + requests.add(new Request(config.getApiKey(), etag)); + if (requestStarted != null) { + requestStarted.countDown(); + } + if (releaseRequest != null) { + await(releaseRequest); + } + final Object response = responses.remove(); + if (response instanceof IOException) { + throw (IOException) response; + } + if (response instanceof RuntimeException) { + throw (RuntimeException) response; + } + return (AgentlessConfigurationSource.UfcHttpResponse) response; + } + + @Override + public void cancel() { + cancelCalls.incrementAndGet(); + if (releaseRequest != null) { + releaseRequest.countDown(); + } + } + + private static void await(final CountDownLatch latch) throws IOException { + try { + if (!latch.await(1, TimeUnit.SECONDS)) { + throw new SocketTimeoutException("test request did not release"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + } + + private static final class Request { + private final String apiKey; + private final String etag; + + private Request(final String apiKey, final String etag) { + this.apiKey = apiKey; + this.etag = etag; + } + } +} From d378350cf1916ddd20a4cdde14393af9a70001de Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:58:40 -0400 Subject: [PATCH 03/26] Support custom agentless UFC endpoints --- .../AgentlessConfigurationSource.java | 22 +++++++++++- .../AgentlessConfigurationSourceTest.java | 36 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index b3c58631d3a..07deeac0ffc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -242,7 +242,11 @@ private void updateEtag(final String nextEtag) { } static HttpUrl endpoint(final Config config) { - final String endpoint = datadogApiServerDistributionEndpoint(config); + final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl(); + final String endpoint = + configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredBaseUrl(configuredBaseUrl); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { throw new IllegalArgumentException( @@ -251,6 +255,22 @@ static HttpUrl endpoint(final Config config) { return parsed; } + private static String endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { + final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim()); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + configuredBaseUrl); + } + if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) { + return parsed + .newBuilder() + .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) + .build() + .toString(); + } + return parsed.toString(); + } + private static String datadogApiServerDistributionEndpoint(final Config config) { final StringBuilder endpoint = new StringBuilder("https://api.") diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 4d5e0651c8e..c79de87502d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -74,6 +74,41 @@ void derivesDatadogApiServerDistributionEndpointWithoutEnv() { AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); } + @Test + void appendsServerDistributionPathToConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080"); + + assertEquals( + "http://mock-backend:8080/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void usesConfiguredAgentlessEndpointWithPathUnchanged() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080/custom/ufc?tenant=test"); + + assertEquals( + "http://mock-backend:8080/custom/ufc?tenant=test", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void rejectsInvalidConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("not a URL"); + + assertThrows( + IllegalArgumentException.class, () -> AgentlessConfigurationSource.endpoint(config)); + } + @Test void rejectsInvalidDatadogApiServerDistributionEndpoint() { assertThrows( @@ -702,6 +737,7 @@ private static Config config(final String site, final String env) { lenient() .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) .thenReturn(2); + lenient().when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(null); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); lenient().when(config.getEnv()).thenReturn(env); From f9e70e67c4f16185dbf09bbb458c5572684fb2b2 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 11 Jul 2026 00:01:22 -0400 Subject: [PATCH 04/26] Select the Feature Flagging configuration source --- .../java/datadog/trace/bootstrap/Agent.java | 17 ++++ .../AgentFeatureFlaggingLifecycleTest.java | 48 ++++++++++ .../featureflag/FeatureFlaggingSystem.java | 94 ++++++++++++++---- .../FeatureFlaggingSystemTest.java | 95 +++++++++++++++++++ .../feature-flagging-api/README.md | 10 +- 5 files changed, 246 insertions(+), 18 deletions(-) create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 876a07f1fd1..056e47147f9 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -528,6 +528,9 @@ public static void shutdown(final boolean sync) { if (flareEnabled) { stopFlarePoller(); } + if (featureFlaggingEnabled) { + shutdownFeatureFlagging(AGENT_CLASSLOADER); + } if (agentlessLogSubmissionEnabled) { shutdownLogsIntake(); @@ -1179,6 +1182,20 @@ private static void maybeStartFeatureFlagging(final Class scoClass, final Obj } } + static void shutdownFeatureFlagging(final ClassLoader agentClassLoader) { + if (agentClassLoader == null) { + return; + } + try { + final Class ffSysClass = + agentClassLoader.loadClass("com.datadog.featureflag.FeatureFlaggingSystem"); + final Method stopMethod = ffSysClass.getMethod("stop"); + stopMethod.invoke(null); + } catch (final Throwable e) { + log.warn("Unable to stop Feature Flagging subsystem", e); + } + } + private static void maybeInstallLogsIntake(Class scoClass, Object sco) { if (agentlessLogSubmissionEnabled || appLogsCollectionEnabled) { StaticEventLogger.begin("Logs Intake"); diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java new file mode 100644 index 00000000000..75e9987c4da --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java @@ -0,0 +1,48 @@ +package datadog.trace.bootstrap; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AgentFeatureFlaggingLifecycleTest { + + @BeforeEach + void reset() { + FakeFeatureFlaggingSystem.stopCalls.set(0); + } + + @Test + void shutdownInvokesFeatureFlaggingSystemStopThroughAgentClassLoader() { + final ClassLoader classLoader = + new ClassLoader(null) { + @Override + public Class loadClass(final String name) throws ClassNotFoundException { + if ("com.datadog.featureflag.FeatureFlaggingSystem".equals(name)) { + return FakeFeatureFlaggingSystem.class; + } + return super.loadClass(name); + } + }; + + Agent.shutdownFeatureFlagging(classLoader); + + assertEquals(1, FakeFeatureFlaggingSystem.stopCalls.get()); + } + + @Test + void shutdownIsNoopBeforeAgentClassLoaderExists() { + Agent.shutdownFeatureFlagging(null); + + assertEquals(0, FakeFeatureFlaggingSystem.stopCalls.get()); + } + + public static final class FakeFeatureFlaggingSystem { + private static final AtomicInteger stopCalls = new AtomicInteger(); + + public static void stop() { + stopCalls.incrementAndGet(); + } + } +} diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 2011d31e83d..f52ef257450 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -14,31 +14,93 @@ public class FeatureFlaggingSystem { private FeatureFlaggingSystem() {} - public static void start(final SharedCommunicationObjects sco) { + public static synchronized void start(final SharedCommunicationObjects sco) { + if (CONFIG_SERVICE != null || EXPOSURE_WRITER != null) { + LOGGER.debug("Feature Flagging system already started"); + return; + } LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + final ConfigurationSourceService configService = createConfigurationSourceService(sco, config); + final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); + initialize(configService, exposureWriter); + + LOGGER.debug("Feature Flagging system started"); + } - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); + static void initialize( + final ConfigurationSourceService configService, final ExposureWriter exposureWriter) { + try { + if (configService != null) { + configService.init(); + } + exposureWriter.init(); + CONFIG_SERVICE = configService; + EXPOSURE_WRITER = exposureWriter; + } catch (final RuntimeException | Error e) { + exposureWriter.close(); + if (configService != null) { + configService.close(); + } + throw e; } - CONFIG_SERVICE = new RemoteConfigServiceImpl(sco, config); - CONFIG_SERVICE.init(); + } - EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); - EXPOSURE_WRITER.init(); + static ConfigurationSourceService createConfigurationSourceService( + final SharedCommunicationObjects sco, final Config config) { + final ConfigurationSource configurationSource = + ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); - LOGGER.debug("Feature Flagging system started"); + if (configurationSource == ConfigurationSource.REMOTE_CONFIG) { + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + return new RemoteConfigServiceImpl(sco, config); + } + if (configurationSource == ConfigurationSource.AGENTLESS) { + return new AgentlessConfigurationSource(config); + } + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + return null; } - public static void stop() { - if (EXPOSURE_WRITER != null) { - EXPOSURE_WRITER.close(); - EXPOSURE_WRITER = null; - } - if (CONFIG_SERVICE != null) { - CONFIG_SERVICE.close(); - CONFIG_SERVICE = null; + public static synchronized void stop() { + final ExposureWriter exposureWriter = EXPOSURE_WRITER; + final ConfigurationSourceService configService = CONFIG_SERVICE; + EXPOSURE_WRITER = null; + CONFIG_SERVICE = null; + try { + if (exposureWriter != null) { + exposureWriter.close(); + } + } finally { + if (configService != null) { + configService.close(); + } } LOGGER.debug("Feature Flagging system stopped"); } + + private enum ConfigurationSource { + AGENTLESS("agentless"), + REMOTE_CONFIG("remote_config"), + OFFLINE("offline"); + + private final String value; + + ConfigurationSource(final String value) { + this.value = value; + } + + private static ConfigurationSource from(final String value) { + for (final ConfigurationSource source : values()) { + if (source.value.equals(value)) { + return source; + } + } + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + value); + } + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 5b088d99b9d..82ce01e4929 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -2,9 +2,13 @@ import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -37,12 +41,14 @@ void testFeatureFlagSystemInitialization() { sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingSystem.start(sharedCommunicationObjects); FeatureFlaggingSystem.start(sharedCommunicationObjects); verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + FeatureFlaggingSystem.stop(); FeatureFlaggingSystem.stop(); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); @@ -64,4 +70,93 @@ void testThatRemoteConfigIsRequired() { FeatureFlaggingSystem.stop(); } } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { + assertInstanceOf( + AgentlessConfigurationSource.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") + void explicitRemoteConfigUsesRemoteConfigService() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))) + .thenReturn(mock(ConfigurationPoller.class)); + + assertInstanceOf( + RemoteConfigServiceImpl.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects, Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") + void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { + assertThrows( + IllegalArgumentException.class, + () -> + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void offlineConfigurationSourceDoesNotStartNetworkSource() { + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void startWithOfflineConfigurationSourceSkipsConfigService() { + try { + assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); + } finally { + FeatureFlaggingSystem.stop(); + } + } + + @Test + void initializationFailureClosesConfigurationSourceAndExposureWriter() { + ConfigurationSourceService configService = mock(ConfigurationSourceService.class); + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + + assertThrows( + IllegalStateException.class, + () -> FeatureFlaggingSystem.initialize(configService, exposureWriter)); + + verify(configService).init(); + verify(configService).close(); + verify(exposureWriter).close(); + } + + @Test + void initializationFailureWithoutConfigurationSourceClosesExposureWriter() { + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + + assertThrows( + IllegalStateException.class, () -> FeatureFlaggingSystem.initialize(null, exposureWriter)); + + verify(exposureWriter).close(); + } + + private static SharedCommunicationObjects sharedCommunicationObjects() { + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + return sharedCommunicationObjects; + } } diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 47733020559..ca9dc9e7d0a 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -85,5 +85,11 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc ## Requirements - Java 11+ -- Datadog Agent with Remote Configuration enabled -- `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=true` +- `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless + backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a + different HTTP backend while keeping agentless delivery semantics. A bare + host uses the standard server-distribution path; a URL with a path is used as + the exact UFC endpoint. `remote_config` uses the existing Agent Remote + Configuration path. `offline` is reserved for startup-provided UFC bytes; + until those bytes are implemented, no network source starts and evaluations + use defaults. From 17a02ad627bef6028356acc8485039458c55660d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 13 Jul 2026 08:15:22 -0400 Subject: [PATCH 05/26] Warn on agentless authentication failures --- .../feature-flagging-lib/build.gradle.kts | 1 + .../AgentlessConfigurationSource.java | 40 ++++++++++++++++--- .../AgentlessConfigurationSourceTest.java | 34 ++++++++++++++++ 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 548f5e26d2b..d1fb611fe5e 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { api(project(":communication")) implementation(project(":internal-api")) api(project(":products:feature-flagging:feature-flagging-bootstrap")) + implementation(project(":utils:logging-utils")) api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 07deeac0ffc..0b3ce685564 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -4,6 +4,7 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; import datadog.communication.http.OkHttpUtils; +import datadog.logging.RatelimitedLogger; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; @@ -38,6 +39,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; + private static final int MINUTES_BETWEEN_AUTH_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; private static final long FIRST_RETRY_MAX_MILLIS = 10_000; private static final long SECOND_RETRY_MIN_MILLIS = 5_000; @@ -51,6 +53,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private final ScheduledExecutorService executor; private final RetrySleeper retrySleeper; private final DoubleSupplier jitter; + private final RatelimitedLogger ratelimitedLogger; private final Object lifecycleLock = new Object(); private final AtomicBoolean polling = new AtomicBoolean(); private volatile boolean closed; @@ -73,7 +76,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint Executors.newSingleThreadScheduledExecutor( new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), TimeUnit.MILLISECONDS::sleep, - () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)); + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER), + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -89,7 +93,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint client, executor, TimeUnit.MILLISECONDS::sleep, - () -> 1.0); + () -> 1.0, + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -100,6 +105,26 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint final ScheduledExecutorService executor, final RetrySleeper retrySleeper, final DoubleSupplier jitter) { + this( + endpoint, + config, + pollIntervalMillis, + client, + executor, + retrySleeper, + jitter, + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter, + final RatelimitedLogger ratelimitedLogger) { this.endpoint = endpoint; this.config = config; this.pollIntervalMillis = pollIntervalMillis; @@ -107,6 +132,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint this.executor = executor; this.retrySleeper = retrySleeper; this.jitter = jitter; + this.ratelimitedLogger = ratelimitedLogger; } @Override @@ -207,9 +233,13 @@ private boolean apply(final UfcHttpResponse response) { return true; } if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED - || response.status == HttpURLConnection.HTTP_FORBIDDEN - || response.status != HttpURLConnection.HTTP_OK - || response.body == null) { + || response.status == HttpURLConnection.HTTP_FORBIDDEN) { + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + response.status); + return false; + } + if (response.status != HttpURLConnection.HTTP_OK || response.body == null) { return false; } final ServerConfiguration configuration; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index c79de87502d..0ecc790b3e4 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import datadog.communication.http.OkHttpUtils; +import datadog.logging.RatelimitedLogger; import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; @@ -413,6 +414,39 @@ void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { verifyNoInteractions(listener); } + @Test + void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); + final FakeClient client = new FakeClient(response(401, null, null), response(403, null, null)); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); + + try { + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + HttpURLConnection.HTTP_UNAUTHORIZED); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + HttpURLConnection.HTTP_FORBIDDEN); + } finally { + service.close(); + } + } + @Test void retriesTimeoutBeforeApplyingConfig() throws Exception { final FakeClient client = From 505753954704653f1b65dc579da8832fa4444408 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 14 Jul 2026 22:24:28 -0400 Subject: [PATCH 06/26] fix(ffe): handle nullable agentless response bodies --- .../datadog/trace/api/ConfigDefaults.java | 7 ++--- .../AgentlessConfigurationSource.java | 16 ++++++---- .../AgentlessConfigurationSourceTest.java | 30 +++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 532e9f47d71..0c008f41094 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -45,14 +45,13 @@ public final class ConfigDefaults { public static final String DEFAULT_SERVLET_ROOT_CONTEXT_SERVICE_NAME = "root-servlet"; public static final String DEFAULT_AGENT_WRITER_TYPE = "DDAgentWriter"; public static final boolean DEFAULT_STARTUP_LOGS_ENABLED = true; - - static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; - static final String DEFAULT_SITE = "datadoghq.com"; - public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; + static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; + static final String DEFAULT_SITE = "datadoghq.com"; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 0b3ce685564..be381893ccf 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.DoubleSupplier; +import javax.annotation.Nullable; import okhttp3.Call; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -354,10 +355,10 @@ interface RetrySleeper { static final class UfcHttpResponse { final int status; - final String etag; - final byte[] body; + @Nullable final String etag; + @Nullable final byte[] body; - UfcHttpResponse(final int status, final String etag, final byte[] body) { + UfcHttpResponse(final int status, @Nullable final String etag, @Nullable final byte[] body) { this.status = status; this.etag = etag; this.body = body; @@ -388,9 +389,12 @@ public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final if (cancelled.get()) { call.cancel(); } - try (Response response = call.execute()) { - final ResponseBody responseBody = response.body(); - return new UfcHttpResponse(response.code(), response.header("ETag"), responseBody.bytes()); + try { + final Response response = call.execute(); + try (ResponseBody responseBody = response.body()) { + final byte[] body = responseBody != null ? responseBody.bytes() : null; + return new UfcHttpResponse(response.code(), response.header("ETag"), body); + } } finally { activeCall.compareAndSet(call, null); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 0ecc790b3e4..4d8714c0e67 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; @@ -35,8 +36,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.Call; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Response; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -191,6 +195,32 @@ void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { } } + @Test + void httpClientAdapterPreservesMissingResponseBody() throws Exception { + final OkHttpClient httpClient = mock(OkHttpClient.class); + final Call call = mock(Call.class); + final HttpUrl endpoint = HttpUrl.get("http://localhost"); + final okhttp3.Request request = new okhttp3.Request.Builder().url(endpoint).build(); + final Response okHttpResponse = + new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(HttpURLConnection.HTTP_OK) + .message("OK") + .build(); + when(httpClient.newCall(any())).thenReturn(call); + when(call.execute()).thenReturn(okHttpResponse); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(endpoint, config(), null); + + assertEquals(HttpURLConnection.HTTP_OK, response.status); + assertNull(response.etag); + assertNull(response.body); + } + @Test void realHttpClientCancellationInterruptsInFlightRequest() throws Exception { final CountDownLatch requestStarted = new CountDownLatch(1); From ffbf97023f38675007e899bd54b0bd56307e039d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 14:36:02 -0400 Subject: [PATCH 07/26] Address agentless poller review feedback --- .../feature-flagging-api/README.md | 7 +- .../AgentlessConfigurationSource.java | 28 +++- .../AgentlessConfigurationSourceTest.java | 155 ++++++++++++++++-- 3 files changed, 169 insertions(+), 21 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index ca9dc9e7d0a..ae861d07afd 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -89,7 +89,12 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a different HTTP backend while keeping agentless delivery semantics. A bare host uses the standard server-distribution path; a URL with a path is used as - the exact UFC endpoint. `remote_config` uses the existing Agent Remote + the exact UFC endpoint. Configured URLs are opaque: the SDK does not add the + Datadog-managed `dd_env` query parameter, so custom backends must include any + required tenant or environment scope in the configured URL. The derived + Datadog-managed endpoint is intended for supported commercial sites; use an + explicit base URL elsewhere. Agentless responses do not have an SDK-imposed + payload-size limit. `remote_config` uses the existing Agent Remote Configuration path. `offline` is reserved for startup-provided UFC bytes; until those bytes are implemented, no network source starts and evaluations use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index be381893ccf..e9f0786d76a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -40,7 +40,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; - private static final int MINUTES_BETWEEN_AUTH_WARNINGS = 5; + private static final int MINUTES_BETWEEN_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; private static final long FIRST_RETRY_MAX_MILLIS = 10_000; private static final long SECOND_RETRY_MIN_MILLIS = 5_000; @@ -78,7 +78,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), TimeUnit.MILLISECONDS::sleep, () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER), - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -95,7 +95,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint executor, TimeUnit.MILLISECONDS::sleep, () -> 1.0, - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -114,7 +114,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint executor, retrySleeper, jitter, - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -192,11 +192,18 @@ private boolean fetchAndApply() { if (closed) { return false; } - if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { - if (!waitBeforeRetry(attempt)) { - return false; + if (isRetryableStatus(response.status)) { + if (attempt < MAX_ATTEMPTS) { + if (!waitBeforeRetry(attempt)) { + return false; + } + continue; } - continue; + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", + MAX_ATTEMPTS, + response.status); + return false; } synchronized (lifecycleLock) { return !closed && apply(response); @@ -206,7 +213,10 @@ private boolean fetchAndApply() { return false; } if (attempt == MAX_ATTEMPTS) { - LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint request failed after {} attempts", + MAX_ATTEMPTS, + e); return false; } if (!waitBeforeRetry(attempt)) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 4d8714c0e67..a5b6dc84bfb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -45,6 +45,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -163,6 +164,41 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { } } + @Test + void downloadsAndAppliesLargeUfcWithoutPayloadLimit() throws Exception { + final int flagCount = 5_000; + final String largeConfig = largeConfig(flagCount); + assertTrue(largeConfig.getBytes(UTF_8).length > 500_000); + + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.get(CONFIG_PATH, api -> api.getResponse().send(largeConfig))))) { + final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + endpoint, + config(), + 30_000, + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient), + Executors.newSingleThreadScheduledExecutor()); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + assertTrue(service.pollOnce()); + + verify(listener).accept(configuration.capture()); + assertEquals(flagCount, configuration.getValue().flags.size()); + } finally { + service.close(); + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + @Test void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { try (JavaTestHttpServer server = @@ -524,33 +560,73 @@ void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { } @Test - void givesUpAfterRetryableFailuresAreExhausted() throws Exception { + void warnsRateLimitedAfterRetryableFailuresAreExhausted() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); final FakeClient client = new FakeClient( response(503, null, null), response(503, null, null), response(503, null, null)); - final AgentlessConfigurationSource service = service(client); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); - assertFalse(service.pollOnce()); + try { + assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); - verifyNoInteractions(listener); + assertEquals(3, client.calls.get()); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", 3, 503); + verifyNoInteractions(listener); + } finally { + service.close(); + } } @Test - void givesUpAfterIoFailuresAreExhausted() throws Exception { + void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); + final SocketTimeoutException finalFailure = + new SocketTimeoutException("slow HTTP configuration source"); final FakeClient client = new FakeClient( new SocketTimeoutException("slow HTTP configuration source"), new SocketTimeoutException("slow HTTP configuration source"), - new SocketTimeoutException("slow HTTP configuration source")); - final AgentlessConfigurationSource service = service(client); + finalFailure); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); - assertFalse(service.pollOnce()); + try { + assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); - verifyNoInteractions(listener); + assertEquals(3, client.calls.get()); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint request failed after {} attempts", + 3, + finalFailure); + verifyNoInteractions(listener); + } finally { + service.close(); + } } @Test @@ -638,6 +714,39 @@ void repeatedInitStartsOnlyOnePoller() throws Exception { assertEquals(1, client.calls.get()); } + @Test + void scheduledPollContinuesAfterListenerRuntimeException() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 10, + client, + Executors.newSingleThreadScheduledExecutor()); + final AtomicInteger listenerCalls = new AtomicInteger(); + final FeatureFlaggingGateway.ConfigListener flakyListener = + configuration -> { + if (listenerCalls.incrementAndGet() == 1) { + throw new IllegalStateException("listener rejected first configuration"); + } + }; + FeatureFlaggingGateway.addConfigListener(flakyListener); + + try { + service.init(); + awaitCalls(client, 2); + + assertEquals(2, listenerCalls.get()); + assertNull(client.requests.get(1).etag); + } finally { + service.close(); + FeatureFlaggingGateway.removeConfigListener(flakyListener); + } + } + @Test void closeCancelsInFlightRequestAndIgnoresLateSuccess() throws Exception { final CountDownLatch requestStarted = new CountDownLatch(1); @@ -823,6 +932,30 @@ private static String emptyConfig() { + "}"; } + private static String largeConfig(final int flagCount) { + final StringBuilder json = + new StringBuilder( + "{\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Large Test\"}," + + "\"flags\":{"); + for (int index = 0; index < flagCount; index++) { + if (index > 0) { + json.append(','); + } + final String flagKey = "large-flag-" + index; + json.append('"') + .append(flagKey) + .append("\":{\"key\":\"") + .append(flagKey) + .append( + "\",\"enabled\":true,\"variationType\":\"STRING\"," + + "\"variations\":{\"on\":{\"key\":\"on\",\"value\":\"on\"}}," + + "\"allocations\":[]}"); + } + return json.append("}}").toString(); + } + private static void awaitCalls(final FakeClient client, final int count) throws Exception { for (int i = 0; i < 100; i++) { if (client.calls.get() >= count) { From 3b9b97c470124442a93f3f40e186d7483bfc2dcd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 14:40:14 -0400 Subject: [PATCH 08/26] Fix feature flag config test imports --- .../src/test/groovy/datadog/trace/api/ConfigTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 953c5ced8f3..a9046475a7c 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -57,8 +57,8 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_METRICS_CONFIGS From eaaea2762128decffb6fa20fc7b1956d17671362 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 15:52:39 -0400 Subject: [PATCH 09/26] Address additional agentless source review feedback --- .../datadog/trace/api/ConfigDefaults.java | 6 +-- .../datadog/trace/api/ConfigTest.groovy | 22 ++++++++ .../featureflag/FeatureFlaggingSystem.java | 9 ++-- .../FeatureFlaggingSystemTest.java | 14 ++++++ .../feature-flagging-lib/build.gradle.kts | 3 -- .../AgentlessConfigurationSource.java | 50 ++++++------------- .../AgentlessConfigurationSourceTest.java | 21 +++++++- 7 files changed, 81 insertions(+), 44 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 0c008f41094..327a1555846 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -45,12 +45,12 @@ public final class ConfigDefaults { public static final String DEFAULT_SERVLET_ROOT_CONTEXT_SERVICE_NAME = "root-servlet"; public static final String DEFAULT_AGENT_WRITER_TYPE = "DDAgentWriter"; public static final boolean DEFAULT_STARTUP_LOGS_ENABLED = true; - public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; - public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; - public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; + static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index a9046475a7c..63b442f9a48 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -57,6 +57,7 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD @@ -3497,6 +3498,27 @@ class ConfigTest extends DDSpecification { config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == 4 } + def "feature flag configuration source normalizes #value to #expected"() { + setup: + Properties properties = new Properties() + if (value != null) { + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE, value) + } + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSource == expected + + where: + value | expected + null | "agentless" + "" | "agentless" + " " | "agentless" + " ReMoTe_ConFiG " | "remote_config" + } + def "agentless feature flag timing falls back for non-positive values"() { setup: Properties properties = new Properties() diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 6d8fd4cb720..5c9cb20540e 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -47,9 +47,12 @@ static void initialize( CONFIG_SERVICE = configService; EXPOSURE_WRITER = exposureWriter; } catch (final RuntimeException | Error e) { - exposureWriter.close(); - if (configService != null) { - configService.close(); + try { + exposureWriter.close(); + } finally { + if (configService != null) { + configService.close(); + } } throw e; } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 4b241fe3cb2..f6fc7516c0b 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -149,6 +149,20 @@ void initializationFailureWithoutConfigurationSourceClosesExposureWriter() { verify(exposureWriter).close(); } + @Test + void initializationFailureClosesConfigurationSourceWhenExposureWriterCloseFails() { + ConfigurationSourceService configService = mock(ConfigurationSourceService.class); + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + doThrow(new IllegalArgumentException("exposure close failed")).when(exposureWriter).close(); + + assertThrows( + IllegalArgumentException.class, + () -> FeatureFlaggingSystem.initialize(configService, exposureWriter)); + + verify(configService).close(); + } + private static SharedCommunicationObjects sharedCommunicationObjects() { DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); when(discovery.supportsEvpProxy()).thenReturn(true); diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 82faa899f3e..425e217e822 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -24,14 +24,11 @@ dependencies { api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one - // Span-enrichment write tier: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan. - compileOnly(project(":internal-api")) // Platform JSON writer for the ffe_* tag values. compileOnly(project(":components:json")) testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) - testImplementation(project(":internal-api")) testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index e9f0786d76a..ba0d712d88c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -2,6 +2,7 @@ import static datadog.communication.http.OkHttpUtils.prepareRequest; import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; +import static datadog.trace.util.Strings.isBlank; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; @@ -11,9 +12,7 @@ import datadog.trace.util.AgentThreadFactory; import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URLEncoder; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -277,26 +276,17 @@ private static boolean isRetryableStatus(final int status) { } private void updateEtag(final String nextEtag) { - if (nextEtag != null && !nextEtag.trim().isEmpty()) { - etag = nextEtag; - } + etag = isBlank(nextEtag) ? null : nextEtag; } static HttpUrl endpoint(final Config config) { final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl(); - final String endpoint = - configuredBaseUrl == null - ? datadogApiServerDistributionEndpoint(config) - : endpointFromConfiguredBaseUrl(configuredBaseUrl); - final HttpUrl parsed = HttpUrl.parse(endpoint); - if (parsed == null) { - throw new IllegalArgumentException( - "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); - } - return parsed; + return configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredBaseUrl(configuredBaseUrl); } - private static String endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { + private static HttpUrl endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim()); if (parsed == null) { throw new IllegalArgumentException( @@ -306,30 +296,22 @@ private static String endpointFromConfiguredBaseUrl(final String configuredBaseU return parsed .newBuilder() .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) - .build() - .toString(); + .build(); } - return parsed.toString(); + return parsed; } - private static String datadogApiServerDistributionEndpoint(final Config config) { - final StringBuilder endpoint = - new StringBuilder("https://api.") - .append(config.getSite().toLowerCase(Locale.ROOT)) - .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); + private static HttpUrl datadogApiServerDistributionEndpoint(final Config config) { + final HttpUrl.Builder endpoint = + new HttpUrl.Builder() + .scheme("https") + .host("api." + config.getSite()) + .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)); final String env = config.getEnv(); if (env != null && !env.isEmpty()) { - endpoint.append("?dd_env=").append(urlEncode(env)); - } - return endpoint.toString(); - } - - private static String urlEncode(final String value) { - try { - return URLEncoder.encode(value, "UTF-8"); - } catch (final IOException e) { - throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + endpoint.addQueryParameter("dd_env", env); } + return endpoint.build(); } static long millis(final int seconds) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index a5b6dc84bfb..07557135c25 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -66,7 +66,7 @@ void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { final Config config = config("datad0g.com", "staging env"); assertEquals( - "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging%20env", AgentlessConfigurationSource.endpoint(config).toString()); } @@ -385,6 +385,25 @@ void ignoresBlankEtag() throws Exception { verify(listener).accept(any(ServerConfiguration.class)); } + @Test + void successfulResponseWithoutEtagClearsPreviousEtag() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), + response(200, null, emptyConfig()), + response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertEquals("etag-a", client.requests.get(1).etag); + assertNull(client.requests.get(2).etag); + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + } + @Test void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { final FakeClient client = From 76917d61f00ab1ae9b5fb2bcd32f2dee2cb587c7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 16:30:56 -0400 Subject: [PATCH 10/26] Support the UFC CDN response contract --- .../feature-flagging-api/README.md | 11 ++- .../AgentlessConfigurationSource.java | 16 ++-- .../featureflag/RemoteConfigServiceImpl.java | 30 ++++++ .../AgentlessConfigurationSourceTest.java | 93 +++++++++++++++---- 4 files changed, 120 insertions(+), 30 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index ae861d07afd..74bbccca2e2 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -88,13 +88,16 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc - `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a different HTTP backend while keeping agentless delivery semantics. A bare - host uses the standard server-distribution path; a URL with a path is used as + host uses the standard rules-based server path; a URL with a path is used as the exact UFC endpoint. Configured URLs are opaque: the SDK does not add the Datadog-managed `dd_env` query parameter, so custom backends must include any required tenant or environment scope in the configured URL. The derived - Datadog-managed endpoint is intended for supported commercial sites; use an - explicit base URL elsewhere. Agentless responses do not have an SDK-imposed - payload-size limit. `remote_config` uses the existing Agent Remote + Datadog-managed endpoint is + `https://ufc-server.ff-cdn./api/v2/feature-flagging/config/rules-based/server` + and expects UFC under the JSON:API `data.attributes` response member. It is + intended for supported commercial sites; use an explicit base URL elsewhere. + Agentless responses do not have an SDK-imposed payload-size limit. + `remote_config` uses the existing Agent Remote Configuration path. `offline` is reserved for startup-provided UFC bytes; until those bytes are implemented, no network source starts and evaluations use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index ba0d712d88c..506fcddc9fa 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -35,9 +35,8 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessConfigurationSource.class); - // TODO before merge: confirm the final backend route with the server-distribution API owners. - private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = - "/api/v2/feature-flagging/config/server-distribution"; + private static final String DATADOG_UFC_RULES_BASED_SERVER_PATH = + "/api/v2/feature-flagging/config/rules-based/server"; private static final int MAX_ATTEMPTS = 3; private static final int MINUTES_BETWEEN_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; @@ -255,8 +254,9 @@ private boolean apply(final UfcHttpResponse response) { final ServerConfiguration configuration; try { configuration = - RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( - response.body); + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserializeApiResponse( + response.body, + config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() != null); } catch (final IOException | RuntimeException e) { LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); return false; @@ -295,7 +295,7 @@ private static HttpUrl endpointFromConfiguredBaseUrl(final String configuredBase if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) { return parsed .newBuilder() - .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) + .addPathSegments(DATADOG_UFC_RULES_BASED_SERVER_PATH.substring(1)) .build(); } return parsed; @@ -305,8 +305,8 @@ private static HttpUrl datadogApiServerDistributionEndpoint(final Config config) final HttpUrl.Builder endpoint = new HttpUrl.Builder() .scheme("https") - .host("api." + config.getSite()) - .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)); + .host("ufc-server.ff-cdn." + config.getSite()) + .addPathSegments(DATADOG_UFC_RULES_BASED_SERVER_PATH.substring(1)); final String env = config.getEnv(); if (env != null && !env.isEmpty()) { endpoint.addQueryParameter("dd_env", env); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index a5a9b2d7f36..12672aa74c1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -66,17 +66,47 @@ public void accept( static class UniversalFlagConfigDeserializer implements ConfigurationDeserializer { + private static final String UNIVERSAL_FLAG_CONFIGURATION_TYPE = "universal-flag-configuration"; static final UniversalFlagConfigDeserializer INSTANCE = new UniversalFlagConfigDeserializer(); private static final Moshi MOSHI = new Moshi.Builder().add(Date.class, new DateAdapter()).add(FlagMapAdapter.FACTORY).build(); private static final JsonAdapter V1_ADAPTER = MOSHI.adapter(ServerConfiguration.class); + private static final Type API_RESPONSE_TYPE = + Types.newParameterizedType(Map.class, String.class, Object.class); + private static final JsonAdapter> API_RESPONSE_ADAPTER = + MOSHI.adapter(API_RESPONSE_TYPE); @Override public ServerConfiguration deserialize(final byte[] content) throws IOException { return V1_ADAPTER.fromJson(Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); } + + @Nullable + ServerConfiguration deserializeApiResponse( + final byte[] content, final boolean allowRawConfiguration) throws IOException { + final Map response = + API_RESPONSE_ADAPTER.fromJson( + Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); + if (response != null && response.containsKey("data")) { + final Object data = response.get("data"); + if (!(data instanceof Map)) { + return null; + } + final Map dataAttributes = (Map) data; + return UNIVERSAL_FLAG_CONFIGURATION_TYPE.equals(dataAttributes.get("type")) + ? validConfiguration(V1_ADAPTER.fromJsonValue(dataAttributes.get("attributes"))) + : null; + } + return allowRawConfiguration ? validConfiguration(deserialize(content)) : null; + } + + @Nullable + private static ServerConfiguration validConfiguration( + @Nullable final ServerConfiguration configuration) { + return configuration != null && configuration.flags != null ? configuration : null; + } } static class FlagMapAdapter extends JsonAdapter> { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 07557135c25..c450c2e2191 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -51,7 +51,7 @@ @ExtendWith(MockitoExtension.class) class AgentlessConfigurationSourceTest { - private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/server-distribution"; + private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/rules-based/server"; @Mock private FeatureFlaggingGateway.ConfigListener listener; @@ -62,33 +62,33 @@ void cleanup() { } @Test - void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { + void derivesDatadogUfcCdnEndpointFromSiteAndEnv() { final Config config = config("datad0g.com", "staging env"); assertEquals( - "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging%20env", + "https://ufc-server.ff-cdn.datad0g.com/api/v2/feature-flagging/config/rules-based/server?dd_env=staging%20env", AgentlessConfigurationSource.endpoint(config).toString()); } @Test - void derivesDatadogApiServerDistributionEndpointWithoutEnv() { + void derivesDatadogUfcCdnEndpointWithoutEnv() { assertEquals( - "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config("datadoghq.com", "")).toString()); assertEquals( - "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); } @Test - void appendsServerDistributionPathToConfiguredAgentlessBaseUrl() { + void appendsRulesBasedServerPathToConfiguredAgentlessBaseUrl() { final Config config = config(); lenient() .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) .thenReturn("http://mock-backend:8080"); assertEquals( - "http://mock-backend:8080/api/v2/feature-flagging/config/server-distribution", + "http://mock-backend:8080/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config).toString()); } @@ -116,7 +116,7 @@ void rejectsInvalidConfiguredAgentlessBaseUrl() { } @Test - void rejectsInvalidDatadogApiServerDistributionEndpoint() { + void rejectsInvalidDatadogUfcCdnEndpoint() { assertThrows( IllegalArgumentException.class, () -> AgentlessConfigurationSource.endpoint(config("datadoghq.com:bad", ""))); @@ -359,18 +359,43 @@ void realHttpClientTimesOutDelayedResponse() throws Exception { } @Test - void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { + void appliesAcceptedJsonApiUfcThroughGatewayAndSendsApiKey() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); final AgentlessConfigurationSource service = service(client); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); - verify(listener).accept(any(ServerConfiguration.class)); + verify(listener).accept(configuration.capture()); + assertEquals("2026-07-15T19:57:07.219869778Z", configuration.getValue().createdAt); + assertNull(configuration.getValue().format); + assertEquals("Staging", configuration.getValue().environment.name); + assertTrue(configuration.getValue().flags.isEmpty()); assertEquals("test-api-key", client.requests.get(0).apiKey); assertNull(client.requests.get(0).etag); } + @Test + void appliesRawUfcFromConfiguredCompatibilityEndpoint() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfigAttributes())); + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://compatibility-backend/custom/ufc"); + final AgentlessConfigurationSource service = service(client, config); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(configuration.capture()); + assertEquals("Staging", configuration.getValue().environment.name); + assertTrue(configuration.getValue().flags.isEmpty()); + } + @Test void ignoresBlankEtag() throws Exception { final FakeClient client = @@ -486,10 +511,19 @@ void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { response(404, null, null), response(600, null, null), response(200, null, null), - response(200, null, "null")); + response(200, null, "null"), + response(200, null, jsonApiResponse("other-configuration", emptyConfigAttributes())), + response(200, null, "{\"data\":null}"), + response( + 200, null, "{\"data\":{\"id\":\"1\",\"type\":\"universal-flag-configuration\"}}"), + response(200, null, emptyConfigAttributes())); final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); @@ -902,6 +936,16 @@ private static AgentlessConfigurationSource service(final FakeClient client) { return service(client, delay -> {}, () -> 1.0); } + private static AgentlessConfigurationSource service( + final FakeClient client, final Config config) { + return new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config, + 30_000, + client, + Executors.newSingleThreadScheduledExecutor()); + } + private static AgentlessConfigurationSource service( final FakeClient client, final AgentlessConfigurationSource.RetrySleeper retrySleeper, @@ -943,19 +987,32 @@ private static AgentlessConfigurationSource.UfcHttpResponse response( } private static String emptyConfig() { + return jsonApiResponse("universal-flag-configuration", emptyConfigAttributes()); + } + + private static String emptyConfigAttributes() { return "{" - + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," - + "\"format\":\"SERVER\"," - + "\"environment\":{\"name\":\"Test\"}," + + "\"createdAt\":\"2026-07-15T19:57:07.219869778Z\"," + + "\"environment\":{\"name\":\"Staging\"}," + "\"flags\":{}" + "}"; } + private static String jsonApiResponse(final String type, final String attributes) { + return "{\"data\":{" + + "\"id\":\"1\"," + + "\"type\":\"" + + type + + "\"," + + "\"attributes\":" + + attributes + + "}}"; + } + private static String largeConfig(final int flagCount) { final StringBuilder json = new StringBuilder( - "{\"createdAt\":\"2024-04-17T19:40:53.716Z\"," - + "\"format\":\"SERVER\"," + "{\"createdAt\":\"2026-07-15T19:57:07.219869778Z\"," + "\"environment\":{\"name\":\"Large Test\"}," + "\"flags\":{"); for (int index = 0; index < flagCount; index++) { @@ -972,7 +1029,7 @@ private static String largeConfig(final int flagCount) { + "\"variations\":{\"on\":{\"key\":\"on\",\"value\":\"on\"}}," + "\"allocations\":[]}"); } - return json.append("}}").toString(); + return jsonApiResponse("universal-flag-configuration", json.append("}}").toString()); } private static void awaitCalls(final FakeClient client, final int count) throws Exception { From 2cee1833833aad9ff7f88bfa0bf5dd6f00812ec3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 16 Jul 2026 22:44:53 -0400 Subject: [PATCH 11/26] Harden agentless configuration responses Rely on OkHttp gzip negotiation and cover truncated response cache preservation. Require JSON:API envelopes for custom endpoints. --- .../AgentlessConfigurationSource.java | 4 +- .../featureflag/RemoteConfigServiceImpl.java | 5 +- .../AgentlessConfigurationSourceTest.java | 78 ++++++++++++++++++- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 506fcddc9fa..9a8ca42f093 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -255,8 +255,7 @@ private boolean apply(final UfcHttpResponse response) { try { configuration = RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserializeApiResponse( - response.body, - config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() != null); + response.body); } catch (final IOException | RuntimeException e) { LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); return false; @@ -373,6 +372,7 @@ public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final if (etag != null) { headers.put("If-None-Match", etag); } + // Leave Accept-Encoding unset so OkHttp negotiates gzip and transparently decompresses it. final Request request = prepareRequest(endpoint, headers, config, true).get().build(); final Call call = httpClient.newCall(request); if (!activeCall.compareAndSet(null, call)) { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index 12672aa74c1..2fceedcfff0 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -84,8 +84,7 @@ public ServerConfiguration deserialize(final byte[] content) throws IOException } @Nullable - ServerConfiguration deserializeApiResponse( - final byte[] content, final boolean allowRawConfiguration) throws IOException { + ServerConfiguration deserializeApiResponse(final byte[] content) throws IOException { final Map response = API_RESPONSE_ADAPTER.fromJson( Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); @@ -99,7 +98,7 @@ ServerConfiguration deserializeApiResponse( ? validConfiguration(V1_ADAPTER.fromJsonValue(dataAttributes.get("attributes"))) : null; } - return allowRawConfiguration ? validConfiguration(deserialize(content)) : null; + return null; } @Nullable diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index c450c2e2191..4fd316ebf3c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -21,10 +21,12 @@ import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; @@ -36,6 +38,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPOutputStream; import okhttp3.Call; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -157,6 +160,7 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { assertEquals("test-api-key", server.getLastRequest().getHeader("DD-API-KEY")); assertEquals("etag-a", server.getLastRequest().getHeader("If-None-Match")); assertEquals("java", server.getLastRequest().getHeader("Datadog-Meta-Lang")); + assertEquals("gzip", server.getLastRequest().getHeader("Accept-Encoding")); } finally { httpClient.dispatcher().executorService().shutdownNow(); httpClient.connectionPool().evictAll(); @@ -164,6 +168,59 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { } } + @Test + void handlesGzipAndKeepsLastKnownGoodWhenNextResponseIsTruncated() throws Exception { + final byte[] compressedConfig = gzip(emptyConfig()); + final byte[] truncatedConfig = Arrays.copyOf(compressedConfig, compressedConfig.length - 8); + final AtomicInteger responses = new AtomicInteger(); + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> { + final boolean firstResponse = responses.getAndIncrement() == 0; + final byte[] body = + firstResponse ? compressedConfig : truncatedConfig; + api.getResponse() + .addHeader("Content-Encoding", "gzip") + .addHeader("ETag", firstResponse ? "etag-good" : "etag-bad") + .sendWithType("application/json", body); + })))) { + final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + endpoint, + config(), + 30_000, + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient), + Executors.newSingleThreadScheduledExecutor(), + delay -> {}, + () -> 1.0); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(listener).accept(configuration.capture()); + assertEquals("Staging", configuration.getValue().environment.name); + assertEquals(4, responses.get()); + assertEquals("gzip", server.getLastRequest().getHeader("Accept-Encoding")); + assertEquals("etag-good", server.getLastRequest().getHeader("If-None-Match")); + } finally { + service.close(); + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + @Test void downloadsAndAppliesLargeUfcWithoutPayloadLimit() throws Exception { final int flagCount = 5_000; @@ -378,22 +435,27 @@ void appliesAcceptedJsonApiUfcThroughGatewayAndSendsApiKey() throws Exception { } @Test - void appliesRawUfcFromConfiguredCompatibilityEndpoint() throws Exception { - final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfigAttributes())); + void customEndpointRequiresJsonApiAndKeepsLastKnownGoodOnRawUfc() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-good", emptyConfig()), + response(200, "etag-raw", emptyConfigAttributes())); final Config config = config(); lenient() .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) - .thenReturn("http://compatibility-backend/custom/ufc"); + .thenReturn("http://custom-backend/custom/ufc"); final AgentlessConfigurationSource service = service(client, config); final ArgumentCaptor configuration = ArgumentCaptor.forClass(ServerConfiguration.class); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); verify(listener).accept(configuration.capture()); assertEquals("Staging", configuration.getValue().environment.name); - assertTrue(configuration.getValue().flags.isEmpty()); + assertNull(client.requests.get(0).etag); + assertEquals("etag-good", client.requests.get(1).etag); } @Test @@ -1032,6 +1094,14 @@ private static String largeConfig(final int flagCount) { return jsonApiResponse("universal-flag-configuration", json.append("}}").toString()); } + private static byte[] gzip(final String value) throws IOException { + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(output)) { + gzip.write(value.getBytes(UTF_8)); + } + return output.toByteArray(); + } + private static void awaitCalls(final FakeClient client, final int count) throws Exception { for (int i = 0; i < 100; i++) { if (client.calls.get() >= count) { From ebc28ca36f5a822c2b0077f1eaa52efdfb26d0d7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 16 Jul 2026 23:52:45 -0400 Subject: [PATCH 12/26] Separate UFC transport parsers Keep Remote Configuration on raw UFC parsing and give agentless JSON:API its own streaming envelope parser. Share UFC adapters without materializing an intermediate response map. --- .../AgentlessConfigurationSource.java | 4 +- .../featureflag/JsonApiUfcResponseParser.java | 94 +++++++++++ .../featureflag/RemoteConfigServiceImpl.java | 150 +----------------- .../UniversalFlagConfigParser.java | 140 ++++++++++++++++ .../JsonApiUfcResponseParserTest.java | 82 ++++++++++ .../RemoteConfigServiceImplTest.java | 29 ++-- 6 files changed, 336 insertions(+), 163 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 9a8ca42f093..f1565ec7554 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -253,9 +253,7 @@ private boolean apply(final UfcHttpResponse response) { } final ServerConfiguration configuration; try { - configuration = - RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserializeApiResponse( - response.body); + configuration = JsonApiUfcResponseParser.INSTANCE.parse(response.body); } catch (final IOException | RuntimeException e) { LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); return false; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java new file mode 100644 index 00000000000..758bb94da5c --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java @@ -0,0 +1,94 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.JsonReader; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import javax.annotation.Nullable; +import okio.BufferedSource; +import okio.Okio; + +final class JsonApiUfcResponseParser { + + private static final String UNIVERSAL_FLAG_CONFIGURATION_TYPE = "universal-flag-configuration"; + private static final JsonReader.Options RESPONSE_FIELDS = JsonReader.Options.of("data"); + private static final JsonReader.Options DATA_FIELDS = JsonReader.Options.of("type", "attributes"); + + static final JsonApiUfcResponseParser INSTANCE = + new JsonApiUfcResponseParser(UniversalFlagConfigParser.INSTANCE); + + private final UniversalFlagConfigParser ufcParser; + + JsonApiUfcResponseParser(final UniversalFlagConfigParser ufcParser) { + this.ufcParser = ufcParser; + } + + @Nullable + ServerConfiguration parse(final byte[] content) throws IOException { + try (BufferedSource source = Okio.buffer(Okio.source(new ByteArrayInputStream(content)))) { + final JsonReader reader = JsonReader.of(source); + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue(); + return null; + } + ServerConfiguration configuration = null; + reader.beginObject(); + while (reader.hasNext()) { + if (reader.selectName(RESPONSE_FIELDS) == 0) { + configuration = parseData(reader); + } else { + reader.skipName(); + reader.skipValue(); + } + } + reader.endObject(); + requireEndOfDocument(reader); + return configuration; + } + } + + @Nullable + private ServerConfiguration parseData(final JsonReader reader) throws IOException { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue(); + return null; + } + String type = null; + ServerConfiguration configuration = null; + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.selectName(DATA_FIELDS)) { + case 0: + if (reader.peek() == JsonReader.Token.STRING) { + type = reader.nextString(); + } else { + reader.skipValue(); + } + break; + case 1: + configuration = ufcParser.parse(reader); + break; + default: + reader.skipName(); + reader.skipValue(); + } + } + reader.endObject(); + return UNIVERSAL_FLAG_CONFIGURATION_TYPE.equals(type) + ? validConfiguration(configuration) + : null; + } + + @Nullable + private static ServerConfiguration validConfiguration( + @Nullable final ServerConfiguration configuration) { + return configuration != null && configuration.flags != null ? configuration : null; + } + + private static void requireEndOfDocument(final JsonReader reader) throws IOException { + if (reader.peek() != JsonReader.Token.END_DOCUMENT) { + throw new JsonDataException("JSON document was not fully consumed"); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index 2fceedcfff0..e66a6231990 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -1,35 +1,15 @@ package com.datadog.featureflag; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.JsonDataException; -import com.squareup.moshi.JsonReader; -import com.squareup.moshi.JsonWriter; -import com.squareup.moshi.Moshi; -import com.squareup.moshi.Types; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.remoteconfig.Capabilities; import datadog.remoteconfig.ConfigurationChangesTypedListener; -import datadog.remoteconfig.ConfigurationDeserializer; import datadog.remoteconfig.ConfigurationPoller; import datadog.remoteconfig.PollingRateHinter; import datadog.remoteconfig.Product; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; -import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.Type; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; import javax.annotation.Nullable; -import okio.Okio; public class RemoteConfigServiceImpl implements ConfigurationSourceService, ConfigurationChangesTypedListener { @@ -43,8 +23,7 @@ public RemoteConfigServiceImpl(final SharedCommunicationObjects sco, final Confi @Override public void init() { configurationPoller.addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); - configurationPoller.addListener( - Product.FFE_FLAGS, UniversalFlagConfigDeserializer.INSTANCE, this); + configurationPoller.addListener(Product.FFE_FLAGS, UniversalFlagConfigParser.INSTANCE, this); configurationPoller.start(); } @@ -62,131 +41,4 @@ public void accept( final PollingRateHinter pollingRateHinter) { FeatureFlaggingGateway.dispatch(configuration); } - - static class UniversalFlagConfigDeserializer - implements ConfigurationDeserializer { - - private static final String UNIVERSAL_FLAG_CONFIGURATION_TYPE = "universal-flag-configuration"; - static final UniversalFlagConfigDeserializer INSTANCE = new UniversalFlagConfigDeserializer(); - - private static final Moshi MOSHI = - new Moshi.Builder().add(Date.class, new DateAdapter()).add(FlagMapAdapter.FACTORY).build(); - private static final JsonAdapter V1_ADAPTER = - MOSHI.adapter(ServerConfiguration.class); - private static final Type API_RESPONSE_TYPE = - Types.newParameterizedType(Map.class, String.class, Object.class); - private static final JsonAdapter> API_RESPONSE_ADAPTER = - MOSHI.adapter(API_RESPONSE_TYPE); - - @Override - public ServerConfiguration deserialize(final byte[] content) throws IOException { - return V1_ADAPTER.fromJson(Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); - } - - @Nullable - ServerConfiguration deserializeApiResponse(final byte[] content) throws IOException { - final Map response = - API_RESPONSE_ADAPTER.fromJson( - Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); - if (response != null && response.containsKey("data")) { - final Object data = response.get("data"); - if (!(data instanceof Map)) { - return null; - } - final Map dataAttributes = (Map) data; - return UNIVERSAL_FLAG_CONFIGURATION_TYPE.equals(dataAttributes.get("type")) - ? validConfiguration(V1_ADAPTER.fromJsonValue(dataAttributes.get("attributes"))) - : null; - } - return null; - } - - @Nullable - private static ServerConfiguration validConfiguration( - @Nullable final ServerConfiguration configuration) { - return configuration != null && configuration.flags != null ? configuration : null; - } - } - - static class FlagMapAdapter extends JsonAdapter> { - - private static final Type FLAGS_TYPE = - Types.newParameterizedType(Map.class, String.class, Flag.class); - - static final Factory FACTORY = - new Factory() { - @Nullable - @Override - public JsonAdapter create( - @Nonnull final Type type, - @Nonnull final Set annotations, - @Nonnull final Moshi moshi) { - if (!annotations.isEmpty() || !Types.equals(type, FLAGS_TYPE)) { - return null; - } - return new FlagMapAdapter(moshi.adapter(Flag.class)); - } - }; - - private final JsonAdapter flagAdapter; - - FlagMapAdapter(final JsonAdapter flagAdapter) { - this.flagAdapter = flagAdapter; - } - - @Nullable - @Override - public Map fromJson(@Nonnull final JsonReader reader) throws IOException { - if (reader.peek() == JsonReader.Token.NULL) { - return reader.nextNull(); - } - final Map flags = new HashMap<>(); - reader.beginObject(); - while (reader.hasNext()) { - final String flagKey = reader.nextName(); - final Object rawFlag = reader.readJsonValue(); - try { - final Flag flag = flagAdapter.fromJsonValue(rawFlag); - if (flag != null) { - flags.put(flagKey, flag); - } - } catch (JsonDataException | IllegalArgumentException ignored) { - // A malformed flag must not prevent other flags in the same config from evaluating. - } - } - reader.endObject(); - return flags; - } - - @Override - public void toJson(@Nonnull final JsonWriter writer, @Nullable final Map value) - throws IOException { - throw new UnsupportedOperationException("Reading only adapter"); - } - } - - static class DateAdapter extends JsonAdapter { - - @Nullable - @Override - public Date fromJson(@Nonnull final JsonReader reader) throws IOException { - final String date = reader.nextString(); - if (date == null) { - return null; - } - try { - final Instant instant = DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(date, Instant::from); - return Date.from(instant); - } catch (Exception e) { - // ignore wrongly set dates - return null; - } - } - - @Override - public void toJson(@Nonnull final JsonWriter writer, @Nullable final Date value) - throws IOException { - throw new UnsupportedOperationException("Reading only adapter"); - } - } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java new file mode 100644 index 00000000000..373d54b1a14 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java @@ -0,0 +1,140 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.remoteconfig.ConfigurationDeserializer; +import datadog.trace.api.featureflag.ufc.v1.Flag; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import okio.BufferedSource; +import okio.Okio; + +final class UniversalFlagConfigParser implements ConfigurationDeserializer { + + static final UniversalFlagConfigParser INSTANCE = new UniversalFlagConfigParser(); + + private static final Moshi MOSHI = + new Moshi.Builder().add(Date.class, new DateAdapter()).add(FlagMapAdapter.FACTORY).build(); + private static final JsonAdapter V1_ADAPTER = + MOSHI.adapter(ServerConfiguration.class); + + private UniversalFlagConfigParser() {} + + @Override + public ServerConfiguration deserialize(final byte[] content) throws IOException { + try (BufferedSource source = Okio.buffer(Okio.source(new ByteArrayInputStream(content)))) { + final JsonReader reader = JsonReader.of(source); + final ServerConfiguration configuration = parse(reader); + requireEndOfDocument(reader); + return configuration; + } + } + + @Nullable + ServerConfiguration parse(final JsonReader reader) throws IOException { + return V1_ADAPTER.fromJson(reader); + } + + private static void requireEndOfDocument(final JsonReader reader) throws IOException { + if (reader.peek() != JsonReader.Token.END_DOCUMENT) { + throw new JsonDataException("JSON document was not fully consumed"); + } + } + + static final class FlagMapAdapter extends JsonAdapter> { + + private static final Type FLAGS_TYPE = + Types.newParameterizedType(Map.class, String.class, Flag.class); + + static final Factory FACTORY = + new Factory() { + @Nullable + @Override + public JsonAdapter create( + @Nonnull final Type type, + @Nonnull final Set annotations, + @Nonnull final Moshi moshi) { + if (!annotations.isEmpty() || !Types.equals(type, FLAGS_TYPE)) { + return null; + } + return new FlagMapAdapter(moshi.adapter(Flag.class)); + } + }; + + private final JsonAdapter flagAdapter; + + FlagMapAdapter(final JsonAdapter flagAdapter) { + this.flagAdapter = flagAdapter; + } + + @Nullable + @Override + public Map fromJson(@Nonnull final JsonReader reader) throws IOException { + if (reader.peek() == JsonReader.Token.NULL) { + return reader.nextNull(); + } + final Map flags = new HashMap<>(); + reader.beginObject(); + while (reader.hasNext()) { + final String flagKey = reader.nextName(); + final Object rawFlag = reader.readJsonValue(); + try { + final Flag flag = flagAdapter.fromJsonValue(rawFlag); + if (flag != null) { + flags.put(flagKey, flag); + } + } catch (JsonDataException | IllegalArgumentException ignored) { + // A malformed flag must not prevent other flags in the same config from evaluating. + } + } + reader.endObject(); + return flags; + } + + @Override + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Map value) + throws IOException { + throw new UnsupportedOperationException("Reading only adapter"); + } + } + + static final class DateAdapter extends JsonAdapter { + + @Nullable + @Override + public Date fromJson(@Nonnull final JsonReader reader) throws IOException { + final String date = reader.nextString(); + if (date == null) { + return null; + } + try { + final Instant instant = DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(date, Instant::from); + return Date.from(instant); + } catch (Exception e) { + // ignore wrongly set dates + return null; + } + } + + @Override + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Date value) + throws IOException { + throw new UnsupportedOperationException("Reading only adapter"); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java new file mode 100644 index 00000000000..bf1bbaa3679 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java @@ -0,0 +1,82 @@ +package com.datadog.featureflag; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import org.junit.jupiter.api.Test; + +class JsonApiUfcResponseParserTest { + + @Test + void parsesJsonApiMembersInAnyOrder() throws Exception { + final ServerConfiguration configuration = + parse( + "{" + + "\"meta\":{\"ignored\":true}," + + "\"data\":{" + + "\"attributes\":" + + emptyConfig() + + ",\"ignored\":true," + + "\"type\":\"universal-flag-configuration\"" + + "}" + + "}"); + + assertNotNull(configuration); + assertEquals("Test", configuration.environment.name); + assertTrue(configuration.flags.isEmpty()); + } + + @Test + void rejectsRawUfc() throws Exception { + assertNull(parse(emptyConfig())); + } + + @Test + void rejectsUnexpectedJsonApiType() throws Exception { + assertNull(parse("{\"data\":{\"type\":\"other-type\",\"attributes\":" + emptyConfig() + "}}")); + } + + @Test + void rejectsConfigurationWithoutFlags() throws Exception { + assertNull( + parse( + "{\"data\":{" + + "\"type\":\"universal-flag-configuration\"," + + "\"attributes\":{\"environment\":{\"name\":\"Test\"}}" + + "}}")); + } + + @Test + void rejectsNonObjectData() throws Exception { + assertNull(parse("{\"data\":[]}")); + } + + @Test + void rejectsTrailingJson() { + assertThrows( + IOException.class, + () -> + parse( + "{\"data\":{\"type\":\"universal-flag-configuration\",\"attributes\":" + + emptyConfig() + + "}}{}")); + } + + private static ServerConfiguration parse(final String json) throws Exception { + return JsonApiUfcResponseParser.INSTANCE.parse(json.getBytes(UTF_8)); + } + + private static String emptyConfig() { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java index d4ff05c75b1..d2ca1f2fee9 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java @@ -32,6 +32,7 @@ import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.lang.annotation.Annotation; +import java.io.IOException; import java.lang.reflect.Type; import java.time.Instant; import java.util.Date; @@ -137,6 +138,11 @@ void ignoresUnknownTopLevelFields() throws Exception { assertTrue(config.flags.isEmpty()); } + @Test + void rejectsTrailingJson() { + assertThrows(IOException.class, () -> deserialize(emptyConfig() + "{}")); + } + @Test void skipsUnknownOperatorFlagAndKeepsValidFlag() throws Exception { final ServerConfiguration config = @@ -189,14 +195,14 @@ void flagMapAdapterFactoryOnlyCreatesFlagMapAdapterForFlagMapType() { final Type flagsType = Types.newParameterizedType(Map.class, String.class, Flag.class); final JsonAdapter adapter = - RemoteConfigServiceImpl.FlagMapAdapter.FACTORY.create(flagsType, emptySet(), moshi); + UniversalFlagConfigParser.FlagMapAdapter.FACTORY.create(flagsType, emptySet(), moshi); assertNotNull(adapter); - assertTrue(adapter instanceof RemoteConfigServiceImpl.FlagMapAdapter); + assertTrue(adapter instanceof UniversalFlagConfigParser.FlagMapAdapter); assertNull( - RemoteConfigServiceImpl.FlagMapAdapter.FACTORY.create(String.class, emptySet(), moshi)); + UniversalFlagConfigParser.FlagMapAdapter.FACTORY.create(String.class, emptySet(), moshi)); assertNull( - RemoteConfigServiceImpl.FlagMapAdapter.FACTORY.create( + UniversalFlagConfigParser.FlagMapAdapter.FACTORY.create( flagsType, singleton(mock(Annotation.class)), moshi)); } @@ -248,8 +254,8 @@ void skipsNullFlagAndKeepsValidFlag() throws Exception { @Test void flagMapAdapterIsReadOnly() { - final RemoteConfigServiceImpl.FlagMapAdapter adapter = - new RemoteConfigServiceImpl.FlagMapAdapter(moshi().adapter(Flag.class)); + final UniversalFlagConfigParser.FlagMapAdapter adapter = + new UniversalFlagConfigParser.FlagMapAdapter(moshi().adapter(Flag.class)); assertThrows( UnsupportedOperationException.class, @@ -280,7 +286,8 @@ void flagMapAdapterIsReadOnly() { void testDateParsing(final String value, final Long expectedEpochMilli) throws Exception { final JsonReader reader = mock(JsonReader.class); when(reader.nextString()).thenReturn(value); - final RemoteConfigServiceImpl.DateAdapter adapter = new RemoteConfigServiceImpl.DateAdapter(); + final UniversalFlagConfigParser.DateAdapter adapter = + new UniversalFlagConfigParser.DateAdapter(); final Date parsed = adapter.fromJson(reader); if (expectedEpochMilli == null) { @@ -293,7 +300,8 @@ void testDateParsing(final String value, final Long expectedEpochMilli) throws E @Test void testParsingOnlyAdapter() { - final RemoteConfigServiceImpl.DateAdapter adapter = new RemoteConfigServiceImpl.DateAdapter(); + final UniversalFlagConfigParser.DateAdapter adapter = + new UniversalFlagConfigParser.DateAdapter(); assertThrows( UnsupportedOperationException.class, @@ -306,12 +314,11 @@ private ConfigurationDeserializer deserializer() { } private static ServerConfiguration deserialize(final String json) throws Exception { - return RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( - json.getBytes(UTF_8)); + return UniversalFlagConfigParser.INSTANCE.deserialize(json.getBytes(UTF_8)); } private static Moshi moshi() { - return new Moshi.Builder().add(Date.class, new RemoteConfigServiceImpl.DateAdapter()).build(); + return new Moshi.Builder().add(Date.class, new UniversalFlagConfigParser.DateAdapter()).build(); } private static String emptyConfig() { From 09843703e4ed56021449604c83668383e76636e7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 17 Jul 2026 00:23:22 -0400 Subject: [PATCH 13/26] Fix feature flagging test formatting --- .../com/datadog/featureflag/RemoteConfigServiceImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java index d2ca1f2fee9..c1f5ef17b87 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java @@ -31,8 +31,8 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; -import java.lang.annotation.Annotation; import java.io.IOException; +import java.lang.annotation.Annotation; import java.lang.reflect.Type; import java.time.Instant; import java.util.Date; From c146f09558056abe98a353264701f939c5d4c203 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 17 Jul 2026 01:02:35 -0400 Subject: [PATCH 14/26] Fix feature flagging parser coverage --- .../com/datadog/featureflag/JsonApiUfcResponseParser.java | 6 ++---- .../com/datadog/featureflag/UniversalFlagConfigParser.java | 5 ++--- .../datadog/featureflag/JsonApiUfcResponseParserTest.java | 5 +++++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java index 758bb94da5c..0a47d316075 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/JsonApiUfcResponseParser.java @@ -1,6 +1,5 @@ package com.datadog.featureflag; -import com.squareup.moshi.JsonDataException; import com.squareup.moshi.JsonReader; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.io.ByteArrayInputStream; @@ -87,8 +86,7 @@ private static ServerConfiguration validConfiguration( } private static void requireEndOfDocument(final JsonReader reader) throws IOException { - if (reader.peek() != JsonReader.Token.END_DOCUMENT) { - throw new JsonDataException("JSON document was not fully consumed"); - } + // A strict JsonReader throws if another top-level value follows the parsed document. + reader.peek(); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java index 373d54b1a14..ba5536601f7 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java @@ -51,9 +51,8 @@ ServerConfiguration parse(final JsonReader reader) throws IOException { } private static void requireEndOfDocument(final JsonReader reader) throws IOException { - if (reader.peek() != JsonReader.Token.END_DOCUMENT) { - throw new JsonDataException("JSON document was not fully consumed"); - } + // A strict JsonReader throws if another top-level value follows the parsed document. + reader.peek(); } static final class FlagMapAdapter extends JsonAdapter> { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java index bf1bbaa3679..a31d47889ba 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java @@ -42,6 +42,11 @@ void rejectsUnexpectedJsonApiType() throws Exception { assertNull(parse("{\"data\":{\"type\":\"other-type\",\"attributes\":" + emptyConfig() + "}}")); } + @Test + void rejectsNonStringJsonApiType() throws Exception { + assertNull(parse("{\"data\":{\"type\":null,\"attributes\":" + emptyConfig() + "}}")); + } + @Test void rejectsConfigurationWithoutFlags() throws Exception { assertNull( From 488b4770b26c0cac2afac8df9539ecb14fc5271c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 17 Jul 2026 16:39:31 -0400 Subject: [PATCH 15/26] Address final feature flagging review feedback --- internal-api/build.gradle.kts | 2 +- .../src/main/java/datadog/trace/api/Config.java | 16 +++++++++++++++- .../groovy/datadog/trace/api/ConfigTest.groovy | 1 + .../feature-flagging-agent/build.gradle.kts | 1 + .../featureflag/FeatureFlaggingSystemTest.java | 15 +++++++++++++-- 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index e47777b4438..2fc77aed0b5 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -268,7 +268,7 @@ dependencies { api(project(":components:context")) api(project(":components:environment")) api(project(":components:json")) - api(project(":products:feature-flagging:feature-flagging-config")) + implementation(project(":products:feature-flagging:feature-flagging-config")) api(project(":utils:config-utils")) api(project(":utils:time-utils")) diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 1da1769e5bf..368e666c701 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -3802,7 +3802,21 @@ private static String normalizeFeatureFlaggingConfigurationSource(final String s return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; } final String normalized = source.trim().toLowerCase(Locale.ROOT); - return normalized.isEmpty() ? DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE : normalized; + if (normalized.isEmpty()) { + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; + } + switch (normalized) { + case "agentless": + case "remote_config": + case "offline": + return normalized; + default: + log.warn( + "Unsupported Feature Flagging configuration source: {}. Defaulting to {}", + source, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE); + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; + } } public boolean isBaggageExtract() { diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 63b442f9a48..7227597108b 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -3517,6 +3517,7 @@ class ConfigTest extends DDSpecification { "" | "agentless" " " | "agentless" " ReMoTe_ConFiG " | "remote_config" + "not-a-real-source" | "agentless" } def "agentless feature flag timing falls back for non-positive values"() { diff --git a/products/feature-flagging/feature-flagging-agent/build.gradle.kts b/products/feature-flagging/feature-flagging-agent/build.gradle.kts index a219a48456c..0c008e373a3 100644 --- a/products/feature-flagging/feature-flagging-agent/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-agent/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) + testImplementation(project(":products:feature-flagging:feature-flagging-config")) testImplementation(project(":utils:test-utils")) testRuntimeOnly(project(":dd-trace-core")) } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index f6fc7516c0b..523736d13db 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -97,12 +97,23 @@ void explicitRemoteConfigUsesRemoteConfigService() { @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") - void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { + void invalidConfigurationSourceUsesAgentlessDefault() { + assertInstanceOf( + AgentlessConfigurationSource.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + void rejectsUnsupportedNormalizedConfigurationSource() { + Config config = mock(Config.class); + when(config.getFeatureFlaggingConfigurationSource()).thenReturn("invalid"); + assertThrows( IllegalArgumentException.class, () -> FeatureFlaggingSystem.createConfigurationSourceService( - sharedCommunicationObjects(), Config.get())); + sharedCommunicationObjects(), config)); } @Test From eaa99ce1723f3c0752287a4e29da94bbd4b1583a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 23 Jul 2026 23:03:34 -0600 Subject: [PATCH 16/26] fix(feature-flags): align configuration source semantics --- .../java/datadog/trace/bootstrap/Agent.java | 43 ++++++++++- .../datadog/trace/api/ConfigDefaults.java | 2 +- .../main/java/datadog/trace/api/Config.java | 66 ++++++++++------- .../datadog/trace/api/ConfigTest.groovy | 39 +++++++++- metadata/supported-configurations.json | 14 +++- .../feature-flagging-config/build.gradle.kts | 9 ++- .../config/FeatureFlaggingConfig.java | 71 ++++++++++++++++++- .../config/FeatureFlaggingConfigTest.java | 55 ++++++++++++++ 8 files changed, 259 insertions(+), 40 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-config/src/test/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfigTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 52bf912f8c1..002fad40caf 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -136,7 +136,7 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, true); private final String configKey; private final String systemProp; @@ -283,7 +283,7 @@ public static void start( agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION); llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); - featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING); + featureFlaggingEnabled = isFeatureFlaggingEnabled(); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { @@ -1756,6 +1756,45 @@ private static boolean isFeatureEnabled(AgentFeature feature) { } } + private static boolean isFeatureFlaggingEnabled() { + final Boolean providerEnabled = + featureFlaggingBooleanSetting(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED); + final String configurationSource = + featureFlaggingSetting(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE); + final Boolean legacyProviderEnabled = + featureFlaggingBooleanSetting(FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED); + + return FeatureFlaggingConfig.resolveConfiguration( + providerEnabled, configurationSource, legacyProviderEnabled) + .isEnabled(); + } + + @SuppressFBWarnings( + value = "NP_BOOLEAN_RETURN_NULL", + justification = "A null value preserves the distinction between absent and explicitly false") + private static Boolean featureFlaggingBooleanSetting(final String configKey) { + final String value = featureFlaggingSetting(configKey); + if (value == null) { + return null; + } + return Boolean.parseBoolean(value) || "1".equals(value); + } + + private static String featureFlaggingSetting(final String configKey) { + final String systemProperty = propertyNameToSystemPropertyName(configKey); + String value = SystemProperties.get(systemProperty); + if (value == null) { + value = getStableConfig(FLEET, configKey); + } + if (value == null) { + value = ddGetEnv(systemProperty); + } + if (value == null) { + value = getStableConfig(LOCAL, configKey); + } + return value; + } + /** * @see datadog.trace.api.ProductActivation#fromString(String) */ diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index d792bd5d84c..78d0eddc9ae 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -50,7 +50,7 @@ public final class ConfigDefaults { static final String DEFAULT_SITE = "datadoghq.com"; static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; - static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; + static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 5; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index ef60e07dd8a..7a38217d5f8 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -87,7 +87,6 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; -import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; @@ -729,10 +728,14 @@ import static datadog.trace.api.config.TracerConfig.WRITER_BAGGAGE_INJECT; import static datadog.trace.api.config.TracerConfig.WRITER_LINKS_INJECT; import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.isSupportedConfigurationSource; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.resolveConfiguration; import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.OTLP_WRITER_TYPE; import static datadog.trace.util.CollectionUtils.tryMakeImmutableList; @@ -749,6 +752,7 @@ import datadog.trace.api.config.OtlpConfig; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.api.config.TracerConfig; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.IastDetectionMode; import datadog.trace.api.iast.telemetry.Verbosity; @@ -1227,6 +1231,7 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; + private final boolean featureFlaggingProviderEnabled; private final String featureFlaggingConfigurationSource; private final String featureFlaggingConfigurationSourceAgentlessBaseUrl; private final int featureFlaggingConfigurationSourcePollIntervalSeconds; @@ -2862,10 +2867,33 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); - featureFlaggingConfigurationSource = - normalizeFeatureFlaggingConfigurationSource( - configProvider.getString( - FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + final Boolean configuredFeatureFlaggingProviderEnabled = + configProvider.getBoolean(FEATURE_FLAGS_ENABLED); + final Boolean legacyFeatureFlaggingProviderEnabled = + configProvider.getBoolean(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED); + final String configuredFeatureFlaggingConfigurationSource = + configProvider.isSet(FEATURE_FLAGS_CONFIGURATION_SOURCE) + ? configProvider.getString(FEATURE_FLAGS_CONFIGURATION_SOURCE) + : null; + final FeatureFlaggingConfig.Resolution resolvedFeatureFlaggingConfiguration = + resolveConfiguration( + configuredFeatureFlaggingProviderEnabled, + configuredFeatureFlaggingConfigurationSource, + legacyFeatureFlaggingProviderEnabled); + if (legacyFeatureFlaggingProviderEnabled != null) { + log.warn( + "Setting {} is deprecated. Use {} and {} instead.", + EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED, + FEATURE_FLAGS_ENABLED, + FEATURE_FLAGS_CONFIGURATION_SOURCE); + } + if (!isSupportedConfigurationSource(configuredFeatureFlaggingConfigurationSource)) { + log.warn( + "Unsupported Feature Flagging configuration source; provider disabled: '{}'", + resolvedFeatureFlaggingConfiguration.getSource()); + } + featureFlaggingProviderEnabled = resolvedFeatureFlaggingConfiguration.isEnabled(); + featureFlaggingConfigurationSource = resolvedFeatureFlaggingConfiguration.getSource(); featureFlaggingConfigurationSourceAgentlessBaseUrl = configProvider.getStringNotEmpty( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); @@ -3808,28 +3836,6 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } - private static String normalizeFeatureFlaggingConfigurationSource(final String source) { - if (source == null) { - return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; - } - final String normalized = source.trim().toLowerCase(Locale.ROOT); - if (normalized.isEmpty()) { - return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; - } - switch (normalized) { - case "agentless": - case "remote_config": - case "offline": - return normalized; - default: - log.warn( - "Unsupported Feature Flagging configuration source: {}. Defaulting to {}", - source, - DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE); - return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; - } - } - public boolean isBaggageExtract() { return tracePropagationStylesToExtract.contains(TracePropagationStyle.BAGGAGE); } @@ -4751,6 +4757,10 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } + public boolean isFeatureFlaggingProviderEnabled() { + return featureFlaggingProviderEnabled; + } + public String getFeatureFlaggingConfigurationSource() { return featureFlaggingConfigurationSource; } @@ -6624,6 +6634,8 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled + + ", featureFlaggingProviderEnabled=" + + featureFlaggingProviderEnabled + ", featureFlaggingConfigurationSource=" + featureFlaggingConfigurationSource + ", featureFlaggingConfigurationSourceAgentlessBaseUrl=" diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 7ed4d19e585..3b96c713974 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -57,9 +57,11 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_METRICS_CONFIGS @@ -3517,7 +3519,42 @@ class ConfigTest extends DDSpecification { "" | "agentless" " " | "agentless" " ReMoTe_ConFiG " | "remote_config" - "not-a-real-source" | "agentless" + "not-a-real-source" | "not-a-real-source" + " OFFLINE " | "offline" + } + + def "feature flag configuration applies migration precedence"() { + setup: + Properties properties = new Properties() + if (providerEnabled != null) { + properties.setProperty(FEATURE_FLAGS_ENABLED, providerEnabled.toString()) + } + if (source != null) { + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE, source) + } + if (legacyProviderEnabled != null) { + properties.setProperty(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED, legacyProviderEnabled.toString()) + } + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingProviderEnabled == expectedEnabled + config.featureFlaggingConfigurationSource == expectedSource + + where: + providerEnabled | source | legacyProviderEnabled | expectedEnabled | expectedSource + null | null | null | true | "agentless" + true | null | null | true | "agentless" + null | null | true | true | "remote_config" + null | null | false | false | null + null | "agentless" | true | true | "agentless" + null | "remote_config" | false | true | "remote_config" + false | "agentless" | true | false | "agentless" + true | null | false | false | null + null | "not-a-source" | null | false | "not-a-source" + null | "offline" | true | false | "offline" } def "agentless feature flag timing falls back for non-positive values"() { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 67b9a977f4a..779def0bb27 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1513,6 +1513,14 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ { "version": "A", @@ -1541,7 +1549,7 @@ { "version": "A", "type": "int", - "default": "2", + "default": "5", "aliases": [] } ], @@ -12002,5 +12010,7 @@ } ] }, - "deprecations": {} + "deprecations": { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "DD_FEATURE_FLAGS_ENABLED" + } } diff --git a/products/feature-flagging/feature-flagging-config/build.gradle.kts b/products/feature-flagging/feature-flagging-config/build.gradle.kts index 14109d8dfd9..97dd3aa9f0d 100644 --- a/products/feature-flagging/feature-flagging-config/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-config/build.gradle.kts @@ -4,9 +4,8 @@ plugins { apply(from = "$rootDir/gradle/java.gradle") -description = "Feature flagging configuration keys (compile-time constants)" +description = "Feature flagging configuration keys and source resolution" -extra["excludedClassesCoverage"] = listOf( - // Constants-only holder — no executable logic to cover. - "datadog.trace.api.featureflag.config.FeatureFlaggingConfig", -) +dependencies { + testImplementation(libs.bundles.junit5) +} diff --git a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java index 379f6470c5b..bae6fda1890 100644 --- a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java +++ b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java @@ -2,11 +2,22 @@ public class FeatureFlaggingConfig { - public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + public static final String CONFIGURATION_SOURCE_AGENTLESS = "agentless"; + public static final String CONFIGURATION_SOURCE_REMOTE_CONFIG = "remote_config"; + + private static final Resolution DISABLED_RESOLUTION = new Resolution(false, null); + private static final Resolution AGENTLESS_CONFIGURATION = + new Resolution(true, CONFIGURATION_SOURCE_AGENTLESS); + private static final Resolution REMOTE_CONFIG_CONFIGURATION = + new Resolution(true, CONFIGURATION_SOURCE_REMOTE_CONFIG); + + public static final String FEATURE_FLAGS_ENABLED = "feature.flags.enabled"; + public static final String EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED = + "experimental.flagging.provider.enabled"; /** * Opt-in gate for APM span enrichment with feature-flag evaluation metadata. DISTINCT from {@link - * #FLAGGING_PROVIDER_ENABLED} and OFF by default — enabling the provider does not enable span + * #FEATURE_FLAGS_ENABLED} and OFF by default — enabling the provider does not enable span * enrichment. */ public static final String EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED = @@ -21,5 +32,61 @@ public class FeatureFlaggingConfig { public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = "feature.flags.configuration.source.agentless.request.timeout.seconds"; + public static Resolution resolveConfiguration( + final Boolean providerEnabled, + final String explicitSource, + final Boolean legacyProviderEnabled) { + final String normalizedSource = normalizeConfigurationSource(explicitSource); + if (Boolean.FALSE.equals(providerEnabled)) { + return new Resolution(false, normalizedSource); + } + if (normalizedSource != null) { + if (CONFIGURATION_SOURCE_AGENTLESS.equals(normalizedSource)) { + return AGENTLESS_CONFIGURATION; + } + if (CONFIGURATION_SOURCE_REMOTE_CONFIG.equals(normalizedSource)) { + return REMOTE_CONFIG_CONFIGURATION; + } + return new Resolution(false, normalizedSource); + } + if (legacyProviderEnabled != null) { + return legacyProviderEnabled ? REMOTE_CONFIG_CONFIGURATION : DISABLED_RESOLUTION; + } + return AGENTLESS_CONFIGURATION; + } + + public static boolean isSupportedConfigurationSource(final String source) { + final String normalizedSource = normalizeConfigurationSource(source); + return normalizedSource == null + || CONFIGURATION_SOURCE_AGENTLESS.equals(normalizedSource) + || CONFIGURATION_SOURCE_REMOTE_CONFIG.equals(normalizedSource); + } + + private static String normalizeConfigurationSource(final String source) { + if (source == null) { + return null; + } + final String normalized = source.trim().toLowerCase(java.util.Locale.ROOT); + return normalized.isEmpty() ? null : normalized; + } + + public static final class Resolution { + private final boolean enabled; + private final String source; + + private Resolution(final boolean enabled, final String source) { + this.enabled = enabled; + this.source = source; + } + + public boolean isEnabled() { + return enabled; + } + + public String getSource() { + return source; + } + } + private FeatureFlaggingConfig() {} } diff --git a/products/feature-flagging/feature-flagging-config/src/test/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfigTest.java b/products/feature-flagging/feature-flagging-config/src/test/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfigTest.java new file mode 100644 index 00000000000..16283869d71 --- /dev/null +++ b/products/feature-flagging/feature-flagging-config/src/test/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfigTest.java @@ -0,0 +1,55 @@ +package datadog.trace.api.featureflag.config; + +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.isSupportedConfigurationSource; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.resolveConfiguration; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class FeatureFlaggingConfigTest { + + @Test + void appliesConfigurationPrecedence() { + assertResolution(true, CONFIGURATION_SOURCE_AGENTLESS, null, null, null); + assertResolution(true, CONFIGURATION_SOURCE_AGENTLESS, null, " ", null); + assertResolution(true, CONFIGURATION_SOURCE_REMOTE_CONFIG, null, null, true); + assertResolution(false, null, null, null, false); + assertResolution(true, CONFIGURATION_SOURCE_AGENTLESS, null, "agentless", true); + assertResolution(true, CONFIGURATION_SOURCE_REMOTE_CONFIG, null, " remote_CONFIG ", false); + assertResolution(false, CONFIGURATION_SOURCE_AGENTLESS, false, "agentless", true); + assertResolution(false, "invalid", null, "invalid", null); + assertResolution(false, "offline", null, " OFFLINE ", true); + } + + @Test + void recognizesSupportedExplicitSources() { + assertTrue(isSupportedConfigurationSource(null)); + assertTrue(isSupportedConfigurationSource(" ")); + assertTrue(isSupportedConfigurationSource("agentless")); + assertTrue(isSupportedConfigurationSource(" REMOTE_CONFIG ")); + assertFalse(isSupportedConfigurationSource("invalid")); + assertFalse(isSupportedConfigurationSource("offline")); + } + + private static void assertResolution( + final boolean enabled, + final String source, + final Boolean providerEnabled, + final String explicitSource, + final Boolean legacyProviderEnabled) { + final FeatureFlaggingConfig.Resolution resolution = + resolveConfiguration(providerEnabled, explicitSource, legacyProviderEnabled); + + assertEquals(enabled, resolution.isEnabled()); + if (source == null) { + assertNull(resolution.getSource()); + } else { + assertEquals(source, resolution.getSource()); + } + } +} From 2c0ee6ccbee7ce8e5952315e7d21350d930a98e0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 23 Jul 2026 23:07:57 -0600 Subject: [PATCH 17/26] feat(feature-flags): delay agentless polling until provider use --- .../feature-flagging-agent/build.gradle.kts | 1 + .../featureflag/FeatureFlaggingSystem.java | 91 +++++++++++++------ .../FeatureFlaggingSystemTest.java | 66 +++++++++++--- .../trace/api/openfeature/DDEvaluator.java | 1 + .../api/openfeature/DDEvaluatorTest.java | 17 ++++ .../featureflag/FeatureFlaggingGateway.java | 18 ++++ .../FeatureFlaggingGatewayTest.java | 13 +++ 7 files changed, 168 insertions(+), 39 deletions(-) diff --git a/products/feature-flagging/feature-flagging-agent/build.gradle.kts b/products/feature-flagging/feature-flagging-agent/build.gradle.kts index 0c008e373a3..d2dc4baea2b 100644 --- a/products/feature-flagging/feature-flagging-agent/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-agent/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { api(libs.slf4j) api(project(":products:feature-flagging:feature-flagging-lib")) api(project(":internal-api")) + compileOnly(project(":products:feature-flagging:feature-flagging-config")) testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 5c9cb20540e..adbae0a6e44 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -1,7 +1,11 @@ package com.datadog.featureflag; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; + import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,17 +16,64 @@ public class FeatureFlaggingSystem { private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; private static volatile SpanEnrichmentWriter SPAN_ENRICHMENT_WRITER; + private static volatile FeatureFlaggingGateway.ActivationListener ACTIVATION_LISTENER; + private static volatile boolean STARTED; private FeatureFlaggingSystem() {} public static synchronized void start(final SharedCommunicationObjects sco) { - if (CONFIG_SERVICE != null || EXPOSURE_WRITER != null) { + if (STARTED) { LOGGER.debug("Feature Flagging system already started"); return; } LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + STARTED = true; + + if (!config.isFeatureFlaggingProviderEnabled()) { + LOGGER.debug("Feature Flagging system disabled"); + return; + } + + if (CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { + final FeatureFlaggingGateway.ActivationListener activationListener = + () -> activateAgentless(sco, config); + ACTIVATION_LISTENER = activationListener; + FeatureFlaggingGateway.addActivationListener(activationListener); + LOGGER.debug("Feature Flagging system awaiting application provider activation"); + return; + } + + try { + initializeSystem(sco, config); + } catch (final RuntimeException | Error e) { + STARTED = false; + throw e; + } + } + + private static synchronized void activateAgentless( + final SharedCommunicationObjects sco, final Config config) { + final FeatureFlaggingGateway.ActivationListener activationListener = ACTIVATION_LISTENER; + if (!STARTED || activationListener == null) { + return; + } + ACTIVATION_LISTENER = null; + FeatureFlaggingGateway.removeActivationListener(activationListener); + try { + initializeSystem(sco, config); + } catch (final RuntimeException | Error e) { + STARTED = false; + throw e; + } + } + + private static void initializeSystem(final SharedCommunicationObjects sco, final Config config) { final ConfigurationSourceService configService = createConfigurationSourceService(sco, config); + if (configService == null) { + LOGGER.debug("Feature Flagging system disabled by unsupported configuration source"); + return; + } final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); initialize(configService, exposureWriter); @@ -60,30 +111,32 @@ static void initialize( static ConfigurationSourceService createConfigurationSourceService( final SharedCommunicationObjects sco, final Config config) { - final ConfigurationSource configurationSource = - ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); - - if (configurationSource == ConfigurationSource.REMOTE_CONFIG) { + final String configurationSource = config.getFeatureFlaggingConfigurationSource(); + if (CONFIGURATION_SOURCE_REMOTE_CONFIG.equals(configurationSource)) { if (!config.isRemoteConfigEnabled()) { throw new IllegalStateException("Feature Flagging system started without RC"); } return new RemoteConfigServiceImpl(sco, config); } - if (configurationSource == ConfigurationSource.AGENTLESS) { + if (CONFIGURATION_SOURCE_AGENTLESS.equals(configurationSource)) { return new AgentlessConfigurationSource(config); } - LOGGER.debug( - "Feature Flagging offline configuration source selected; no config service started"); return null; } public static synchronized void stop() { + final FeatureFlaggingGateway.ActivationListener activationListener = ACTIVATION_LISTENER; final SpanEnrichmentWriter spanEnrichmentWriter = SPAN_ENRICHMENT_WRITER; final ExposureWriter exposureWriter = EXPOSURE_WRITER; final ConfigurationSourceService configService = CONFIG_SERVICE; + STARTED = false; + ACTIVATION_LISTENER = null; SPAN_ENRICHMENT_WRITER = null; EXPOSURE_WRITER = null; CONFIG_SERVICE = null; + if (activationListener != null) { + FeatureFlaggingGateway.removeActivationListener(activationListener); + } try { if (spanEnrichmentWriter != null) { spanEnrichmentWriter.close(); @@ -102,25 +155,7 @@ public static synchronized void stop() { LOGGER.debug("Feature Flagging system stopped"); } - private enum ConfigurationSource { - AGENTLESS("agentless"), - REMOTE_CONFIG("remote_config"), - OFFLINE("offline"); - - private final String value; - - ConfigurationSource(final String value) { - this.value = value; - } - - private static ConfigurationSource from(final String value) { - for (final ConfigurationSource source : values()) { - if (source.value.equals(value)) { - return source; - } - } - throw new IllegalArgumentException( - "Unsupported Feature Flagging configuration source: " + value); - } + static boolean isAwaitingApplicationActivation() { + return ACTIVATION_LISTENER != null; } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index f31e3f13732..820dbe74885 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -2,15 +2,20 @@ import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; @@ -20,6 +25,7 @@ import datadog.remoteconfig.ConfigurationPoller; import datadog.remoteconfig.Product; import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.test.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -27,6 +33,31 @@ class FeatureFlaggingSystemTest { + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig( + key = FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://127.0.0.1:1") + void agentlessStartWaitsForApplicationProviderActivation() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + clearInvocations(sharedCommunicationObjects); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects); + + assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + verifyNoInteractions(sharedCommunicationObjects); + + FeatureFlaggingGateway.activate(); + + assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + } finally { + FeatureFlaggingSystem.stop(); + } + + assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") @@ -97,23 +128,20 @@ void explicitRemoteConfigUsesRemoteConfigService() { @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") - void invalidConfigurationSourceUsesAgentlessDefault() { - assertInstanceOf( - AgentlessConfigurationSource.class, + void invalidConfigurationSourceDoesNotStartNetworkSource() { + assertNull( FeatureFlaggingSystem.createConfigurationSourceService( sharedCommunicationObjects(), Config.get())); } @Test - void rejectsUnsupportedNormalizedConfigurationSource() { + void unsupportedNormalizedConfigurationSourceDoesNotStartNetworkSource() { Config config = mock(Config.class); when(config.getFeatureFlaggingConfigurationSource()).thenReturn("invalid"); - assertThrows( - IllegalArgumentException.class, - () -> - FeatureFlaggingSystem.createConfigurationSourceService( - sharedCommunicationObjects(), config)); + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), config)); } @Test @@ -126,9 +154,25 @@ void offlineConfigurationSourceDoesNotStartNetworkSource() { @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") - void startWithOfflineConfigurationSourceSkipsConfigService() { + void startWithOfflineConfigurationSourceDisablesSystem() { + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + + try { + assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); + verifyNoInteractions(sharedCommunicationObjects); + } finally { + FeatureFlaggingSystem.stop(); + } + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") + void startWithInvalidConfigurationSourceDisablesSystem() { + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + try { - assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); + assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); + verifyNoInteractions(sharedCommunicationObjects); } finally { FeatureFlaggingSystem.stop(); } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 654fb6e8f6c..6d55c54a5b3 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -67,6 +67,7 @@ public DDEvaluator(final Runnable configCallback) { @Override public boolean initialize( final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception { + FeatureFlaggingGateway.activate(); FeatureFlaggingGateway.addConfigListener(this); return initializationLatch.await(timeout, unit) || hasConfiguration(); } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index 3421fe3454e..7b6cc01e021 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -23,6 +23,7 @@ import com.squareup.moshi.JsonWriter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import dev.openfeature.sdk.ErrorCode; @@ -64,6 +65,22 @@ public class DDEvaluatorTest { private static final JsonAdapter> FIXTURE_LIST_ADAPTER = MOSHI.adapter(FIXTURE_LIST_TYPE); + @Test + public void testInitializeSignalsApplicationProviderActivation() throws Exception { + final FeatureFlaggingGateway.ActivationListener listener = + mock(FeatureFlaggingGateway.ActivationListener.class); + final DDEvaluator evaluator = new DDEvaluator(mock(Runnable.class)); + FeatureFlaggingGateway.addActivationListener(listener); + try { + evaluator.initialize(1, MILLISECONDS, mock(EvaluationContext.class)); + + verify(listener).activate(); + } finally { + evaluator.shutdown(); + FeatureFlaggingGateway.removeActivationListener(listener); + } + } + private static Arguments[] valueMappingTestCases() { return new Arguments[] { // String mappings diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index 2704b4be341..c8f5625c855 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -11,11 +11,16 @@ public abstract class FeatureFlaggingGateway { public interface ConfigListener extends Consumer {} + public interface ActivationListener { + void activate(); + } + public interface ExposureListener extends Consumer {} public interface SpanEnrichmentListener extends Consumer {} private static final List CONFIG_LISTENERS = new CopyOnWriteArrayList<>(); + private static final List ACTIVATION_LISTENERS = new CopyOnWriteArrayList<>(); private static final List EXPOSURE_LISTENERS = new CopyOnWriteArrayList<>(); private static final List SPAN_ENRICHMENT_LISTENERS = new CopyOnWriteArrayList<>(); @@ -42,6 +47,19 @@ public static void dispatch(final ServerConfiguration config) { CONFIG_LISTENERS.forEach(listener -> listener.accept(config)); } + public static void addActivationListener(final ActivationListener listener) { + ACTIVATION_LISTENERS.add(listener); + } + + public static void removeActivationListener(final ActivationListener listener) { + ACTIVATION_LISTENERS.remove(listener); + } + + /** Signals that application code initialized the Datadog OpenFeature provider. */ + public static void activate() { + ACTIVATION_LISTENERS.forEach(ActivationListener::activate); + } + public static void addExposureListener(final ExposureListener listener) { EXPOSURE_LISTENERS.add(listener); } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java index 3af2ed5add8..daaaf8d7001 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java @@ -13,6 +13,7 @@ class FeatureFlaggingGatewayTest { private FeatureFlaggingGateway.ConfigListener configListener; + private FeatureFlaggingGateway.ActivationListener activationListener; private FeatureFlaggingGateway.ExposureListener exposureListener; private FeatureFlaggingGateway.SpanEnrichmentListener spanEnrichmentListener; private ServerConfiguration firstConfiguration; @@ -23,6 +24,7 @@ class FeatureFlaggingGatewayTest { @BeforeEach void setUp() { configListener = mock(FeatureFlaggingGateway.ConfigListener.class); + activationListener = mock(FeatureFlaggingGateway.ActivationListener.class); exposureListener = mock(FeatureFlaggingGateway.ExposureListener.class); spanEnrichmentListener = mock(FeatureFlaggingGateway.SpanEnrichmentListener.class); firstConfiguration = mock(ServerConfiguration.class); @@ -34,10 +36,21 @@ void setUp() { @AfterEach void tearDown() { FeatureFlaggingGateway.removeConfigListener(configListener); + FeatureFlaggingGateway.removeActivationListener(activationListener); FeatureFlaggingGateway.removeExposureListener(exposureListener); FeatureFlaggingGateway.removeSpanEnrichmentListener(spanEnrichmentListener); } + @Test + void testProviderActivationListener() { + FeatureFlaggingGateway.addActivationListener(activationListener); + + FeatureFlaggingGateway.activate(); + + verify(activationListener).activate(); + verifyNoMoreInteractions(activationListener); + } + @Test void testAttachingAConfigListener() { clearCurrentServerConfiguration(); From bd0e347394c86b62165a43367f84efee8120e648 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 23 Jul 2026 23:14:38 -0600 Subject: [PATCH 18/26] feat(communication): support mapped retried HTTP calls --- .../communication/http/HttpRetryPolicy.java | 8 ++- .../communication/http/OkHttpUtils.java | 32 +++++++++-- .../http/OkHttpUtilsRetryTest.java | 53 +++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 communication/src/test/java/datadog/communication/http/OkHttpUtilsRetryTest.java diff --git a/communication/src/main/java/datadog/communication/http/HttpRetryPolicy.java b/communication/src/main/java/datadog/communication/http/HttpRetryPolicy.java index 3578f0e1c93..a645f53e29a 100644 --- a/communication/src/main/java/datadog/communication/http/HttpRetryPolicy.java +++ b/communication/src/main/java/datadog/communication/http/HttpRetryPolicy.java @@ -54,7 +54,13 @@ public class HttpRetryPolicy implements AutoCloseable { private final double delayFactor; private final boolean suppressInterrupts; - private HttpRetryPolicy( + /** + * Creates a retry policy. + * + *

Protected so products with a stricter cross-SDK retry contract can reuse the shared HTTP + * retry loop while supplying their own response, exception, and backoff rules. + */ + protected HttpRetryPolicy( int retriesLeft, long delay, double delayFactor, boolean suppressInterrupts) { this.retriesLeft = retriesLeft; this.delay = delay; diff --git a/communication/src/main/java/datadog/communication/http/OkHttpUtils.java b/communication/src/main/java/datadog/communication/http/OkHttpUtils.java index 9dc13229131..44105ae772d 100644 --- a/communication/src/main/java/datadog/communication/http/OkHttpUtils.java +++ b/communication/src/main/java/datadog/communication/http/OkHttpUtils.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import javax.annotation.Nullable; +import okhttp3.Call; import okhttp3.ConnectionPool; import okhttp3.ConnectionSpec; import okhttp3.Credentials; @@ -405,19 +406,39 @@ public void writeTo(BufferedSink sink) throws IOException { public static Response sendWithRetries( OkHttpClient httpClient, HttpRetryPolicy.Factory retryPolicyFactory, Request request) throws IOException { + return sendWithRetries((Call.Factory) httpClient, retryPolicyFactory, request); + } + + public static Response sendWithRetries( + Call.Factory callFactory, HttpRetryPolicy.Factory retryPolicyFactory, Request request) + throws IOException { + return sendWithRetries(callFactory, retryPolicyFactory, request, response -> response); + } + + public static T sendWithRetries( + Call.Factory callFactory, + HttpRetryPolicy.Factory retryPolicyFactory, + Request request, + ResponseMapper responseMapper) + throws IOException { try (HttpRetryPolicy retryPolicy = retryPolicyFactory.create()) { while (true) { + Response response = null; try { - Response response = httpClient.newCall(request).execute(); + response = callFactory.newCall(request).execute(); if (response.isSuccessful()) { - return response; + return responseMapper.map(response); } if (!retryPolicy.shouldRetry(response)) { - return response; + return responseMapper.map(response); } else { closeQuietly(response); + response = null; } } catch (Exception ex) { + if (response != null) { + closeQuietly(response); + } if (!retryPolicy.shouldRetry(ex)) { throw ex; } @@ -428,6 +449,11 @@ public static Response sendWithRetries( } } + @FunctionalInterface + public interface ResponseMapper { + T map(Response response) throws IOException; + } + private static void closeQuietly(Response response) { try { response.close(); diff --git a/communication/src/test/java/datadog/communication/http/OkHttpUtilsRetryTest.java b/communication/src/test/java/datadog/communication/http/OkHttpUtilsRetryTest.java new file mode 100644 index 00000000000..45f84a4810a --- /dev/null +++ b/communication/src/test/java/datadog/communication/http/OkHttpUtilsRetryTest.java @@ -0,0 +1,53 @@ +package datadog.communication.http; + +import static datadog.communication.http.OkHttpUtils.sendWithRetries; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.ConnectException; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.ResponseBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.jupiter.api.Test; + +class OkHttpUtilsRetryTest { + + @Test + void retriesResponseMappingFailureThroughCallFactory() throws Exception { + final MockWebServer server = new MockWebServer(); + final OkHttpClient client = new OkHttpClient(); + final AtomicInteger mappingAttempts = new AtomicInteger(); + server.enqueue(new MockResponse().setBody("first")); + server.enqueue(new MockResponse().setBody("second")); + server.start(); + + try { + final Request request = new Request.Builder().url(server.url("/configuration")).build(); + + final String body = + sendWithRetries( + (Call.Factory) client, + new HttpRetryPolicy.Factory(1, 0, 1), + request, + response -> { + try (ResponseBody responseBody = response.body()) { + if (mappingAttempts.getAndIncrement() == 0) { + throw new ConnectException("response body could not be mapped"); + } + return responseBody.string(); + } + }); + + assertEquals("second", body); + assertEquals(2, mappingAttempts.get()); + assertEquals(2, server.getRequestCount()); + } finally { + client.dispatcher().executorService().shutdownNow(); + client.connectionPool().evictAll(); + server.shutdown(); + } + } +} From cfaca97e1b2ecc85054c63374d3b5c1838d8a934 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 23 Jul 2026 23:15:09 -0600 Subject: [PATCH 19/26] fix(feature-flags): align agentless HTTP behavior --- .../AgentlessConfigurationSource.java | 239 +++++++++++------- .../AgentlessConfigurationSourceTest.java | 238 +++++++++++------ 2 files changed, 312 insertions(+), 165 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index f1565ec7554..35d1c68fd38 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -1,9 +1,11 @@ package com.datadog.featureflag; import static datadog.communication.http.OkHttpUtils.prepareRequest; +import static datadog.communication.http.OkHttpUtils.sendWithRetries; import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; import static datadog.trace.util.Strings.isBlank; +import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; import datadog.trace.api.Config; @@ -11,6 +13,7 @@ import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.util.AgentThreadFactory; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.HttpURLConnection; import java.util.HashMap; import java.util.Map; @@ -50,8 +53,6 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private final long pollIntervalMillis; private final UfcHttpClient client; private final ScheduledExecutorService executor; - private final RetrySleeper retrySleeper; - private final DoubleSupplier jitter; private final RatelimitedLogger ratelimitedLogger; private final Object lifecycleLock = new Object(); private final AtomicBoolean polling = new AtomicBoolean(); @@ -71,11 +72,12 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint new OkHttpUfcHttpClient( OkHttpUtils.buildHttpClient( endpoint, - millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()))), + millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds())), + millis(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()), + TimeUnit.MILLISECONDS::sleep, + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)), Executors.newSingleThreadScheduledExecutor( new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), - TimeUnit.MILLISECONDS::sleep, - () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER), new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } @@ -91,8 +93,6 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint pollIntervalMillis, client, executor, - TimeUnit.MILLISECONDS::sleep, - () -> 1.0, new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } @@ -102,35 +102,12 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint final long pollIntervalMillis, final UfcHttpClient client, final ScheduledExecutorService executor, - final RetrySleeper retrySleeper, - final DoubleSupplier jitter) { - this( - endpoint, - config, - pollIntervalMillis, - client, - executor, - retrySleeper, - jitter, - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); - } - - AgentlessConfigurationSource( - final HttpUrl endpoint, - final Config config, - final long pollIntervalMillis, - final UfcHttpClient client, - final ScheduledExecutorService executor, - final RetrySleeper retrySleeper, - final DoubleSupplier jitter, final RatelimitedLogger ratelimitedLogger) { this.endpoint = endpoint; this.config = config; this.pollIntervalMillis = pollIntervalMillis; this.client = client; this.executor = executor; - this.retrySleeper = retrySleeper; - this.jitter = jitter; this.ratelimitedLogger = ratelimitedLogger; } @@ -184,55 +161,28 @@ private void pollOnceSafely() { } private boolean fetchAndApply() { - for (int attempt = 1; ; attempt++) { - try { - final UfcHttpResponse response = client.fetch(endpoint, config, etag); - if (closed) { - return false; - } - if (isRetryableStatus(response.status)) { - if (attempt < MAX_ATTEMPTS) { - if (!waitBeforeRetry(attempt)) { - return false; - } - continue; - } - ratelimitedLogger.warn( - "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", - MAX_ATTEMPTS, - response.status); - return false; - } - synchronized (lifecycleLock) { - return !closed && apply(response); - } - } catch (final IOException e) { - if (closed) { - return false; - } - if (attempt == MAX_ATTEMPTS) { - ratelimitedLogger.warn( - "Feature Flagging agentless endpoint request failed after {} attempts", - MAX_ATTEMPTS, - e); - return false; - } - if (!waitBeforeRetry(attempt)) { - return false; - } - } - } - } - - private boolean waitBeforeRetry(final int attempt) { - if (closed) { - return false; - } try { - retrySleeper.sleep(retryDelayMillis(pollIntervalMillis, attempt, jitter.getAsDouble())); - return !closed; - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); + final UfcHttpResponse response = client.fetch(endpoint, config, etag); + if (closed) { + return false; + } + if (isRetryableStatus(response.status)) { + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", + MAX_ATTEMPTS, + response.status); + return false; + } + synchronized (lifecycleLock) { + return !closed && apply(response); + } + } catch (final IOException e) { + if (!closed) { + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint request failed after {} attempts", + MAX_ATTEMPTS, + e); + } return false; } } @@ -244,7 +194,7 @@ private boolean apply(final UfcHttpResponse response) { if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED || response.status == HttpURLConnection.HTTP_FORBIDDEN) { ratelimitedLogger.warn( - "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + "Feature Flagging agentless endpoint returned HTTP {}; verify endpoint authentication", response.status); return false; } @@ -356,11 +306,30 @@ static final class UfcHttpResponse { static final class OkHttpUfcHttpClient implements UfcHttpClient { private final OkHttpClient httpClient; - private final AtomicReference activeCall = new AtomicReference<>(); + private final long pollIntervalMillis; + private final RetrySleeper retrySleeper; + private final DoubleSupplier jitter; + private final AtomicBoolean fetching = new AtomicBoolean(); private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicReference activeCall = new AtomicReference<>(); OkHttpUfcHttpClient(final OkHttpClient httpClient) { + this( + httpClient, + TimeUnit.SECONDS.toMillis(30), + TimeUnit.MILLISECONDS::sleep, + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)); + } + + OkHttpUfcHttpClient( + final OkHttpClient httpClient, + final long pollIntervalMillis, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter) { this.httpClient = httpClient; + this.pollIntervalMillis = pollIntervalMillis; + this.retrySleeper = retrySleeper; + this.jitter = jitter; } @Override @@ -371,22 +340,50 @@ public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final headers.put("If-None-Match", etag); } // Leave Accept-Encoding unset so OkHttp negotiates gzip and transparently decompresses it. - final Request request = prepareRequest(endpoint, headers, config, true).get().build(); - final Call call = httpClient.newCall(request); - if (!activeCall.compareAndSet(null, call)) { + final Request request = + prepareRequest(endpoint, headers, config, isDatadogManagedEndpoint(endpoint, config)) + .get() + .build(); + if (!fetching.compareAndSet(false, true)) { throw new IllegalStateException("Feature Flagging HTTP request already in flight"); } if (cancelled.get()) { - call.cancel(); + fetching.set(false); + throw new InterruptedIOException("Feature Flagging HTTP client is closed"); } try { - final Response response = call.execute(); - try (ResponseBody responseBody = response.body()) { - final byte[] body = responseBody != null ? responseBody.bytes() : null; - return new UfcHttpResponse(response.code(), response.header("ETag"), body); - } + final HttpRetryPolicy.Factory retryPolicyFactory = + new HttpRetryPolicy.Factory(0, 0, 0) { + @Override + public HttpRetryPolicy create() { + return new AgentlessRetryPolicy( + cancelled, pollIntervalMillis, retrySleeper, jitter); + } + }; + final Call.Factory callFactory = + retryRequest -> { + final Call call = httpClient.newCall(retryRequest); + activeCall.set(call); + if (cancelled.get()) { + call.cancel(); + } + return call; + }; + return sendWithRetries( + callFactory, + retryPolicyFactory, + request, + response -> { + final int status = response.code(); + final String responseEtag = response.header("ETag"); + try (ResponseBody responseBody = response.body()) { + final byte[] body = responseBody != null ? responseBody.bytes() : null; + return new UfcHttpResponse(status, responseEtag, body); + } + }); } finally { - activeCall.compareAndSet(call, null); + activeCall.set(null); + fetching.set(false); } } @@ -398,5 +395,67 @@ public void cancel() { call.cancel(); } } + + private static boolean isDatadogManagedEndpoint(final HttpUrl endpoint, final Config config) { + return config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() == null + && endpoint.isHttps() + && endpoint.host().equalsIgnoreCase("ufc-server.ff-cdn." + config.getSite()); + } + } + + static final class AgentlessRetryPolicy extends HttpRetryPolicy { + private final AtomicBoolean cancelled; + private final long pollIntervalMillis; + private final RetrySleeper retrySleeper; + private final DoubleSupplier jitter; + private int retriesLeft = MAX_ATTEMPTS - 1; + private int retryAttempt; + + AgentlessRetryPolicy( + final AtomicBoolean cancelled, + final long pollIntervalMillis, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter) { + super(0, 0, 0, false); + this.cancelled = cancelled; + this.pollIntervalMillis = pollIntervalMillis; + this.retrySleeper = retrySleeper; + this.jitter = jitter; + } + + @Override + public boolean shouldRetry(final Exception exception) { + return exception instanceof IOException + && !Thread.currentThread().isInterrupted() + && reserveRetry(); + } + + @Override + public boolean shouldRetry(@Nullable final Response response) { + return response != null && isRetryableStatus(response.code()) && reserveRetry(); + } + + private boolean reserveRetry() { + if (cancelled.get() || retriesLeft == 0) { + return false; + } + retriesLeft--; + retryAttempt++; + return true; + } + + @Override + public void backoff() throws IOException { + if (cancelled.get()) { + throw new InterruptedIOException("Feature Flagging HTTP client is closed"); + } + try { + retrySleeper.sleep( + retryDelayMillis(pollIntervalMillis, retryAttempt, jitter.getAsDouble())); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new InterruptedIOException("Feature Flagging retry interrupted"); + } + } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 4fd316ebf3c..3e10a1beeff 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -41,9 +41,11 @@ import java.util.zip.GZIPOutputStream; import okhttp3.Call; import okhttp3.HttpUrl; +import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Protocol; import okhttp3.Response; +import okhttp3.ResponseBody; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -134,7 +136,7 @@ void defaultConstructorBuildsHttpClientFromConfig() { } @Test - void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { + void realHttpClientDoesNotSendApiKeyOverHttp() throws Exception { try (JavaTestHttpServer server = JavaTestHttpServer.httpServer( s -> @@ -157,7 +159,7 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { assertEquals(HttpURLConnection.HTTP_OK, response.status); assertEquals("etag-b", response.etag); assertEquals(emptyConfig(), new String(response.body, UTF_8)); - assertEquals("test-api-key", server.getLastRequest().getHeader("DD-API-KEY")); + assertNull(server.getLastRequest().getHeader("DD-API-KEY")); assertEquals("etag-a", server.getLastRequest().getHeader("If-None-Match")); assertEquals("java", server.getLastRequest().getHeader("Datadog-Meta-Lang")); assertEquals("gzip", server.getLastRequest().getHeader("Accept-Encoding")); @@ -168,6 +170,34 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { } } + @Test + void sendsApiKeyToDefaultDatadogHttpsEndpoint() throws Exception { + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient(requests, delay -> {}, () -> 1.0, response(200, "etag-a", emptyConfig())); + final HttpUrl endpoint = HttpUrl.get("https://ufc-server.ff-cdn.datadoghq.com" + CONFIG_PATH); + + client.fetch(endpoint, config(), null); + + assertEquals("test-api-key", requests.get(0).header("DD-API-KEY")); + } + + @Test + void stripsApiKeyFromCustomHttpsEndpoint() throws Exception { + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient(requests, delay -> {}, () -> 1.0, response(200, "etag-a", emptyConfig())); + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("https://flags.example.test/custom/ufc"); + final HttpUrl endpoint = HttpUrl.get("https://flags.example.test/custom/ufc"); + + client.fetch(endpoint, config, null); + + assertNull(requests.get(0).header("DD-API-KEY")); + } + @Test void handlesGzipAndKeepsLastKnownGoodWhenNextResponseIsTruncated() throws Exception { final byte[] compressedConfig = gzip(emptyConfig()); @@ -196,10 +226,9 @@ void handlesGzipAndKeepsLastKnownGoodWhenNextResponseIsTruncated() throws Except endpoint, config(), 30_000, - new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient), - Executors.newSingleThreadScheduledExecutor(), - delay -> {}, - () -> 1.0); + new AgentlessConfigurationSource.OkHttpUfcHttpClient( + httpClient, 30_000, delay -> {}, () -> 1.0), + Executors.newSingleThreadScheduledExecutor()); final ArgumentCaptor configuration = ArgumentCaptor.forClass(ServerConfiguration.class); FeatureFlaggingGateway.addConfigListener(listener); @@ -404,7 +433,8 @@ void realHttpClientTimesOutDelayedResponse() throws Exception { final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); final OkHttpClient httpClient = OkHttpUtils.buildHttpClient(endpoint, 50); final AgentlessConfigurationSource.OkHttpUfcHttpClient client = - new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + new AgentlessConfigurationSource.OkHttpUfcHttpClient( + httpClient, 30_000, delay -> {}, () -> 1.0); try { assertThrows(IOException.class, () -> client.fetch(endpoint, config(), null)); @@ -416,7 +446,7 @@ void realHttpClientTimesOutDelayedResponse() throws Exception { } @Test - void appliesAcceptedJsonApiUfcThroughGatewayAndSendsApiKey() throws Exception { + void appliesAcceptedJsonApiUfcThroughGateway() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); final AgentlessConfigurationSource service = service(client); final ArgumentCaptor configuration = @@ -430,7 +460,6 @@ void appliesAcceptedJsonApiUfcThroughGatewayAndSendsApiKey() throws Exception { assertNull(configuration.getValue().format); assertEquals("Staging", configuration.getValue().environment.name); assertTrue(configuration.getValue().flags.isEmpty()); - assertEquals("test-api-key", client.requests.get(0).apiKey); assertNull(client.requests.get(0).etag); } @@ -607,8 +636,6 @@ void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception { 30_000, client, executor, - delay -> {}, - () -> 1.0, ratelimitedLogger); try { @@ -617,11 +644,11 @@ void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception { verify(ratelimitedLogger) .warn( - "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + "Feature Flagging agentless endpoint returned HTTP {}; verify endpoint authentication", HttpURLConnection.HTTP_UNAUTHORIZED); verify(ratelimitedLogger) .warn( - "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + "Feature Flagging agentless endpoint returned HTTP {}; verify endpoint authentication", HttpURLConnection.HTTP_FORBIDDEN); } finally { service.close(); @@ -630,8 +657,12 @@ void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception { @Test void retriesTimeoutBeforeApplyingConfig() throws Exception { - final FakeClient client = - new FakeClient( + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, new SocketTimeoutException("slow HTTP configuration source"), new SocketTimeoutException("slow HTTP configuration source"), response(200, "etag-a", emptyConfig())); @@ -640,14 +671,18 @@ void retriesTimeoutBeforeApplyingConfig() throws Exception { assertTrue(service.pollOnce()); - assertEquals(3, client.calls.get()); + assertEquals(3, requests.size()); verify(listener).accept(any(ServerConfiguration.class)); } @Test void retriesClientTimeoutAndRateLimitStatusBeforeApplyingConfig() throws Exception { - final FakeClient client = - new FakeClient( + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, response(408, null, null), response(200, "etag-a", emptyConfig()), response(429, null, null), @@ -658,28 +693,37 @@ void retriesClientTimeoutAndRateLimitStatusBeforeApplyingConfig() throws Excepti assertTrue(service.pollOnce()); assertTrue(service.pollOnce()); - assertEquals(4, client.calls.get()); + assertEquals(4, requests.size()); verify(listener, times(2)).accept(any(ServerConfiguration.class)); } @Test void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { - final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, delay -> {}, () -> 1.0, response(500, null, null), response(304, null, null)); final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); - assertEquals(2, client.calls.get()); + assertEquals(2, requests.size()); verifyNoInteractions(listener); } @Test void warnsRateLimitedAfterRetryableFailuresAreExhausted() throws Exception { final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); - final FakeClient client = - new FakeClient( - response(503, null, null), response(503, null, null), response(503, null, null)); + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, + response(503, null, null), + response(503, null, null), + response(503, null, null)); final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); final AgentlessConfigurationSource service = new AgentlessConfigurationSource( @@ -688,15 +732,13 @@ void warnsRateLimitedAfterRetryableFailuresAreExhausted() throws Exception { 30_000, client, executor, - delay -> {}, - () -> 1.0, ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); try { assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); + assertEquals(3, requests.size()); verify(ratelimitedLogger) .warn( "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", 3, 503); @@ -711,8 +753,12 @@ void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception { final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); final SocketTimeoutException finalFailure = new SocketTimeoutException("slow HTTP configuration source"); - final FakeClient client = - new FakeClient( + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, new SocketTimeoutException("slow HTTP configuration source"), new SocketTimeoutException("slow HTTP configuration source"), finalFailure); @@ -724,15 +770,13 @@ void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception { 30_000, client, executor, - delay -> {}, - () -> 1.0, ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); try { assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); + assertEquals(3, requests.size()); verify(ratelimitedLogger) .warn( "Feature Flagging agentless endpoint request failed after {} attempts", @@ -747,17 +791,22 @@ void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception { @Test void usesIntervalAwareRetryBackoff() throws Exception { final List delays = new ArrayList<>(); - final FakeClient client = - new FakeClient( + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delays::add, + () -> 1.0, response(503, null, null), new SocketTimeoutException("slow HTTP configuration source"), response(200, "etag-a", emptyConfig())); - final AgentlessConfigurationSource service = service(client, delays::add, () -> 1.0); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); - assertEquals(java.util.Arrays.asList(5_000L, 10_000L), delays); + assertEquals(Arrays.asList(5_000L, 10_000L), delays); + assertEquals(3, requests.size()); verify(listener).accept(any(ServerConfiguration.class)); } @@ -915,23 +964,21 @@ void closeDuringIoFailurePreventsRetry() throws Exception { @Test void closeInterruptsRetryBackoff() throws Exception { final CountDownLatch backoffStarted = new CountDownLatch(1); + final List requests = new ArrayList<>(); final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - final FakeClient client = - new FakeClient( - new SocketTimeoutException("slow HTTP configuration source"), - response(200, "etag-a", emptyConfig())); - final AgentlessConfigurationSource service = - new AgentlessConfigurationSource( - HttpUrl.get("http://localhost" + CONFIG_PATH), - config(), - 30_000, - client, - executor, + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, delay -> { backoffStarted.countDown(); TimeUnit.MINUTES.sleep(1); }, - () -> 1.0); + () -> 1.0, + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), config(), 30_000, client, executor); service.init(); assertTrue(backoffStarted.await(1, TimeUnit.SECONDS)); @@ -939,7 +986,7 @@ void closeInterruptsRetryBackoff() throws Exception { service.close(); assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); - assertEquals(1, client.calls.get()); + assertEquals(1, requests.size()); } @Test @@ -968,8 +1015,12 @@ void initAfterCloseDoesNotSchedulePoll() throws Exception { class SystemTestParity { @Test void preservesSystemTestSourceTransitionsAndLastKnownGoodState() throws Exception { - final FakeClient client = - new FakeClient( + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, response(200, "etag-a", emptyConfig()), response(304, "etag-must-not-replace-a", null), response(509, null, null), @@ -986,41 +1037,80 @@ void preservesSystemTestSourceTransitionsAndLastKnownGoodState() throws Exceptio assertFalse(service.pollOnce()); verify(listener, times(2)).accept(any(ServerConfiguration.class)); - assertEquals("etag-a", client.requests.get(1).etag); - assertEquals("etag-a", client.requests.get(2).etag); - assertEquals("etag-a", client.requests.get(3).etag); - assertEquals("etag-b", client.requests.get(4).etag); - assertEquals("etag-b", client.requests.get(5).etag); + assertEquals("etag-a", requests.get(1).header("If-None-Match")); + assertEquals("etag-a", requests.get(2).header("If-None-Match")); + assertEquals("etag-a", requests.get(3).header("If-None-Match")); + assertEquals("etag-b", requests.get(4).header("If-None-Match")); + assertEquals("etag-b", requests.get(5).header("If-None-Match")); } } - private static AgentlessConfigurationSource service(final FakeClient client) { - return service(client, delay -> {}, () -> 1.0); - } - private static AgentlessConfigurationSource service( - final FakeClient client, final Config config) { + final AgentlessConfigurationSource.UfcHttpClient client) { return new AgentlessConfigurationSource( HttpUrl.get("http://localhost" + CONFIG_PATH), - config, + config(), 30_000, client, Executors.newSingleThreadScheduledExecutor()); } private static AgentlessConfigurationSource service( - final FakeClient client, - final AgentlessConfigurationSource.RetrySleeper retrySleeper, - final java.util.function.DoubleSupplier jitter) { - final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final FakeClient client, final Config config) { return new AgentlessConfigurationSource( HttpUrl.get("http://localhost" + CONFIG_PATH), - config(), + config, 30_000, client, - executor, - retrySleeper, - jitter); + Executors.newSingleThreadScheduledExecutor()); + } + + private static AgentlessConfigurationSource.OkHttpUfcHttpClient scriptedClient( + final List requests, + final AgentlessConfigurationSource.RetrySleeper retrySleeper, + final java.util.function.DoubleSupplier jitter, + final Object... outcomes) + throws IOException { + final BlockingQueue scriptedOutcomes = new LinkedBlockingQueue<>(); + scriptedOutcomes.addAll(Arrays.asList(outcomes)); + final OkHttpClient httpClient = mock(OkHttpClient.class); + when(httpClient.newCall(any())) + .thenAnswer( + invocation -> { + final okhttp3.Request request = invocation.getArgument(0); + requests.add(request); + final Object outcome = scriptedOutcomes.remove(); + final Call call = mock(Call.class); + when(call.execute()) + .thenAnswer( + ignored -> { + if (outcome instanceof IOException) { + throw (IOException) outcome; + } + return okHttpResponse( + request, (AgentlessConfigurationSource.UfcHttpResponse) outcome); + }); + return call; + }); + return new AgentlessConfigurationSource.OkHttpUfcHttpClient( + httpClient, 30_000, retrySleeper, jitter); + } + + private static Response okHttpResponse( + final okhttp3.Request request, final AgentlessConfigurationSource.UfcHttpResponse response) { + final Response.Builder builder = + new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(response.status) + .message(Integer.toString(response.status)); + if (response.etag != null) { + builder.header("ETag", response.etag); + } + if (response.body != null) { + builder.body(ResponseBody.create(MediaType.get("application/json"), response.body)); + } + return builder.build(); } private static Config config() { @@ -1034,7 +1124,7 @@ private static Config config(final String site, final String env) { .thenReturn(30); lenient() .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) - .thenReturn(2); + .thenReturn(5); lenient().when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(null); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); @@ -1135,7 +1225,7 @@ private void block(final CountDownLatch requestStarted, final CountDownLatch rel public AgentlessConfigurationSource.UfcHttpResponse fetch( final HttpUrl endpoint, final Config config, final String etag) throws IOException { calls.incrementAndGet(); - requests.add(new Request(config.getApiKey(), etag)); + requests.add(new Request(etag)); if (requestStarted != null) { requestStarted.countDown(); } @@ -1173,11 +1263,9 @@ private static void await(final CountDownLatch latch) throws IOException { } private static final class Request { - private final String apiKey; private final String etag; - private Request(final String apiKey, final String etag) { - this.apiKey = apiKey; + private Request(final String etag) { this.etag = etag; } } From 6df48d1cfd4d543efc609bfc5921f21efd124ecb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 23 Jul 2026 23:30:40 -0600 Subject: [PATCH 20/26] fix(feature-flags): complete initial poll during activation --- .../AgentlessConfigurationSource.java | 31 ++++++++++-- .../AgentlessConfigurationSourceTest.java | 49 ++++++++++++++++--- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 35d1c68fd38..b08d00ac08e 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -56,7 +56,9 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private final RatelimitedLogger ratelimitedLogger; private final Object lifecycleLock = new Object(); private final AtomicBoolean polling = new AtomicBoolean(); + private final AtomicReference pollingThread = new AtomicReference<>(); private volatile boolean closed; + private volatile boolean started; private volatile ScheduledFuture scheduledPoll; private volatile String etag; @@ -114,12 +116,26 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint @Override public void init() { synchronized (lifecycleLock) { - if (closed || scheduledPoll != null) { + if (closed || started) { return; } - scheduledPoll = - executor.scheduleWithFixedDelay( - this::pollOnceSafely, 0, pollIntervalMillis, TimeUnit.MILLISECONDS); + started = true; + } + + // Complete the first poll cycle on the activation thread. This lets OpenFeature provider + // initialization observe a successful retry before it checks whether configuration is ready. + // No request occurs before application code activates the provider. + pollOnceSafely(); + + synchronized (lifecycleLock) { + if (!closed) { + scheduledPoll = + executor.scheduleWithFixedDelay( + this::pollOnceSafely, + pollIntervalMillis, + pollIntervalMillis, + TimeUnit.MILLISECONDS); + } } } @@ -127,9 +143,11 @@ boolean pollOnce() { if (closed || !polling.compareAndSet(false, true)) { return false; } + pollingThread.set(Thread.currentThread()); try { return fetchAndApply(); } finally { + pollingThread.compareAndSet(Thread.currentThread(), null); polling.set(false); } } @@ -142,6 +160,7 @@ public void close() { return; } closed = true; + started = false; poll = scheduledPoll; scheduledPoll = null; } @@ -149,6 +168,10 @@ public void close() { poll.cancel(true); } client.cancel(); + final Thread activePollingThread = pollingThread.get(); + if (activePollingThread != null) { + activePollingThread.interrupt(); + } executor.shutdownNow(); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 3e10a1beeff..2ee67cb2e79 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -847,7 +847,7 @@ void rejectsOverlappingPolls() throws Exception { } @Test - void initSchedulesPollAndCloseCancelsFuture() throws Exception { + void initCompletesFirstPollAndCloseCancelsScheduledFuture() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); final AgentlessConfigurationSource service = new AgentlessConfigurationSource( @@ -859,12 +859,41 @@ void initSchedulesPollAndCloseCancelsFuture() throws Exception { FeatureFlaggingGateway.addConfigListener(listener); service.init(); - awaitCalls(client, 1); + assertEquals(1, client.calls.get()); service.close(); verify(listener).accept(any(ServerConfiguration.class)); } + @Test + void initCompletesInitialRetryCycleBeforeReturning() throws Exception { + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient( + requests, + delay -> {}, + () -> 1.0, + response(500, null, null), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 60_000, + client, + Executors.newSingleThreadScheduledExecutor()); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + service.init(); + + assertEquals(2, requests.size()); + verify(listener).accept(any(ServerConfiguration.class)); + } finally { + service.close(); + } + } + @Test void repeatedInitStartsOnlyOnePoller() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); @@ -979,14 +1008,20 @@ void closeInterruptsRetryBackoff() throws Exception { final AgentlessConfigurationSource service = new AgentlessConfigurationSource( HttpUrl.get("http://localhost" + CONFIG_PATH), config(), 30_000, client, executor); + final ExecutorService runner = Executors.newSingleThreadExecutor(); - service.init(); - assertTrue(backoffStarted.await(1, TimeUnit.SECONDS)); + try { + final Future initialization = runner.submit(service::init); + assertTrue(backoffStarted.await(1, TimeUnit.SECONDS)); - service.close(); + service.close(); - assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); - assertEquals(1, requests.size()); + initialization.get(1, TimeUnit.SECONDS); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + assertEquals(1, requests.size()); + } finally { + runner.shutdownNow(); + } } @Test From 0e14ef5ca5d78d5034d07065052dc2618376c40b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 15:05:38 -0600 Subject: [PATCH 21/26] chore(feature-flags): remove unused agent feature --- .../src/main/java/datadog/trace/bootstrap/Agent.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 002fad40caf..2e316eeb253 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -135,8 +135,7 @@ private enum AgentFeature { AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false), APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), - LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, true); + LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false); private final String configKey; private final String systemProp; From 43bfeff0b288dedc9bec9862efb8d532ef1136d6 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Fri, 24 Jul 2026 17:46:03 -0400 Subject: [PATCH 22/26] Fix race in agentless config source listener test scheduledPollContinuesAfterListenerRuntimeException waited on FakeClient.calls, which is incremented when a request starts rather than when it completes. The barrier therefore released as soon as the second poll began, letting the assertion race the poll thread that applies the configuration and notifies the listener. Wait on a CountDownLatch counted down by the listener itself, so the second notification is guaranteed to have happened before the assertions run. Environment: Datadog workspace Co-Authored-By: Claude Opus 5 (1M context) Co-authored-by: vickie.fridge --- .../featureflag/AgentlessConfigurationSourceTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 2ee67cb2e79..95bb814c4ab 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -920,9 +920,14 @@ void scheduledPollContinuesAfterListenerRuntimeException() throws Exception { client, Executors.newSingleThreadScheduledExecutor()); final AtomicInteger listenerCalls = new AtomicInteger(); + // Wait on the listener rather than on FakeClient.calls: the call counter is incremented when + // a request starts, so it reaches 2 before the second configuration has been applied. + final CountDownLatch listenerNotified = new CountDownLatch(2); final FeatureFlaggingGateway.ConfigListener flakyListener = configuration -> { - if (listenerCalls.incrementAndGet() == 1) { + final int call = listenerCalls.incrementAndGet(); + listenerNotified.countDown(); + if (call == 1) { throw new IllegalStateException("listener rejected first configuration"); } }; @@ -930,7 +935,7 @@ void scheduledPollContinuesAfterListenerRuntimeException() throws Exception { try { service.init(); - awaitCalls(client, 2); + assertTrue(listenerNotified.await(5, TimeUnit.SECONDS)); assertEquals(2, listenerCalls.get()); assertNull(client.requests.get(1).etag); From ad7d62676ae8d3536283d3f1cfa3f90e20f90e7d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 16:04:29 -0600 Subject: [PATCH 23/26] test(feature-flags): cover agentless lifecycle branches --- .../FeatureFlaggingSystemTest.java | 23 +++++++++ .../AgentlessConfigurationSourceTest.java | 51 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 820dbe74885..d408d91da4e 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -58,6 +58,29 @@ void agentlessStartWaitsForApplicationProviderActivation() { assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig( + key = FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://127.0.0.1:1") + void agentlessStopRemovesPendingApplicationProviderActivation() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + clearInvocations(sharedCommunicationObjects); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects); + assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + + FeatureFlaggingSystem.stop(); + FeatureFlaggingGateway.activate(); + + assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + verifyNoInteractions(sharedCommunicationObjects); + } finally { + FeatureFlaggingSystem.stop(); + } + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 95bb814c4ab..b02bd05642b 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -23,6 +23,7 @@ import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.util.ArrayList; @@ -37,6 +38,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPOutputStream; import okhttp3.Call; @@ -198,6 +200,18 @@ void stripsApiKeyFromCustomHttpsEndpoint() throws Exception { assertNull(requests.get(0).header("DD-API-KEY")); } + @Test + void stripsApiKeyFromUnexpectedHttpsEndpoint() throws Exception { + final List requests = new ArrayList<>(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + scriptedClient(requests, delay -> {}, () -> 1.0, response(200, "etag-a", emptyConfig())); + final HttpUrl endpoint = HttpUrl.get("https://flags.example.test/custom/ufc"); + + client.fetch(endpoint, config(), null); + + assertNull(requests.get(0).header("DD-API-KEY")); + } + @Test void handlesGzipAndKeepsLastKnownGoodWhenNextResponseIsTruncated() throws Exception { final byte[] compressedConfig = gzip(emptyConfig()); @@ -810,6 +824,37 @@ void usesIntervalAwareRetryBackoff() throws Exception { verify(listener).accept(any(ServerConfiguration.class)); } + @Test + void retryPolicyRejectsNonIoAndInterruptedIoFailures() { + final AgentlessConfigurationSource.AgentlessRetryPolicy policy = + retryPolicy(new AtomicBoolean()); + + assertFalse(policy.shouldRetry(new IllegalStateException("not an I/O failure"))); + + Thread.currentThread().interrupt(); + try { + assertFalse(policy.shouldRetry(new IOException("interrupted"))); + } finally { + assertTrue(Thread.interrupted()); + } + } + + @Test + void retryPolicyRejectsMissingResponse() { + final AgentlessConfigurationSource.AgentlessRetryPolicy policy = + retryPolicy(new AtomicBoolean()); + + assertFalse(policy.shouldRetry((Response) null)); + } + + @Test + void cancelledRetryPolicyRejectsBackoff() { + final AgentlessConfigurationSource.AgentlessRetryPolicy policy = + retryPolicy(new AtomicBoolean(true)); + + assertThrows(InterruptedIOException.class, policy::backoff); + } + @Test void clampsAndJittersRetryBackoff() { assertEquals(2_000, AgentlessConfigurationSource.retryDelayMillis(1_000, 1, 1.0)); @@ -1136,6 +1181,12 @@ private static AgentlessConfigurationSource.OkHttpUfcHttpClient scriptedClient( httpClient, 30_000, retrySleeper, jitter); } + private static AgentlessConfigurationSource.AgentlessRetryPolicy retryPolicy( + final AtomicBoolean cancelled) { + return new AgentlessConfigurationSource.AgentlessRetryPolicy( + cancelled, 30_000, delay -> {}, () -> 1.0); + } + private static Response okHttpResponse( final okhttp3.Request request, final AgentlessConfigurationSource.UfcHttpResponse response) { final Response.Builder builder = From 29c1fd405152c822a2bad4ebfa859970526b41d9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 16:08:13 -0600 Subject: [PATCH 24/26] chore(agent): raise jar size budget to 34 MiB --- metadata/agent-jar-checks.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/agent-jar-checks.properties b/metadata/agent-jar-checks.properties index d2565d0f6fd..a096b2cf519 100644 --- a/metadata/agent-jar-checks.properties +++ b/metadata/agent-jar-checks.properties @@ -1,7 +1,7 @@ # Agent jar structural invariants — edit intentionally, commit with the change that justified it. -# Max agent jar size in bytes. Raise only when the size growth is intentional (~33.02 MiB). -jar.size.budget = 34619392 +# Max agent jar size in bytes. Raise only when the size growth is intentional (34 MiB = 35651584). +jar.size.budget = 35651584 # Minimum combined class + classdata count in the assembled agent jar. # Set to ~98% of the actual count at the time of the last intentional change. From 1c4a2234fe5cdbc886582a2e34f263ecb31f9c2c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 16:33:04 -0600 Subject: [PATCH 25/26] fix(feature-flags): document retry policy confinement --- .../datadog/featureflag/AgentlessConfigurationSource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index b08d00ac08e..f4fc35cce30 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -12,6 +12,7 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.util.AgentThreadFactory; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.io.InterruptedIOException; import java.net.HttpURLConnection; @@ -426,6 +427,10 @@ private static boolean isDatadogManagedEndpoint(final HttpUrl endpoint, final Co } } + @SuppressFBWarnings( + value = "AT_NONATOMIC_OPERATIONS_ON_SHARED_VARIABLE", + justification = + "Each retry policy belongs to one synchronous HTTP request and is confined to one thread") static final class AgentlessRetryPolicy extends HttpRetryPolicy { private final AtomicBoolean cancelled; private final long pollIntervalMillis; From 40c9e0f4e06a82e7143052bc8142c950a40f2869 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 27 Jul 2026 15:29:34 -0600 Subject: [PATCH 26/26] Revert "chore(agent): raise jar size budget to 34 MiB" This reverts commit 29c1fd405152c822a2bad4ebfa859970526b41d9. --- metadata/agent-jar-checks.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/agent-jar-checks.properties b/metadata/agent-jar-checks.properties index a096b2cf519..d2565d0f6fd 100644 --- a/metadata/agent-jar-checks.properties +++ b/metadata/agent-jar-checks.properties @@ -1,7 +1,7 @@ # Agent jar structural invariants — edit intentionally, commit with the change that justified it. -# Max agent jar size in bytes. Raise only when the size growth is intentional (34 MiB = 35651584). -jar.size.budget = 35651584 +# Max agent jar size in bytes. Raise only when the size growth is intentional (~33.02 MiB). +jar.size.budget = 34619392 # Minimum combined class + classdata count in the assembled agent jar. # Set to ~98% of the actual count at the time of the last intentional change.