diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 876a07f1fd1..98ec5e57718 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -72,6 +72,7 @@ import java.net.URL; import java.security.CodeSource; import java.util.EnumSet; +import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.PatternSyntaxException; @@ -135,7 +136,7 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, false); private final String configKey; private final String systemProp; @@ -282,7 +283,7 @@ public static void start( agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION); llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); - featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING); + featureFlaggingEnabled = isFeatureFlaggingEnabled(); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { @@ -1631,6 +1632,53 @@ private static boolean isFeatureEnabled(AgentFeature feature) { } } + private static boolean isFeatureFlaggingEnabled() { + final String enabled = getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED); + if ("false".equalsIgnoreCase(enabled) || "0".equals(enabled)) { + return false; + } + if ("true".equalsIgnoreCase(enabled) || "1".equals(enabled)) { + return true; + } + + final String oldEnabled = getConfiguredValue(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED); + if ("false".equalsIgnoreCase(oldEnabled) || "0".equals(oldEnabled)) { + return false; + } + if ("true".equalsIgnoreCase(oldEnabled) || "1".equals(oldEnabled)) { + return true; + } + + return isFeatureFlaggingConfigurationSourceAutoEnabled(); + } + + private static boolean isFeatureFlaggingConfigurationSourceAutoEnabled() { + final String configurationSource = + getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE); + if (configurationSource == null) { + return true; + } + final String normalized = configurationSource.trim().toLowerCase(Locale.ROOT); + return "agentless".equals(normalized) + || "remote_config".equals(normalized) + || "offline".equals(normalized); + } + + private static String getConfiguredValue(final String configKey) { + final String systemProp = propertyNameToSystemPropertyName(configKey); + String settingValue = getNullIfEmpty(SystemProperties.get(systemProp)); + if (settingValue == null) { + settingValue = getNullIfEmpty(getStableConfig(FLEET, configKey)); + } + if (settingValue == null) { + settingValue = getNullIfEmpty(ddGetEnv(systemProp)); + } + if (settingValue == null) { + settingValue = getNullIfEmpty(getStableConfig(LOCAL, configKey)); + } + return settingValue; + } + /** * @see datadog.trace.api.ProductActivation#fromString(String) */ diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 1aba2cb276b..c3c46201c69 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,6 +49,12 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = + 30.0D; + public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = + 2.0D; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..97d7c105caa 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -2,5 +2,19 @@ public class FeatureFlaggingConfig { + public static final String FEATURE_FLAGS_ENABLED = "feature.flags.enabled"; public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = + "feature.flags.configuration.source"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = + "feature.flags.configuration.source.agentless.base.url"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = + "feature.flags.configuration.source.agentless.poll.interval.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = + "feature.flags.configuration.source.agentless.request.timeout.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS = + "feature.flags.configuration.source.agentless.extra.headers"; + + private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 5ced61622ca..82c64929f4f 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -86,6 +86,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_SERVER_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_HEALTH_METRICS_ENABLED; @@ -361,6 +364,11 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_EXCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_QUEUE_SIZE; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL; @@ -1213,6 +1221,12 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; + private final String featureFlaggingConfigurationSource; + private final String featureFlaggingConfigurationSourceBaseUrl; + private final double featureFlaggingConfigurationSourcePollIntervalSeconds; + private final double featureFlaggingConfigurationSourceRequestTimeoutSeconds; + private final Map featureFlaggingConfigurationSourceExtraHeaders; + private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; private final boolean dbmTracePreparedStatements; @@ -2835,6 +2849,25 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); + featureFlaggingConfigurationSource = + normalizeFeatureFlaggingConfigurationSource( + configProvider.getString( + FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + featureFlaggingConfigurationSourceBaseUrl = + configProvider.getStringNotEmpty( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); + featureFlaggingConfigurationSourcePollIntervalSeconds = + configProvider.getDouble( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); + featureFlaggingConfigurationSourceRequestTimeoutSeconds = + configProvider.getDouble( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); + featureFlaggingConfigurationSourceExtraHeaders = + configProvider.getMergedMap( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS, '='); + dynamicInstrumentationEnabled = configProvider.getBoolean( DYNAMIC_INSTRUMENTATION_ENABLED, DEFAULT_DYNAMIC_INSTRUMENTATION_ENABLED); @@ -3747,6 +3780,14 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } + private static String normalizeFeatureFlaggingConfigurationSource(final String source) { + if (source == null) { + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; + } + final String normalized = source.trim().toLowerCase(Locale.ROOT); + return normalized.isEmpty() ? DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE : normalized; + } + public boolean isBaggageExtract() { return tracePropagationStylesToExtract.contains(TracePropagationStyle.BAGGAGE); } @@ -4654,6 +4695,26 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } + public String getFeatureFlaggingConfigurationSource() { + return featureFlaggingConfigurationSource; + } + + public String getFeatureFlaggingConfigurationSourceBaseUrl() { + return featureFlaggingConfigurationSourceBaseUrl; + } + + public double getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { + return featureFlaggingConfigurationSourcePollIntervalSeconds; + } + + public double getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { + return featureFlaggingConfigurationSourceRequestTimeoutSeconds; + } + + public Map getFeatureFlaggingConfigurationSourceExtraHeaders() { + return featureFlaggingConfigurationSourceExtraHeaders; + } + public boolean isDynamicInstrumentationEnabled() { return dynamicInstrumentationEnabled; } @@ -6486,6 +6547,16 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled + + ", featureFlaggingConfigurationSource=" + + featureFlaggingConfigurationSource + + ", featureFlaggingConfigurationSourceBaseUrl=" + + featureFlaggingConfigurationSourceBaseUrl + + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + + featureFlaggingConfigurationSourcePollIntervalSeconds + + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" + + featureFlaggingConfigurationSourceRequestTimeoutSeconds + + ", featureFlaggingConfigurationSourceExtraHeaderNames=" + + featureFlaggingConfigurationSourceExtraHeaders.keySet() + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..19c2b2fdce4 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1497,6 +1497,54 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ + { + "version": "A", + "type": "string", + "default": "agentless", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS": [ + { + "version": "A", + "type": "map", + "default": null, + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ + { + "version": "A", + "type": "decimal", + "default": "30", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ + { + "version": "A", + "type": "decimal", + "default": "2", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] + } + ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..f2db8754a27 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -9,7 +9,7 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - private static volatile RemoteConfigService CONFIG_SERVICE; + private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; private FeatureFlaggingSystem() {} @@ -17,12 +17,11 @@ private FeatureFlaggingSystem() {} public static void start(final SharedCommunicationObjects sco) { LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + CONFIG_SERVICE = createConfigurationSourceService(sco, config); - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); + if (CONFIG_SERVICE != null) { + CONFIG_SERVICE.init(); } - CONFIG_SERVICE = new RemoteConfigServiceImpl(sco, config); - CONFIG_SERVICE.init(); EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); @@ -30,6 +29,30 @@ public static void start(final SharedCommunicationObjects sco) { LOGGER.debug("Feature Flagging system started"); } + static ConfigurationSourceService createConfigurationSourceService( + final SharedCommunicationObjects sco, final Config config) { + final ConfigurationSource configurationSource = + ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); + + switch (configurationSource) { + case REMOTE_CONFIG: + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + return new RemoteConfigServiceImpl(sco, config); + case AGENTLESS: + return new UfcHttpConfigService(config); + case OFFLINE: + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + return null; + default: + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + + config.getFeatureFlaggingConfigurationSource()); + } + } + public static void stop() { if (EXPOSURE_WRITER != null) { EXPOSURE_WRITER.close(); @@ -41,4 +64,26 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } + + private enum ConfigurationSource { + AGENTLESS("agentless"), + REMOTE_CONFIG("remote_config"), + OFFLINE("offline"); + + private final String value; + + ConfigurationSource(final String value) { + this.value = value; + } + + private static ConfigurationSource from(final String value) { + for (final ConfigurationSource source : values()) { + if (source.value.equals(value)) { + return source; + } + } + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + value); + } + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..df8511aa3a7 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,6 +1,9 @@ package com.datadog.featureflag; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -23,6 +26,8 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -48,6 +53,7 @@ void testFeatureFlagSystemInitialization() { } @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); @@ -60,4 +66,57 @@ void testThatRemoteConfigIsRequired() { FeatureFlaggingSystem.stop(); } } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { + assertInstanceOf( + UfcHttpConfigService.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") + void explicitRemoteConfigUsesRemoteConfigService() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))) + .thenReturn(mock(ConfigurationPoller.class)); + + assertInstanceOf( + RemoteConfigServiceImpl.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects, Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") + void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { + assertThrows( + IllegalArgumentException.class, + () -> + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void offlineConfigurationSourceDoesNotStartNetworkSource() { + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + private static SharedCommunicationObjects sharedCommunicationObjects() { + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + return sharedCommunicationObjects; + } } diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 47733020559..b55f9ad988c 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -85,5 +85,5 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc ## Requirements - Java 11+ -- Datadog Agent with Remote Configuration enabled -- `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=true` +- Feature Flags provider enabled with `DD_FEATURE_FLAGS_ENABLED=true`, or left unset + for the default `agentless` configuration source. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java similarity index 60% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java index 5f84a78f7d4..83495545717 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java @@ -2,7 +2,7 @@ import java.io.Closeable; -public interface RemoteConfigService extends Closeable { +public interface ConfigurationSourceService extends Closeable { void init(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index ec28f684a96..a5a9b2d7f36 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -32,7 +32,7 @@ import okio.Okio; public class RemoteConfigServiceImpl - implements RemoteConfigService, ConfigurationChangesTypedListener { + implements ConfigurationSourceService, ConfigurationChangesTypedListener { private final ConfigurationPoller configurationPoller; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java new file mode 100644 index 00000000000..f0ecfa049ab --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java @@ -0,0 +1,295 @@ +package com.datadog.featureflag; + +import static datadog.communication.http.OkHttpUtils.prepareRequest; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URLEncoder; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class UfcHttpConfigService implements ConfigurationSourceService { + private static final Logger LOGGER = LoggerFactory.getLogger(UfcHttpConfigService.class); + + private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = + "/api/v2/feature-flagging/config/server-distribution"; + private static final int MAX_ATTEMPTS = 3; + + private final HttpUrl endpoint; + private final Config config; + private final Map extraHeaders; + private final long pollIntervalMillis; + private final UfcHttpClient client; + private final ScheduledExecutorService executor; + private final AtomicBoolean polling = new AtomicBoolean(); + private volatile boolean closed; + private volatile ScheduledFuture scheduledPoll; + private volatile String etag; + + UfcHttpConfigService(final Config config) { + this(config, endpoint(config)); + } + + private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { + this( + endpoint, + config, + millis(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()), + new OkHttpUfcHttpClient( + OkHttpUtils.buildHttpClient( + endpoint, + millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()))), + Executors.newSingleThreadScheduledExecutor(new UfcHttpThreadFactory())); + } + + UfcHttpConfigService( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor) { + this.endpoint = endpoint; + this.config = config; + final Map configuredExtraHeaders = + config.getFeatureFlaggingConfigurationSourceExtraHeaders(); + this.extraHeaders = + configuredExtraHeaders == null ? Collections.emptyMap() : configuredExtraHeaders; + this.pollIntervalMillis = pollIntervalMillis; + this.client = client; + this.executor = executor; + } + + @Override + public void init() { + if (closed) { + return; + } + scheduledPoll = + executor.scheduleWithFixedDelay( + this::pollOnceSafely, 0, pollIntervalMillis, TimeUnit.MILLISECONDS); + } + + boolean pollOnce() { + if (closed || !polling.compareAndSet(false, true)) { + return false; + } + try { + return fetchAndApply(); + } finally { + polling.set(false); + } + } + + @Override + public void close() { + closed = true; + if (scheduledPoll != null) { + scheduledPoll.cancel(true); + scheduledPoll = null; + } + executor.shutdownNow(); + } + + private void pollOnceSafely() { + try { + pollOnce(); + } catch (final RuntimeException e) { + LOGGER.debug("Unexpected error while polling Feature Flagging HTTP configuration source", e); + } + } + + private boolean fetchAndApply() { + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + final UfcHttpResponse response = client.fetch(endpoint, config, extraHeaders, etag); + if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { + continue; + } + return apply(response); + } catch (final IOException e) { + if (attempt == MAX_ATTEMPTS) { + LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); + return false; + } + } + } + return false; + } + + private boolean apply(final UfcHttpResponse response) { + if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { + updateEtag(response.etag); + return true; + } + if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED + || response.status == HttpURLConnection.HTTP_FORBIDDEN + || response.status != HttpURLConnection.HTTP_OK + || response.body == null) { + return false; + } + final ServerConfiguration configuration; + try { + configuration = + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( + response.body); + } catch (final IOException | RuntimeException e) { + LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); + return false; + } + if (configuration == null) { + return false; + } + updateEtag(response.etag); + FeatureFlaggingGateway.dispatch(configuration); + return true; + } + + private static boolean isRetryableStatus(final int status) { + return status == HttpURLConnection.HTTP_CLIENT_TIMEOUT + || status == 429 + || (status >= 500 && status <= 599); + } + + private void updateEtag(final String nextEtag) { + if (nextEtag != null && !nextEtag.trim().isEmpty()) { + etag = nextEtag; + } + } + + static HttpUrl endpoint(final Config config) { + final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceBaseUrl(); + final String endpoint = + configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredUrl(configuredBaseUrl); + final HttpUrl parsed = HttpUrl.parse(endpoint); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); + } + return parsed; + } + + private static String endpointFromConfiguredUrl(final String configuredUrl) { + final HttpUrl parsed = HttpUrl.parse(configuredUrl.trim()); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + configuredUrl); + } + if (isRootPath(parsed)) { + return parsed + .newBuilder() + .addPathSegments(datadogApiServerDistributionPath()) + .build() + .toString(); + } + return parsed.toString(); + } + + private static boolean isRootPath(final HttpUrl url) { + return "/".equals(url.encodedPath()) || url.encodedPath().isEmpty(); + } + + private static String datadogApiServerDistributionEndpoint(final Config config) { + final StringBuilder endpoint = + new StringBuilder("https://api.") + .append(config.getSite().toLowerCase(Locale.ROOT)) + .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); + final String env = config.getEnv(); + if (env != null && !env.isEmpty()) { + endpoint.append("?dd_env=").append(urlEncode(env)); + } + return endpoint.toString(); + } + + private static String datadogApiServerDistributionPath() { + return DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1); + } + + private static String urlEncode(final String value) { + try { + return URLEncoder.encode(value, "UTF-8"); + } catch (final IOException e) { + throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + } + } + + static long millis(final double seconds) { + return Math.max(1L, Math.round(seconds * 1000)); + } + + interface UfcHttpClient { + UfcHttpResponse fetch( + HttpUrl endpoint, Config config, Map extraHeaders, String etag) + throws IOException; + } + + static final class UfcHttpResponse { + final int status; + final String etag; + final byte[] body; + + UfcHttpResponse(final int status, final String etag, final byte[] body) { + this.status = status; + this.etag = etag; + this.body = body; + } + } + + private static final class OkHttpUfcHttpClient implements UfcHttpClient { + private final OkHttpClient httpClient; + + private OkHttpUfcHttpClient(final OkHttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public UfcHttpResponse fetch( + final HttpUrl endpoint, + final Config config, + final Map extraHeaders, + final String etag) + throws IOException { + final Map headers = new HashMap<>(extraHeaders); + if (etag != null) { + headers.put("If-None-Match", etag); + } + final Request request = prepareRequest(endpoint, headers, config, true).get().build(); + try (Response response = httpClient.newCall(request).execute()) { + final ResponseBody responseBody = response.body(); + return new UfcHttpResponse( + response.code(), + response.header("ETag"), + responseBody == null ? null : responseBody.bytes()); + } + } + } + + private static final class UfcHttpThreadFactory implements ThreadFactory { + @Override + public Thread newThread(final Runnable runnable) { + final Thread thread = new Thread(runnable, "dd-feature-flagging-http-poller"); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java new file mode 100644 index 00000000000..107b06bf53f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java @@ -0,0 +1,313 @@ +package com.datadog.featureflag; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class UfcHttpConfigServiceTest { + + @Mock private FeatureFlaggingGateway.ConfigListener listener; + + @AfterEach + void cleanup() { + FeatureFlaggingGateway.removeConfigListener(listener); + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + @Test + void appendsDatadogApiServerDistributionPathWhenConfiguredUrlIsRoot() { + final Config config = config("http://mock-backend:8092", "datadoghq.com", ""); + + assertEquals( + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution", + UfcHttpConfigService.endpoint(config).toString()); + } + + @Test + void preservesConfiguredFullEndpointUrl() { + final Config config = + config( + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", + "datadoghq.com", + ""); + + assertEquals( + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", + UfcHttpConfigService.endpoint(config).toString()); + } + + @Test + void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { + final Config config = config(null, "datad0g.com", "staging env"); + + assertEquals( + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + UfcHttpConfigService.endpoint(config).toString()); + } + + @Test + void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final UfcHttpConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("test-api-key", client.requests.get(0).apiKey); + assertEquals("app-key", client.requests.get(0).extraHeaders.get("DD-APPLICATION-KEY")); + assertEquals("1", client.requests.get(0).extraHeaders.get("fastly-client")); + assertNull(client.requests.get(0).etag); + } + + @Test + void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { + final FakeClient client = + new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); + final UfcHttpConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + } + + @Test + void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { + final FakeClient client = + new FakeClient( + response(401, null, null), + response(200, null, "{not-json}"), + response(200, null, "{\"flags\":[]}")); + final UfcHttpConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verifyNoInteractions(listener); + } + + @Test + void retriesTimeoutBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final UfcHttpConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { + final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); + final UfcHttpConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(2, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void rejectsOverlappingPolls() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final UfcHttpConfigService service = service(client, extraHeaders()); + final ExecutorService runner = Executors.newFixedThreadPool(2); + + try { + final Future first = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + final Future second = runner.submit(service::pollOnce); + + assertFalse(second.get(1, TimeUnit.SECONDS)); + releaseRequest.countDown(); + assertTrue(first.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closePreventsFurtherPolls() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final UfcHttpConfigService service = service(client, extraHeaders()); + + service.close(); + + assertFalse(service.pollOnce()); + assertEquals(0, client.calls.get()); + } + + private static UfcHttpConfigService service( + final FakeClient client, final Map extraHeaders) { + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + return new UfcHttpConfigService( + HttpUrl.get("http://localhost/api/v2/feature-flagging/config/server-distribution"), + config("http://localhost", "datadoghq.com", "", extraHeaders), + 60_000, + client, + executor); + } + + private static Config config(final String baseUrl, final String site, final String env) { + return config(baseUrl, site, env, new HashMap<>()); + } + + private static Config config( + final String baseUrl, + final String site, + final String env, + final Map extraHeaders) { + final Config config = mock(Config.class); + lenient().when(config.getFeatureFlaggingConfigurationSourceBaseUrl()).thenReturn(baseUrl); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceExtraHeaders()) + .thenReturn(extraHeaders); + lenient() + .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) + .thenReturn(30.0D); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) + .thenReturn(2.0D); + lenient().when(config.getApiKey()).thenReturn("test-api-key"); + lenient().when(config.getSite()).thenReturn(site); + lenient().when(config.getEnv()).thenReturn(env); + return config; + } + + private static Map extraHeaders() { + final Map headers = new HashMap<>(); + headers.put("DD-APPLICATION-KEY", "app-key"); + headers.put("fastly-client", "1"); + return headers; + } + + private static UfcHttpConfigService.UfcHttpResponse response( + final int status, final String etag, final String body) { + return new UfcHttpConfigService.UfcHttpResponse( + status, etag, body == null ? null : body.getBytes(UTF_8)); + } + + private static String emptyConfig() { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + + private static final class FakeClient implements UfcHttpConfigService.UfcHttpClient { + private final AtomicInteger calls = new AtomicInteger(); + private final List requests = new ArrayList<>(); + private final BlockingQueue responses = new LinkedBlockingQueue<>(); + private CountDownLatch requestStarted; + private CountDownLatch releaseRequest; + + private FakeClient(final Object... responses) { + for (final Object response : responses) { + this.responses.add(response); + } + } + + private void block(final CountDownLatch requestStarted, final CountDownLatch releaseRequest) { + this.requestStarted = requestStarted; + this.releaseRequest = releaseRequest; + } + + @Override + public UfcHttpConfigService.UfcHttpResponse fetch( + final HttpUrl endpoint, + final Config config, + final Map extraHeaders, + final String etag) + throws IOException { + calls.incrementAndGet(); + requests.add(new Request(config.getApiKey(), etag, extraHeaders)); + if (requestStarted != null) { + requestStarted.countDown(); + } + if (releaseRequest != null) { + await(releaseRequest); + } + final Object response = responses.remove(); + if (response instanceof IOException) { + throw (IOException) response; + } + return (UfcHttpConfigService.UfcHttpResponse) response; + } + + private static void await(final CountDownLatch latch) throws IOException { + try { + if (!latch.await(1, TimeUnit.SECONDS)) { + throw new SocketTimeoutException("test request did not release"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + } + + private static final class Request { + private final String apiKey; + private final String etag; + private final Map extraHeaders; + + private Request( + final String apiKey, final String etag, final Map extraHeaders) { + this.apiKey = apiKey; + this.etag = etag; + this.extraHeaders = new HashMap<>(extraHeaders); + } + } +}