Inferred Routes and Resource Renaming: new APM trace metrics tag

dd-smoke-tests/springboot/src/test/groovy/datadog/smoketest/HttpEndpointTaggingSmokeTest.groovy

@@ -0,0 +1,252 @@
+package datadog.smoketest
+
+import okhttp3.Request
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.regex.Pattern
+
+class HttpEndpointTaggingSmokeTest extends AbstractServerSmokeTest {
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    String springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path")
+
+    List<String> command = new ArrayList<>()
+    command.add(javaPath())
+    command.addAll(defaultJavaProperties)
+    command.addAll((String[]) [
+      "-Ddd.writer.type=MultiWriter:TraceStructureWriter:${output.getAbsolutePath()}:includeResource,DDAgentWriter",
+      "-Ddd.trace.resource.renaming.enabled=true",
+      "-jar",
+      springBootShadowJar,
+      "--server.port=${httpPort}"
+    ])
+    ProcessBuilder processBuilder = new ProcessBuilder(command)
+    processBuilder.directory(new File(buildDirectory))
+  }
+
+  @Override
+  File createTemporaryFile() {
+    return File.createTempFile("http-endpoint-tagging-trace", "out")
+  }
+
+  @Override
+  protected Set<String> expectedTraces() {
+    return [
+      Pattern.quote("[servlet.request:GET /greeting[spring.handler:IastWebController.greeting]]")
+    ]
+  }
+
+  @Override
+  protected Set<String> assertTraceCounts(Set<String> expected, Map<String, AtomicInteger> traceCounts) {
+    List<Pattern> remaining = expected.collect { Pattern.compile(it) }.toList()
+    for (def i = remaining.size() - 1; i >= 0; i--) {
+      for (Map.Entry<String, AtomicInteger> entry : traceCounts.entrySet()) {
+        if (entry.getValue() > 0 && remaining.get(i).matcher(entry.getKey()).matches()) {
+          remaining.remove(i)
+          break
+        }
+      }
+    }
+    return remaining.collect { it.pattern() }.toSet()
+  }
+
+  def "test basic HTTP endpoint tagging"() {
+    setup:
+    String url = "http://localhost:${httpPort}/greeting"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    def responseBodyStr = response.body().string()
+    responseBodyStr != null
+    responseBodyStr.contains("Sup Dawg")
+    response.code() == 200
+    waitForTraceCount(1)
+  }
+
+  def "test URL parameterization for numeric IDs"() {
+    setup:
+    String url = "http://localhost:${httpPort}/users/123"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    // May return 404 since endpoint doesn't exist, but span should still be created
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test URL parameterization for hex patterns"() {
+    setup:
+    String url = "http://localhost:${httpPort}/session/abc123def456"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    // May return 404 since endpoint doesn't exist, but span should still be created
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test int_id pattern parameterization"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/versions/12.34.56"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test hex_id pattern parameterization"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/tokens/550e8400-e29b-41d4-a716-446655440000"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test str pattern parameterization for long strings"() {
+    setup:
+    String url = "http://localhost:${httpPort}/files/very-long-filename-that-exceeds-twenty-characters.pdf"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test str pattern parameterization for special characters"() {
+    setup:
+    String url = "http://localhost:${httpPort}/search/query%20with%20spaces"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test mixed URL patterns with multiple segments"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/users/123/orders/abc456def/items/789"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test URL with query parameters"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/users/123?filter=active&limit=10"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test URL with trailing slash"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/users/456/"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test static paths are preserved"() {
+    setup:
+    String url = "http://localhost:${httpPort}/health"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test root path handling"() {
+    setup:
+    String url = "http://localhost:${httpPort}/"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test pattern precedence - int pattern wins over int_id"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/items/12345"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test pattern precedence - hex pattern wins over hex_id"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/hashes/abc123def"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+
+  def "test edge case - short segments not parameterized"() {
+    setup:
+    String url = "http://localhost:${httpPort}/api/x/y"
+    def request = new Request.Builder().url(url).get().build()
+
+    when:
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() in [200, 404]
+    waitForTraceCount(1)
+  }
+}

dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

@@ -170,6 +170,12 @@ public final class ConfigDefaults {
       "datadog.trace.*:org.apache.commons.*:org.mockito.*";
   static final boolean DEFAULT_CIVISIBILITY_GIT_UPLOAD_ENABLED = true;
   static final boolean DEFAULT_CIVISIBILITY_GIT_UNSHALLOW_ENABLED = true;
+
+  // HTTP Endpoint Tagging feature flags
+  static final boolean DEFAULT_RESOURCE_RENAMING_ENABLED =
+      false; // Default enablement of resource renaming
+  static final boolean DEFAULT_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT =
+      false; // Manual disablement of resource renaming
   static final long DEFAULT_CIVISIBILITY_GIT_COMMAND_TIMEOUT_MILLIS = 30_000;
   static final long DEFAULT_CIVISIBILITY_BACKEND_API_TIMEOUT_MILLIS = 30_000;
   static final long DEFAULT_CIVISIBILITY_GIT_UPLOAD_TIMEOUT_MILLIS = 60_000;

dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java

@@ -68,6 +68,11 @@ public final class TracerConfig {
       "trace.http.resource.remove-trailing-slash";
   public static final String TRACE_HTTP_SERVER_PATH_RESOURCE_NAME_MAPPING =
       "trace.http.server.path-resource-name-mapping";
+
+  // HTTP Endpoint Tagging feature flags
+  public static final String TRACE_RESOURCE_RENAMING_ENABLED = "trace.resource.renaming.enabled";
+  public static final String TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT =
+      "trace.resource.renaming.always-simplified-endpoint";
   public static final String TRACE_HTTP_CLIENT_PATH_RESOURCE_NAME_MAPPING =
       "trace.http.client.path-resource-name-mapping";
   // Use TRACE_HTTP_SERVER_ERROR_STATUSES instead

dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java

@@ -3,6 +3,8 @@
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT;
 import static datadog.trace.api.DDTags.BASE_SERVICE;
 import static datadog.trace.api.Functions.UTF8_ENCODE;
+import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ENDPOINT;
+import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_METHOD;
 import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND;
 import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT;
 import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CONSUMER;
@@ -100,6 +102,7 @@ public final class ConflatingMetricsAggregator implements MetricsAggregator, Eve
   private final TimeUnit reportingIntervalTimeUnit;
   private final DDAgentFeaturesDiscovery features;
   private final HealthMetrics healthMetrics;
+  private final boolean resourceRenamingEnabled;
 
   private volatile AgentTaskScheduler.Scheduled<?> cancellation;
 
@@ -120,7 +123,8 @@ public ConflatingMetricsAggregator(
             false,
             DEFAULT_HEADERS),
         config.getTracerMetricsMaxAggregates(),
-        config.getTracerMetricsMaxPending());
+        config.getTracerMetricsMaxPending(),
+        config.isResourceRenamingEnabled());
   }
 
   ConflatingMetricsAggregator(
@@ -140,7 +144,30 @@ public ConflatingMetricsAggregator(
         maxAggregates,
         queueSize,
         10,
-        SECONDS);
+        SECONDS,
+        false);
+  }
+
+  ConflatingMetricsAggregator(
+      WellKnownTags wellKnownTags,
+      Set<String> ignoredResources,
+      DDAgentFeaturesDiscovery features,
+      HealthMetrics healthMetric,
+      Sink sink,
+      int maxAggregates,
+      int queueSize,
+      boolean resourceRenamingEnabled) {
+    this(
+        wellKnownTags,
+        ignoredResources,
+        features,
+        healthMetric,
+        sink,
+        maxAggregates,
+        queueSize,
+        10,
+        SECONDS,
+        resourceRenamingEnabled);
   }
 
   ConflatingMetricsAggregator(
@@ -152,7 +179,8 @@ public ConflatingMetricsAggregator(
       int maxAggregates,
       int queueSize,
       long reportingInterval,
-      TimeUnit timeUnit) {
+      TimeUnit timeUnit,
+      boolean resourceRenamingEnabled) {
     this(
         ignoredResources,
         features,
@@ -162,7 +190,8 @@ public ConflatingMetricsAggregator(
         maxAggregates,
         queueSize,
         reportingInterval,
-        timeUnit);
+        timeUnit,
+        resourceRenamingEnabled);
   }
 
   ConflatingMetricsAggregator(
@@ -174,7 +203,8 @@ public ConflatingMetricsAggregator(
       int maxAggregates,
       int queueSize,
       long reportingInterval,
-      TimeUnit timeUnit) {
+      TimeUnit timeUnit,
+      boolean resourceRenamingEnabled) {
     this.ignoredResources = ignoredResources;
     this.inbox = new MpscCompoundQueue<>(queueSize);
     this.batchPool = new SpmcArrayQueue<>(maxAggregates);
@@ -183,6 +213,7 @@ public ConflatingMetricsAggregator(
     this.features = features;
     this.healthMetrics = healthMetric;
     this.sink = sink;
+    this.resourceRenamingEnabled = resourceRenamingEnabled;
     this.aggregator =
         new Aggregator(
             metricWriter,
@@ -307,6 +338,9 @@ private boolean spanKindEligible(CoreSpan<?> span) {
 
   private boolean publish(CoreSpan<?> span, boolean isTopLevel) {
     final CharSequence spanKind = span.getTag(SPAN_KIND, "");
+    // Only include HTTP tags in metric key when resource renaming is enabled
+    final CharSequence httpMethod = resourceRenamingEnabled ? span.getTag(HTTP_METHOD, "") : "";
+    final CharSequence httpEndpoint = resourceRenamingEnabled ? span.getTag(HTTP_ENDPOINT, "") : "";
     MetricKey newKey =
         new MetricKey(
             span.getResourceName(),
@@ -318,7 +352,9 @@ private boolean publish(CoreSpan<?> span, boolean isTopLevel) {
             span.getParentId() == 0,
             SPAN_KINDS.computeIfAbsent(
                 spanKind, UTF8BytesString::create), // save repeated utf8 conversions
-            getPeerTags(span, spanKind.toString()));
+            getPeerTags(span, spanKind.toString()),
+            httpMethod,
+            httpEndpoint);
     boolean isNewKey = false;
     MetricKey key = keys.putIfAbsent(newKey, newKey);
     if (null == key) {