[WIP] Add JFR scrubbing before profile upload

dd-java-agent/agent-profiling/build.gradle

@@ -13,7 +13,9 @@ excludedClassesCoverage += [
   'com.datadog.profiling.agent.ProfilingAgent',
   'com.datadog.profiling.agent.ProfilingAgent.ShutdownHook',
   'com.datadog.profiling.agent.ProfilingAgent.DataDumper',
-  'com.datadog.profiling.agent.ProfilerFlare'
+  'com.datadog.profiling.agent.ProfilerFlare',
+  'com.datadog.profiling.agent.ScrubRecordingDataListener',
+  'com.datadog.profiling.agent.ScrubRecordingDataListener.ScrubbedRecordingData'
 ]
 
 dependencies {
@@ -23,6 +25,7 @@ dependencies {
   api project(':dd-java-agent:agent-profiling:profiling-ddprof')
   api project(':dd-java-agent:agent-profiling:profiling-uploader')
   api project(':dd-java-agent:agent-profiling:profiling-controller')
+  implementation project(':dd-java-agent:agent-profiling:profiling-scrubber')
   api project(':dd-java-agent:agent-profiling:profiling-controller-jfr')
   api project(':dd-java-agent:agent-profiling:profiling-controller-jfr:implementation')
   api project(':dd-java-agent:agent-profiling:profiling-controller-ddprof')
@@ -42,6 +45,11 @@ configurations {
 
 tasks.named("shadowJar", ShadowJar) {
   dependencies deps.excludeShared
+
+  // Exclude multi-release versioned classes from jafar-parser.
+  // These are duplicates of base classes for newer Java APIs and confuse
+  // the GraalVM native-image builder when the profiling jar is embedded in the agent.
+  exclude 'META-INF/versions/**'
 }
 
 tasks.named("jar", Jar) {

dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java

@@ -7,6 +7,7 @@
 import java.nio.file.Path;
 import java.time.Instant;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 final class DatadogProfilerRecordingData extends RecordingData {
   private final Path recordingFile;
@@ -36,4 +37,10 @@ public void release() {
   public String getName() {
     return "ddprof";
   }
+
+  @Nullable
+  @Override
+  public Path getPath() {
+    return recordingFile;
+  }
 }

dd-java-agent/agent-profiling/profiling-scrubber/build.gradle

@@ -0,0 +1,17 @@
+apply from: "$rootDir/gradle/java.gradle"
+
+minimumInstructionCoverage = 0.0
+minimumBranchCoverage = 0.0
+
+dependencies {
+  api libs.slf4j
+
+  implementation(libs.jafar.tools) {
+    // Agent has its own slf4j binding
+    exclude group: 'org.slf4j', module: 'slf4j-simple'
+  }
+
+  testImplementation libs.bundles.junit5
+  testImplementation libs.bundles.mockito
+  testImplementation libs.bundles.jmc
+}

dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/DefaultScrubDefinition.java

@@ -0,0 +1,53 @@
+package com.datadog.profiling.scrubber;
+
+import io.jafar.tools.Scrubber;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/** Provides the default scrub definition targeting sensitive JFR event fields. */
+public final class DefaultScrubDefinition {
+
+  private static final Map<String, Scrubber.ScrubField> DEFAULT_SCRUB_FIELDS;
+
+  static {
+    Map<String, Scrubber.ScrubField> fields = new HashMap<>();
+    // ScrubField(keyField, valueField, predicate): null keyField = scrub all values unconditionally
+    // System properties may contain API keys, passwords
+    fields.put("jdk.InitialSystemProperty", new Scrubber.ScrubField(null, "value", (k, v) -> true));
+    // JVM args may contain credentials in -D flags
+    fields.put("jdk.JVMInformation", new Scrubber.ScrubField(null, "jvmArguments", (k, v) -> true));
+    // Env vars may contain secrets
+    fields.put(
+        "jdk.InitialEnvironmentVariable", new Scrubber.ScrubField(null, "value", (k, v) -> true));
+    // Process command lines may reveal infrastructure
+    fields.put("jdk.SystemProcess", new Scrubber.ScrubField(null, "commandLine", (k, v) -> true));
+    DEFAULT_SCRUB_FIELDS = Collections.unmodifiableMap(fields);
+  }
+
+  /**
+   * Creates a scrubber with the default scrub definition.
+   *
+   * @param excludeEventTypes list of event type names to exclude from scrubbing, or null for none
+   * @return a configured {@link JfrScrubber}
+   */
+  public static JfrScrubber create(List<String> excludeEventTypes) {
+    Set<String> excludeSet =
+        excludeEventTypes != null
+            ? new HashSet<>(excludeEventTypes)
+            : Collections.<String>emptySet();
+
+    return new JfrScrubber(
+        eventTypeName -> {
+          if (excludeSet.contains(eventTypeName)) {
+            return null;
+          }
+          return DEFAULT_SCRUB_FIELDS.get(eventTypeName);
+        });
+  }
+
+  private DefaultScrubDefinition() {}
+}

dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/JfrScrubber.java

@@ -0,0 +1,30 @@
+package com.datadog.profiling.scrubber;
+
+import io.jafar.tools.Scrubber;
+import java.nio.file.Path;
+import java.util.function.Function;
+
+/**
+ * Thin wrapper around {@link Scrubber} from jafar-tools, hiding jafar types from consumers outside
+ * the profiling-scrubber module.
+ */
+public final class JfrScrubber {
+
+  private final Function<String, Scrubber.ScrubField> scrubDefinition;
+
+  /** Package-private: use {@link DefaultScrubDefinition#create} to obtain an instance. */
+  JfrScrubber(Function<String, Scrubber.ScrubField> scrubDefinition) {
+    this.scrubDefinition = scrubDefinition;
+  }
+
+  /**
+   * Scrub the given file by replacing targeted field values with 'x' bytes.
+   *
+   * @param input the input file to scrub
+   * @param output the output file to write the scrubbed content to
+   * @throws Exception if an error occurs during parsing or writing
+   */
+  public void scrubFile(Path input, Path output) throws Exception {
+    Scrubber.scrubFile(input, output, scrubDefinition);
+  }
+}

dd-java-agent/agent-profiling/profiling-scrubber/src/test/java/com/datadog/profiling/scrubber/JfrScrubberTest.java

@@ -0,0 +1,116 @@
+package com.datadog.profiling.scrubber;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.openjdk.jmc.common.item.Attribute.attr;
+import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.openjdk.jmc.common.item.IAttribute;
+import org.openjdk.jmc.common.item.IItem;
+import org.openjdk.jmc.common.item.IItemCollection;
+import org.openjdk.jmc.common.item.IItemIterable;
+import org.openjdk.jmc.common.item.IMemberAccessor;
+import org.openjdk.jmc.common.item.ItemFilters;
+import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit;
+
+class JfrScrubberTest {
+
+  @TempDir Path tempDir;
+
+  private Path inputFile;
+
+  @BeforeEach
+  void setUp() throws IOException {
+    inputFile = tempDir.resolve("input.jfr");
+    try (InputStream is = getClass().getResourceAsStream("/test-recording.jfr")) {
+      if (is == null) {
+        throw new IllegalStateException("test-recording.jfr not found in test resources");
+      }
+      Files.copy(is, inputFile, StandardCopyOption.REPLACE_EXISTING);
+    }
+  }
+
+  @Test
+  void scrubInitialSystemPropertyValues() throws Exception {
+    JfrScrubber scrubber = DefaultScrubDefinition.create(null);
+    Path outputFile = tempDir.resolve("output.jfr");
+    scrubber.scrubFile(inputFile, outputFile);
+
+    assertTrue(Files.exists(outputFile));
+    assertTrue(Files.size(outputFile) > 0, "Scrubbed file should not be empty");
+
+    // Verify scrubbed values contain only 'x' characters
+    IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
+    IItemCollection systemPropertyEvents =
+        events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
+    assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
+
+    IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
+    for (IItemIterable itemIterable : systemPropertyEvents) {
+      IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
+      for (IItem item : itemIterable) {
+        String value = accessor.getMember(item);
+        if (value != null && !value.isEmpty()) {
+          assertTrue(
+              value.chars().allMatch(c -> c == 'x'),
+              "System property value should be scrubbed: " + value);
+        }
+      }
+    }
+  }
+
+  @Test
+  void scrubWithNoMatchingEvents() throws Exception {
+    // Scrubber with all default events excluded — nothing matches
+    JfrScrubber scrubber = new JfrScrubber(name -> null);
+    Path outputFile = tempDir.resolve("output.jfr");
+    scrubber.scrubFile(inputFile, outputFile);
+
+    // Output should be identical to input when no events match
+    assertEquals(Files.size(inputFile), Files.size(outputFile));
+  }
+
+  @Test
+  void scrubWithExcludedEventType() throws Exception {
+    // Exclude jdk.InitialSystemProperty from scrubbing
+    JfrScrubber scrubber =
+        DefaultScrubDefinition.create(Collections.singletonList("jdk.InitialSystemProperty"));
+    Path outputFile = tempDir.resolve("output.jfr");
+    scrubber.scrubFile(inputFile, outputFile);
+
+    assertTrue(Files.exists(outputFile));
+    assertTrue(Files.size(outputFile) > 0);
+
+    // Verify excluded event type values are preserved (not scrubbed to 'x')
+    IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
+    IItemCollection systemPropertyEvents =
+        events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
+    assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
+
+    IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
+    boolean foundNonTrivialValue = false;
+    for (IItemIterable itemIterable : systemPropertyEvents) {
+      IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
+      for (IItem item : itemIterable) {
+        String value = accessor.getMember(item);
+        if (value != null && !value.isEmpty()) {
+          // At least one value should NOT be all-x (proving exclusion worked)
+          if (!value.chars().allMatch(c -> c == 'x')) {
+            foundNonTrivialValue = true;
+          }
+        }
+      }
+    }
+    assertTrue(
+        foundNonTrivialValue, "Excluded event type values should be preserved, not scrubbed");
+  }
+}

dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java

@@ -2,6 +2,10 @@
 
 import static datadog.environment.JavaVirtualMachine.isJavaVersion;
 import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
+import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED;
+import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED_DEFAULT;
+import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN;
+import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN_DEFAULT;
 import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST;
 import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST_DEFAULT;
 import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
@@ -32,6 +36,7 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.time.Duration;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -137,6 +142,25 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
 
         uploader = new ProfileUploader(config, configProvider);
 
+        RecordingDataListener listener = uploader::upload;
+        if (dumper != null) {
+          RecordingDataListener upload = listener;
+          listener =
+              (type, data, sync) -> {
+                dumper.onNewData(type, data, sync);
+                upload.onNewData(type, data, sync);
+              };
+        }
+        // Scrubber wraps the combined dumper+uploader so debug dumps also contain scrubbed data
+        if (configProvider.getBoolean(PROFILING_SCRUB_ENABLED, PROFILING_SCRUB_ENABLED_DEFAULT)) {
+          List<String> excludeEventTypes =
+              configProvider.getList(ProfilingConfig.PROFILING_SCRUB_EXCLUDE_EVENTS);
+          boolean failOpen =
+              configProvider.getBoolean(
+                  PROFILING_SCRUB_FAIL_OPEN, PROFILING_SCRUB_FAIL_OPEN_DEFAULT);
+          listener = wrapWithScrubber(listener, excludeEventTypes, failOpen);
+        }
+
         final Duration startupDelay = Duration.ofSeconds(config.getProfilingStartDelay());
         final Duration uploadPeriod = Duration.ofSeconds(config.getProfilingUploadPeriod());
 
@@ -149,12 +173,7 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
                 configProvider,
                 controller,
                 context.snapshot(),
-                dumper == null
-                    ? uploader::upload
-                    : (type, data, sync) -> {
-                      dumper.onNewData(type, data, sync);
-                      uploader.upload(type, data, sync);
-                    },
+                listener,
                 startupDelay,
                 startupDelayRandomRange,
                 uploadPeriod,
@@ -181,6 +200,16 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
     return false;
   }
 
+  private static RecordingDataListener wrapWithScrubber(
+      RecordingDataListener listener, List<String> excludeEventTypes, boolean failOpen) {
+    try {
+      return ScrubRecordingDataListener.wrap(listener, excludeEventTypes, failOpen);
+    } catch (Exception e) {
+      log.warn(SEND_TELEMETRY, "Failed to initialize JFR scrubber", e);
+      return listener;
+    }
+  }
+
   private static boolean isStartForceFirstSafe() {
     return isJavaVersionAtLeast(14)
         || (isJavaVersion(13) && isJavaVersionAtLeast(13, 0, 4))