Make redaction configurable for APM tracing (#92358)

Closes #92338.

When tracing REST requests with APM, we capture HTTP headers as labels
on the trace, but redact sensitive values. However, we can't know ahead
of time what are all possible sensitive values.

Push this redaction into the tracer, and make the redaction terms
configurable. Switch the defaults to the APM Java agent's defaults.

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java

@@ -101,6 +101,7 @@ public class APM extends Plugin implements NetworkPlugin, TracerPlugin {
             APMAgentSettings.APM_ENABLED_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING,
+            APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES,
             APMAgentSettings.APM_AGENT_SETTINGS,
             APMAgentSettings.APM_SECRET_TOKEN_SETTING,
             APMAgentSettings.APM_API_KEY_SETTING

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java

@@ -57,6 +57,7 @@ class APMAgentSettings {
         });
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_INCLUDE_SETTING, apmTracer::setIncludeNames);
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_EXCLUDE_SETTING, apmTracer::setExcludeNames);
+        clusterSettings.addSettingsUpdateConsumer(APM_TRACING_SANITIZE_FIELD_NAMES, apmTracer::setLabelFilters);
         clusterSettings.addAffixMapUpdateConsumer(APM_AGENT_SETTINGS, map -> map.forEach(this::setAgentSetting), (x, y) -> {});
     }
 
@@ -143,6 +144,27 @@ class APMAgentSettings {
         NodeScope
     );
 
+    static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
+        APM_SETTING_PREFIX + "sanitize_field_names",
+        List.of(
+            "password",
+            "passwd",
+            "pwd",
+            "secret",
+            "*key",
+            "*token*",
+            "*session*",
+            "*credit*",
+            "*card*",
+            "*auth*",
+            "*principal*",
+            "set-cookie"
+        ),
+        Function.identity(),
+        OperatorDynamic,
+        NodeScope
+    );
+
     static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
         APM_SETTING_PREFIX + "enabled",
         false,

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java

@@ -44,6 +44,7 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING;
+import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES;
 
 /**
  * This is an implementation of the {@link org.elasticsearch.tracing.Tracer} interface, which uses
@@ -65,8 +66,10 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
 
     private List<String> includeNames;
     private List<String> excludeNames;
+    private List<String> labelFilters;
     /** Built using {@link #includeNames} and {@link #excludeNames}, and filters out spans based on their name. */
     private volatile CharacterRunAutomaton filterAutomaton;
+    private volatile CharacterRunAutomaton labelFilterAutomaton;
     private String clusterName;
     private String nodeName;
 
@@ -86,7 +89,10 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
     public APMTracer(Settings settings) {
         this.includeNames = APM_TRACING_NAMES_INCLUDE_SETTING.get(settings);
         this.excludeNames = APM_TRACING_NAMES_EXCLUDE_SETTING.get(settings);
+        this.labelFilters = APM_TRACING_SANITIZE_FIELD_NAMES.get(settings);
+
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
         this.enabled = APM_ENABLED_SETTING.get(settings);
     }
 
@@ -109,6 +115,16 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
     }
 
+    void setLabelFilters(List<String> labelFilters) {
+        this.labelFilters = labelFilters;
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
+    }
+
+    // package-private for testing
+    CharacterRunAutomaton getLabelFilterAutomaton() {
+        return labelFilterAutomaton;
+    }
+
     @Override
     protected void doStart() {
         if (enabled) {
@@ -271,6 +287,12 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
             for (Map.Entry<String, Object> entry : spanAttributes.entrySet()) {
                 final String key = entry.getKey();
                 final Object value = entry.getValue();
+
+                if (this.labelFilterAutomaton.run(key)) {
+                    spanBuilder.setAttribute(key, "[REDACTED]");
+                    continue;
+                }
+
                 if (value instanceof String) {
                     spanBuilder.setAttribute(key, (String) value);
                 } else if (value instanceof Long) {
@@ -394,9 +416,9 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
         return spans;
     }
 
-    private static CharacterRunAutomaton buildAutomaton(List<String> includeNames, List<String> excludeNames) {
-        Automaton includeAutomaton = patternsToAutomaton(includeNames);
-        Automaton excludeAutomaton = patternsToAutomaton(excludeNames);
+    private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns, List<String> excludePatterns) {
+        Automaton includeAutomaton = patternsToAutomaton(includePatterns);
+        Automaton excludeAutomaton = patternsToAutomaton(excludePatterns);
 
         if (includeAutomaton == null) {
             includeAutomaton = Automata.makeAnyString();

modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java

@@ -8,12 +8,14 @@
 
 package org.elasticsearch.tracing.apm;
 
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
@@ -166,6 +168,43 @@ public class APMTracerTests extends ESTestCase {
         assertThat(apmTracer.getSpans(), hasKey("id3"));
     }
 
+    /**
+     * Check that sensitive attributes are not added verbatim to a span, but instead the value is redacted.
+     */
+    public void test_whenAddingAttributes_thenSensitiveValuesAreRedacted() {
+        Settings settings = Settings.builder().put(APM_ENABLED_SETTING.getKey(), false).build();
+        APMTracer apmTracer = buildTracer(settings);
+        CharacterRunAutomaton labelFilterAutomaton = apmTracer.getLabelFilterAutomaton();
+
+        Stream.of(
+            "auth",
+            "auth-header",
+            "authValue",
+            "card",
+            "card-details",
+            "card-number",
+            "credit",
+            "credit-card",
+            "key",
+            "my-credit-number",
+            "my_session_id",
+            "passwd",
+            "password",
+            "principal",
+            "principal-value",
+            "pwd",
+            "secret",
+            "secure-key",
+            "sensitive-token*",
+            "session",
+            "session_id",
+            "set-cookie",
+            "some-auth",
+            "some-principal",
+            "token-for login"
+        ).forEach(key -> assertTrue("Expected label filter automaton to redact [" + key + "]", labelFilterAutomaton.run(key)));
+    }
+
     private APMTracer buildTracer(Settings settings) {
         APMTracer tracer = new APMTracer(settings);
         tracer.doStart();

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -461,14 +461,10 @@ public class RestController implements HttpServerTransport.Dispatcher {
             name = restPath;
         }
 
-        Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
+        final Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
         req.getHeaders().forEach((key, values) -> {
             final String lowerKey = key.toLowerCase(Locale.ROOT).replace('-', '_');
-            final String value = switch (lowerKey) {
-                case "authorization", "cookie", "secret", "session", "set_cookie", "token", "x_elastic_app_auth" -> "[REDACTED]";
-                default -> String.join("; ", values);
-            };
-            attributes.put("http.request.headers." + lowerKey, value);
+            attributes.put("http.request.headers." + lowerKey, values.size() == 1 ? values.get(0) : String.join("; ", values));
         });
         attributes.put("http.method", method);
         attributes.put("http.url", req.uri());