Audit log filter and marker (#45456)

This adds a log marker and a marker filter for the audit log.

Closes #47251

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -8,6 +8,11 @@ package org.elasticsearch.xpack.security.audit.logfile;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+import org.apache.logging.log4j.core.Filter.Result;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.filter.MarkerFilter;
 import org.apache.logging.log4j.message.StringMapMessage;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -16,6 +21,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -35,6 +41,7 @@ import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
+import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
@@ -151,6 +158,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             "indices",
             (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic));
 
+    private static final Marker AUDIT_MARKER = MarkerManager.getMarker("org.elasticsearch.xpack.security.audit");
+
     private final Logger logger;
     private final ThreadContext threadContext;
     final EventFilterPolicyRegistry eventFilterPolicyRegistry;
@@ -166,7 +175,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     public LoggingAuditTrail(Settings settings, ClusterService clusterService, ThreadPool threadPool) {
-        this(settings, clusterService, LogManager.getLogger(), threadPool.getThreadContext());
+        this(settings, clusterService, LogManager.getLogger(LoggingAuditTrail.class), threadPool.getThreadContext());
     }
 
     LoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) {
@@ -207,6 +216,14 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)).changeIndicesFilter(filtersList);
             this.eventFilterPolicyRegistry.set(policyName, newPolicy);
         }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList));
+        // this log filter ensures that audit events are not filtered out because of the log level
+        final LoggerContext ctx = LoggerContext.getContext(false);
+        MarkerFilter auditMarkerFilter = MarkerFilter.createFilter(AUDIT_MARKER.getName(), Result.ACCEPT, Result.NEUTRAL);
+        ctx.addFilter(auditMarkerFilter);
+        ctx.updateLoggers();
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(ignored -> {
+            LogManager.getLogger(Security.class).warn("Changing log level for [" + LoggingAuditTrail.class.getName() + "] has no effect");
+        }, List.of(Loggers.LOG_LEVEL_SETTING.getConcreteSettingForNamespace(LoggingAuditTrail.class.getName())));
     }
 
     @Override
@@ -225,7 +242,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -248,7 +265,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -270,7 +287,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -289,7 +306,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -311,7 +328,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -329,7 +346,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -350,7 +367,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -370,7 +387,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -393,7 +410,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -414,7 +431,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -440,7 +457,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withXForwardedFor(threadContext)
                         .with(authorizationInfo.asMap())
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -480,7 +497,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .with(ORIGIN_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(remoteAddress.address()));
                 }
-                logger.info(logEntryBuilder.build());
+                logger.info(AUDIT_MARKER, logEntryBuilder.build());
             }
         }
     }
@@ -505,7 +522,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -523,7 +540,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -544,7 +561,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -567,7 +584,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -586,7 +603,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -604,7 +621,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 
@@ -628,7 +645,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -653,7 +670,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
-                logger.info(logEntry);
+                logger.info(AUDIT_MARKER, logEntry);
             }
         }
     }
@@ -675,7 +692,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                     .withOpaqueId(threadContext)
                     .withXForwardedFor(threadContext)
                     .build();
-            logger.info(logEntry);
+            logger.info(AUDIT_MARKER, logEntry);
         }
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java

@@ -199,7 +199,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
             threadContext.putHeader(AuditTrail.X_FORWARDED_FOR_HEADER,
                     randomFrom("2001:db8:85a3:8d3:1319:8a2e:370:7348", "203.0.113.195", "203.0.113.195, 70.41.3.18, 150.172.238.178"));
         }
-        logger = CapturingLogger.newCapturingLogger(Level.INFO, patternLayout);
+        logger = CapturingLogger.newCapturingLogger(randomFrom(Level.OFF, Level.FATAL, Level.ERROR, Level.WARN, Level.INFO), patternLayout);
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
     }