Expose API key authentication metrics (#103178)

This PR adds metrics for recording successful and failed authentications
for API keys as well the authentication time itself. Exposed metrics are:

- `es.security.authc.api_key.success.count`
- `es.security.authc.api_key.failures.count`
- `es.security.authc.api_key.time`

Each of the metric is exposed at node level and includes
additional API key information through these attributes:

- `es.security.api_key_id` - unique API key identifier
- `es.security.api_key_type` - API key type (`rest` or `cross_cluster`)
- `es.security.api_key_authc_failure_reason` - failure message (e.g.  `api key is expired`)

Relates: ES-7468

docs/changelog/103178.yaml

@@ -0,0 +1,5 @@
+pr: 103178
+summary: Expose API key authentication metrics
+area: Authentication
+type: enhancement
+issues: []

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -89,6 +89,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
@@ -648,7 +649,8 @@ public class Security extends Plugin
                 services.xContentRegistry(),
                 services.environment(),
                 services.nodeEnvironment().nodeMetadata(),
-                services.indexNameExpressionResolver()
+                services.indexNameExpressionResolver(),
+                services.telemetryProvider()
             );
         } catch (final Exception e) {
             throw new IllegalStateException("security initialization failed", e);
@@ -666,7 +668,8 @@ public class Security extends Plugin
         NamedXContentRegistry xContentRegistry,
         Environment environment,
         NodeMetadata nodeMetadata,
-        IndexNameExpressionResolver expressionResolver
+        IndexNameExpressionResolver expressionResolver,
+        TelemetryProvider telemetryProvider
     ) throws Exception {
         logger.info("Security is {}", enabled ? "enabled" : "disabled");
         if (enabled == false) {
@@ -944,7 +947,8 @@ public class Security extends Plugin
                 tokenService,
                 apiKeyService,
                 serviceAccountService,
-                operatorPrivilegesService.get()
+                operatorPrivilegesService.get(),
+                telemetryProvider.getMeterRegistry()
             )
         );
         components.add(authcService.get());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyAuthenticator.java

@@ -10,23 +10,46 @@ package org.elasticsearch.xpack.security.authc;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.xpack.core.security.action.apikey.ApiKey;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials;
+import org.elasticsearch.xpack.security.metric.InstrumentedSecurityActionListener;
+import org.elasticsearch.xpack.security.metric.SecurityMetricType;
+import org.elasticsearch.xpack.security.metric.SecurityMetrics;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.LongSupplier;
 
 import static org.elasticsearch.core.Strings.format;
 
 class ApiKeyAuthenticator implements Authenticator {
 
+    public static final String ATTRIBUTE_API_KEY_ID = "es.security.api_key_id";
+    public static final String ATTRIBUTE_API_KEY_TYPE = "es.security.api_key_type";
+    public static final String ATTRIBUTE_API_KEY_AUTHC_FAILURE_REASON = "es.security.api_key_authc_failure_reason";
+
     private static final Logger logger = LogManager.getLogger(ApiKeyAuthenticator.class);
 
+    private final SecurityMetrics<ApiKeyCredentials> authenticationMetrics;
     private final ApiKeyService apiKeyService;
     private final String nodeName;
 
-    ApiKeyAuthenticator(ApiKeyService apiKeyService, String nodeName) {
+    ApiKeyAuthenticator(ApiKeyService apiKeyService, String nodeName, MeterRegistry meterRegistry) {
+        this(apiKeyService, nodeName, meterRegistry, System::nanoTime);
+    }
+
+    ApiKeyAuthenticator(ApiKeyService apiKeyService, String nodeName, MeterRegistry meterRegistry, LongSupplier nanoTimeSupplier) {
+        this.authenticationMetrics = new SecurityMetrics<>(
+            SecurityMetricType.AUTHC_API_KEY,
+            meterRegistry,
+            this::buildMetricAttributes,
+            nanoTimeSupplier
+        );
         this.apiKeyService = apiKeyService;
         this.nodeName = nodeName;
     }
@@ -51,30 +74,44 @@ class ApiKeyAuthenticator implements Authenticator {
             return;
         }
         ApiKeyCredentials apiKeyCredentials = (ApiKeyCredentials) authenticationToken;
-        apiKeyService.tryAuthenticate(context.getThreadContext(), apiKeyCredentials, ActionListener.wrap(authResult -> {
-            if (authResult.isAuthenticated()) {
-                final Authentication authentication = Authentication.newApiKeyAuthentication(authResult, nodeName);
-                listener.onResponse(AuthenticationResult.success(authentication));
-            } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
-                Exception e = (authResult.getException() != null)
-                    ? authResult.getException()
-                    : Exceptions.authenticationError(authResult.getMessage());
-                logger.debug(() -> "API key service terminated authentication for request [" + context.getRequest() + "]", e);
-                context.getRequest().exceptionProcessingRequest(e, authenticationToken);
-                listener.onFailure(e);
-            } else {
-                if (authResult.getMessage() != null) {
-                    if (authResult.getException() != null) {
-                        logger.warn(
-                            () -> format("Authentication using apikey failed - %s", authResult.getMessage()),
-                            authResult.getException()
-                        );
-                    } else {
-                        logger.warn("Authentication using apikey failed - {}", authResult.getMessage());
+        apiKeyService.tryAuthenticate(
+            context.getThreadContext(),
+            apiKeyCredentials,
+            InstrumentedSecurityActionListener.wrapForAuthc(authenticationMetrics, apiKeyCredentials, ActionListener.wrap(authResult -> {
+                if (authResult.isAuthenticated()) {
+                    final Authentication authentication = Authentication.newApiKeyAuthentication(authResult, nodeName);
+                    listener.onResponse(AuthenticationResult.success(authentication));
+                } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
+                    Exception e = (authResult.getException() != null)
+                        ? authResult.getException()
+                        : Exceptions.authenticationError(authResult.getMessage());
+                    logger.debug(() -> "API key service terminated authentication for request [" + context.getRequest() + "]", e);
+                    context.getRequest().exceptionProcessingRequest(e, authenticationToken);
+                    listener.onFailure(e);
+                } else {
+                    if (authResult.getMessage() != null) {
+                        if (authResult.getException() != null) {
+                            logger.warn(
+                                () -> format("Authentication using apikey failed - %s", authResult.getMessage()),
+                                authResult.getException()
+                            );
+                        } else {
+                            logger.warn("Authentication using apikey failed - {}", authResult.getMessage());
+                        }
                     }
+                    listener.onResponse(AuthenticationResult.unsuccessful(authResult.getMessage(), authResult.getException()));
                 }
-                listener.onResponse(AuthenticationResult.unsuccessful(authResult.getMessage(), authResult.getException()));
-            }
-        }, e -> listener.onFailure(context.getRequest().exceptionProcessingRequest(e, null))));
+            }, e -> listener.onFailure(context.getRequest().exceptionProcessingRequest(e, null))))
+        );
+    }
+
+    private Map<String, Object> buildMetricAttributes(ApiKeyCredentials credentials, String failureReason) {
+        final Map<String, Object> attributes = new HashMap<>(failureReason != null ? 3 : 2);
+        attributes.put(ATTRIBUTE_API_KEY_ID, credentials.getId());
+        attributes.put(ATTRIBUTE_API_KEY_TYPE, credentials.getExpectedType().value());
+        if (failureReason != null) {
+            attributes.put(ATTRIBUTE_API_KEY_AUTHC_FAILURE_REASON, failureReason);
+        }
+        return attributes;
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -20,6 +20,7 @@ import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpPreRequest;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -88,7 +89,8 @@ public class AuthenticationService {
         TokenService tokenService,
         ApiKeyService apiKeyService,
         ServiceAccountService serviceAccountService,
-        OperatorPrivilegesService operatorPrivilegesService
+        OperatorPrivilegesService operatorPrivilegesService,
+        MeterRegistry meterRegistry
     ) {
         this.realms = realms;
         this.auditTrailService = auditTrailService;
@@ -111,7 +113,7 @@ public class AuthenticationService {
             new AuthenticationContextSerializer(),
             new ServiceAccountAuthenticator(serviceAccountService, nodeName),
             new OAuth2TokenAuthenticator(tokenService),
-            new ApiKeyAuthenticator(apiKeyService, nodeName),
+            new ApiKeyAuthenticator(apiKeyService, nodeName, meterRegistry),
             new RealmsAuthenticator(numInvalidation, lastSuccessfulAuthCache)
         );
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/InstrumentedSecurityActionListener.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+
+public class InstrumentedSecurityActionListener {
+
+    /**
+     * Wraps the provided {@code listener} and returns a new wrapped listener which handles authentication metrics collection.
+     *
+     * @param metrics The metrics to collect.
+     * @param context The context object is used to collect and attach additional metric attributes.
+     * @param listener The authentication result handling listener.
+     * @return a new "wrapped" listener which overrides onResponse and onFailure methods in order to collect authentication metrics.
+     * @param <R> The type of authentication result value.
+     * @param <C> The type of context object which is used to attach additional attributes to collected authentication metrics.
+     */
+    public static <R, C> ActionListener<AuthenticationResult<R>> wrapForAuthc(
+        final SecurityMetrics<C> metrics,
+        final C context,
+        final ActionListener<AuthenticationResult<R>> listener
+    ) {
+        assert metrics.type().group() == SecurityMetricGroup.AUTHC;
+        final long startTimeNano = metrics.relativeTimeInNanos();
+        return ActionListener.runBefore(ActionListener.wrap(result -> {
+            if (result.isAuthenticated()) {
+                metrics.recordSuccess(context);
+            } else {
+                metrics.recordFailure(context, result.getMessage());
+            }
+            listener.onResponse(result);
+        }, e -> {
+            metrics.recordFailure(context, e.getMessage());
+            listener.onFailure(e);
+        }), () -> metrics.recordTime(context, startTimeNano));
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityMetricAttributesBuilder.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+import java.util.Map;
+
+@FunctionalInterface
+public interface SecurityMetricAttributesBuilder<C> {
+
+    Map<String, Object> build(C context, String failureReason);
+
+    default Map<String, Object> build(C context) {
+        return build(context, null);
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityMetricGroup.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+/**
+ * Enumerates all metric groups we want to collect.
+ */
+public enum SecurityMetricGroup {
+
+    AUTHC,
+
+    AUTHZ,
+
+    ;
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityMetricInfo.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.LongHistogram;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.Objects;
+
+/**
+ * Holds all metric information needed to register a metric in {@link MeterRegistry}.
+ *
+ * @param name          The unique metric name.
+ * @param description   The brief metric description.
+ * @param unit          The metric unit (e.g. count).
+ */
+public record SecurityMetricInfo(String name, String description, String unit) {
+
+    public SecurityMetricInfo(String name, String description, String unit) {
+        this.name = Objects.requireNonNull(name);
+        this.description = Objects.requireNonNull(description);
+        this.unit = Objects.requireNonNull(unit);
+    }
+
+    public LongCounter registerAsLongCounter(MeterRegistry meterRegistry) {
+        return meterRegistry.registerLongCounter(this.name(), this.description(), this.unit());
+    }
+
+    public LongHistogram registerAsLongHistogram(MeterRegistry meterRegistry) {
+        return meterRegistry.registerLongHistogram(this.name(), this.description(), this.unit());
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityMetricType.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+/**
+ * Defines all security metric types that can be collected.
+ */
+public enum SecurityMetricType {
+
+    AUTHC_API_KEY(
+        SecurityMetricGroup.AUTHC,
+        new SecurityMetricInfo("es.security.authc.api_key.success.count", "Number of successful API key authentications.", "count"),
+        new SecurityMetricInfo("es.security.authc.api_key.failures.count", "Number of failed API key authentications.", "count"),
+        new SecurityMetricInfo("es.security.authc.api_key.time", "Time it took (in nanoseconds) to execute API key authentication.", "ns")
+    ),
+
+    ;
+
+    private final SecurityMetricGroup group;
+    private final SecurityMetricInfo successMetricInfo;
+    private final SecurityMetricInfo failuresMetricInfo;
+    private final SecurityMetricInfo timeMetricInfo;
+
+    SecurityMetricType(
+        SecurityMetricGroup group,
+        SecurityMetricInfo successMetricInfo,
+        SecurityMetricInfo failuresMetricInfo,
+        SecurityMetricInfo timeMetricInfo
+    ) {
+        this.group = group;
+        this.successMetricInfo = successMetricInfo;
+        this.failuresMetricInfo = failuresMetricInfo;
+        this.timeMetricInfo = timeMetricInfo;
+    }
+
+    public SecurityMetricGroup group() {
+        return this.group;
+    }
+
+    public SecurityMetricInfo successMetricInfo() {
+        return successMetricInfo;
+    }
+
+    public SecurityMetricInfo failuresMetricInfo() {
+        return failuresMetricInfo;
+    }
+
+    public SecurityMetricInfo timeMetricInfo() {
+        return timeMetricInfo;
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityMetrics.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.LongHistogram;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.Objects;
+import java.util.function.LongSupplier;
+
+/**
+ * This class provides a common way for registering and collecting different types of security metrics.
+ * It allows for recoding the number of successful and failed executions as well as to record the execution time.
+ *
+ * @param <C> The type of context object which is used to attach additional attributes to collected metrics.
+ */
+public final class SecurityMetrics<C> {
+
+    private final LongCounter successCounter;
+    private final LongCounter failuresCounter;
+    private final LongHistogram timeHistogram;
+
+    private final SecurityMetricAttributesBuilder<C> attributesBuilder;
+    private final LongSupplier nanoTimeSupplier;
+    private final SecurityMetricType metricType;
+
+    public SecurityMetrics(
+        final SecurityMetricType metricType,
+        final MeterRegistry meterRegistry,
+        final SecurityMetricAttributesBuilder<C> attributesBuilder,
+        final LongSupplier nanoTimeSupplier
+    ) {
+        this.metricType = Objects.requireNonNull(metricType);
+        this.successCounter = metricType.successMetricInfo().registerAsLongCounter(meterRegistry);
+        this.failuresCounter = metricType.failuresMetricInfo().registerAsLongCounter(meterRegistry);
+        this.timeHistogram = metricType.timeMetricInfo().registerAsLongHistogram(meterRegistry);
+        this.attributesBuilder = Objects.requireNonNull(attributesBuilder);
+        this.nanoTimeSupplier = Objects.requireNonNull(nanoTimeSupplier);
+    }
+
+    public SecurityMetricType type() {
+        return this.metricType;
+    }
+
+    /**
+     * Returns a value of nanoseconds that may be used for relative time calculations.
+     * This method should only be used for calculating time deltas.
+     */
+    public long relativeTimeInNanos() {
+        return nanoTimeSupplier.getAsLong();
+    }
+
+    /**
+     * Records a single success execution.
+     *
+     * @param context The context object which is used to attach additional attributes to success metric.
+     */
+    public void recordSuccess(final C context) {
+        this.successCounter.incrementBy(1L, attributesBuilder.build(context));
+    }
+
+    /**
+     * Records a single failed execution.
+     *
+     * @param context       The context object which is used to attach additional attributes to failed metric.
+     * @param failureReason The optional failure reason which is stored as an attributed with recorded failure metric.
+     */
+    public void recordFailure(final C context, final String failureReason) {
+        this.failuresCounter.incrementBy(1L, attributesBuilder.build(context, failureReason));
+    }
+
+    /**
+     * Records a time in nanoseconds. This method should be called after the execution with provided start time.
+     * The {@link #relativeTimeInNanos()} should be used to record the start time.
+     *
+     * @param context       The context object which is used to attach additional attributes to collected metric.
+     * @param startTimeNano The start time (in nanoseconds) before the execution.
+     */
+    public void recordTime(final C context, final long startTimeNano) {
+        final long timeInNanos = relativeTimeInNanos() - startTimeNano;
+        this.timeHistogram.record(timeInNanos, this.attributesBuilder.build(context));
+    }
+
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -58,6 +58,7 @@ import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -224,7 +225,8 @@ public class SecurityTests extends ESTestCase {
             xContentRegistry(),
             env,
             nodeMetadata,
-            TestIndexNameExpressionResolver.newInstance(threadContext)
+            TestIndexNameExpressionResolver.newInstance(threadContext),
+            TelemetryProvider.NOOP
         );
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyAuthenticatorTests.java

@@ -14,16 +14,26 @@ import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.action.apikey.ApiKey;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials;
 import org.elasticsearch.xpack.security.authc.AuthenticationService.AuditableRequest;
+import org.elasticsearch.xpack.security.metric.SecurityMetricType;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.LongSupplier;
 
 import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.ArgumentMatchers.any;
@@ -37,15 +47,15 @@ public class ApiKeyAuthenticatorTests extends ESTestCase {
 
     public void testAuditingOnAuthenticationTermination() {
         final ApiKeyService apiKeyService = mock(ApiKeyService.class);
-        final ApiKeyAuthenticator apiKeyAuthenticator = new ApiKeyAuthenticator(apiKeyService, randomAlphaOfLengthBetween(3, 8));
+        final ApiKeyAuthenticator apiKeyAuthenticator = new ApiKeyAuthenticator(
+            apiKeyService,
+            randomAlphaOfLengthBetween(3, 8),
+            MeterRegistry.NOOP
+        );
 
         final Authenticator.Context context = mock(Authenticator.Context.class);
 
-        final ApiKeyCredentials apiKeyCredentials = new ApiKeyCredentials(
-            randomAlphaOfLength(20),
-            new SecureString(randomAlphaOfLength(20).toCharArray()),
-            randomFrom(ApiKey.Type.values())
-        );
+        final ApiKeyCredentials apiKeyCredentials = randomApiKeyCredentials();
         when(context.getMostRecentAuthenticationToken()).thenReturn(apiKeyCredentials);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         when(context.getThreadContext()).thenReturn(threadContext);
@@ -72,4 +82,253 @@ public class ApiKeyAuthenticatorTests extends ESTestCase {
         }
     }
 
+    public void testRecordingSuccessfulAuthenticationMetrics() {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final long initialNanoTime = randomLongBetween(0, 100);
+        final TestNanoTimeSupplier nanoTimeSupplier = new TestNanoTimeSupplier(initialNanoTime);
+        final ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        final ApiKeyAuthenticator apiKeyAuthenticator = createApiKeyAuthenticator(apiKeyService, telemetryPlugin, nanoTimeSupplier);
+
+        final ApiKeyCredentials apiKeyCredentials = randomApiKeyCredentials();
+        final Authenticator.Context context = mockApiKeyAuthenticationContext(apiKeyCredentials);
+
+        final long executionTimeInNanos = randomLongBetween(0, 500);
+        doAnswer(invocation -> {
+            final ActionListener<AuthenticationResult<User>> listener = invocation.getArgument(2);
+            nanoTimeSupplier.advanceTime(executionTimeInNanos);
+            listener.onResponse(
+                AuthenticationResult.success(
+                    new User(randomAlphaOfLengthBetween(3, 8)),
+                    Map.ofEntries(
+                        Map.entry(AuthenticationField.API_KEY_ID_KEY, apiKeyCredentials.getId()),
+                        Map.entry(AuthenticationField.API_KEY_TYPE_KEY, apiKeyCredentials.getExpectedType().value())
+                    )
+                )
+            );
+            return null;
+        }).when(apiKeyService).tryAuthenticate(any(), same(apiKeyCredentials), anyActionListener());
+
+        final PlainActionFuture<AuthenticationResult<Authentication>> future = new PlainActionFuture<>();
+        apiKeyAuthenticator.authenticate(context, future);
+        final AuthenticationResult<Authentication> authResult = future.actionGet();
+        assertThat(authResult.isAuthenticated(), equalTo(true));
+
+        List<Measurement> successMetrics = telemetryPlugin.getLongCounterMeasurement(
+            SecurityMetricType.AUTHC_API_KEY.successMetricInfo().name()
+        );
+        assertThat(successMetrics.size(), equalTo(1));
+
+        // verify that we always record a single authentication
+        assertThat(successMetrics.get(0).getLong(), equalTo(1L));
+        // and that all attributes are present
+        assertThat(
+            successMetrics.get(0).attributes(),
+            equalTo(
+                Map.ofEntries(
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_ID, apiKeyCredentials.getId()),
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_TYPE, apiKeyCredentials.getExpectedType().value())
+                )
+            )
+        );
+
+        // verify that there were no failures recorded
+        assertZeroFailedAuthMetrics(telemetryPlugin);
+
+        // verify we recorded authentication time
+        assertAuthenticationTimeMetric(telemetryPlugin, apiKeyCredentials, executionTimeInNanos);
+    }
+
+    public void testRecordingFailedAuthenticationMetrics() {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final long initialNanoTime = randomLongBetween(1, 100);
+        final TestNanoTimeSupplier nanoTimeSupplier = new TestNanoTimeSupplier(initialNanoTime);
+        final ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        final ApiKeyAuthenticator apiKeyAuthenticator = createApiKeyAuthenticator(apiKeyService, telemetryPlugin, nanoTimeSupplier);
+
+        final ApiKeyCredentials apiKeyCredentials = randomApiKeyCredentials();
+        final Authenticator.Context context = mockApiKeyAuthenticationContext(apiKeyCredentials);
+
+        final Exception exception = randomFrom(new ElasticsearchException("API key auth exception"), null);
+        final boolean failWithTermination = randomBoolean();
+        final AuthenticationResult<User> failedAuth;
+        if (failWithTermination) {
+            failedAuth = AuthenticationResult.terminate("terminated API key auth", exception);
+        } else {
+            failedAuth = AuthenticationResult.unsuccessful("unsuccessful API key auth", exception);
+        }
+
+        final long executionTimeInNanos = randomLongBetween(0, 500);
+        doAnswer(invocation -> {
+            nanoTimeSupplier.advanceTime(executionTimeInNanos);
+            final ActionListener<AuthenticationResult<User>> listener = invocation.getArgument(2);
+            listener.onResponse(failedAuth);
+            return Void.TYPE;
+        }).when(apiKeyService).tryAuthenticate(any(), same(apiKeyCredentials), anyActionListener());
+        final PlainActionFuture<AuthenticationResult<Authentication>> future = new PlainActionFuture<>();
+        apiKeyAuthenticator.authenticate(context, future);
+
+        if (failWithTermination) {
+            final Exception e = expectThrows(Exception.class, future::actionGet);
+            if (exception == null) {
+                assertThat(e, instanceOf(ElasticsearchSecurityException.class));
+                assertThat(e.getMessage(), containsString("terminated API key auth"));
+            } else {
+                assertThat(e, sameInstance(exception));
+            }
+            assertSingleFailedAuthMetric(telemetryPlugin, apiKeyCredentials, "terminated API key auth");
+        } else {
+            var authResult = future.actionGet();
+            assertThat(authResult.isAuthenticated(), equalTo(false));
+            assertSingleFailedAuthMetric(telemetryPlugin, apiKeyCredentials, "unsuccessful API key auth");
+        }
+
+        // verify that there were no successes recorded
+        assertZeroSuccessAuthMetrics(telemetryPlugin);
+
+        // verify we recorded authentication time
+        assertAuthenticationTimeMetric(telemetryPlugin, apiKeyCredentials, executionTimeInNanos);
+    }
+
+    public void testRecordingFailedAuthenticationMetricsOnExceptions() {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final long initialNanoTime = randomLongBetween(0, 100);
+        final TestNanoTimeSupplier nanoTimeSupplier = new TestNanoTimeSupplier(initialNanoTime);
+        final ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        final ApiKeyAuthenticator apiKeyAuthenticator = createApiKeyAuthenticator(apiKeyService, telemetryPlugin, nanoTimeSupplier);
+
+        final ApiKeyCredentials apiKeyCredentials = randomApiKeyCredentials();
+        final Authenticator.Context context = mockApiKeyAuthenticationContext(apiKeyCredentials);
+
+        final ElasticsearchSecurityException exception = new ElasticsearchSecurityException("API key auth exception");
+        when(context.getRequest().exceptionProcessingRequest(same(exception), any())).thenReturn(exception);
+
+        final long executionTimeInNanos = randomLongBetween(0, 500);
+        doAnswer(invocation -> {
+            nanoTimeSupplier.advanceTime(executionTimeInNanos);
+            final ActionListener<AuthenticationResult<User>> listener = invocation.getArgument(2);
+            listener.onFailure(exception);
+            return Void.TYPE;
+        }).when(apiKeyService).tryAuthenticate(any(), same(apiKeyCredentials), anyActionListener());
+
+        final PlainActionFuture<AuthenticationResult<Authentication>> future = new PlainActionFuture<>();
+        apiKeyAuthenticator.authenticate(context, future);
+
+        var e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+        assertThat(e, sameInstance(exception));
+
+        // expecting single recorded auth failure with message same as the thrown exception
+        assertSingleFailedAuthMetric(telemetryPlugin, apiKeyCredentials, "API key auth exception");
+
+        // verify that there were no successes recorded
+        assertZeroSuccessAuthMetrics(telemetryPlugin);
+
+        // verify we recorded authentication time
+        assertAuthenticationTimeMetric(telemetryPlugin, apiKeyCredentials, executionTimeInNanos);
+    }
+
+    private void assertSingleFailedAuthMetric(
+        TestTelemetryPlugin telemetryPlugin,
+        ApiKeyCredentials apiKeyCredentials,
+        String failureMessage
+    ) {
+        List<Measurement> failuresMetrics = telemetryPlugin.getLongCounterMeasurement(
+            SecurityMetricType.AUTHC_API_KEY.failuresMetricInfo().name()
+        );
+        assertThat(failuresMetrics.size(), equalTo(1));
+        assertThat(
+            failuresMetrics.get(0).attributes(),
+            equalTo(
+                Map.ofEntries(
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_ID, apiKeyCredentials.getId()),
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_TYPE, apiKeyCredentials.getExpectedType().value()),
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_AUTHC_FAILURE_REASON, failureMessage)
+                )
+            )
+        );
+    }
+
+    private void assertAuthenticationTimeMetric(
+        TestTelemetryPlugin telemetryPlugin,
+        ApiKeyCredentials credentials,
+        long expectedAuthenticationTime
+    ) {
+        List<Measurement> authTimeMetrics = telemetryPlugin.getLongHistogramMeasurement(
+            SecurityMetricType.AUTHC_API_KEY.timeMetricInfo().name()
+        );
+        assertThat(authTimeMetrics.size(), equalTo(1));
+        assertThat(authTimeMetrics.get(0).getLong(), equalTo(expectedAuthenticationTime));
+        assertThat(
+            authTimeMetrics.get(0).attributes(),
+            equalTo(
+                Map.ofEntries(
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_ID, credentials.getId()),
+                    Map.entry(ApiKeyAuthenticator.ATTRIBUTE_API_KEY_TYPE, credentials.getExpectedType().value())
+                )
+            )
+        );
+    }
+
+    private void assertZeroSuccessAuthMetrics(TestTelemetryPlugin telemetryPlugin) {
+        List<Measurement> successMetrics = telemetryPlugin.getLongCounterMeasurement(
+            SecurityMetricType.AUTHC_API_KEY.successMetricInfo().name()
+        );
+        assertThat(successMetrics.size(), equalTo(0));
+    }
+
+    private void assertZeroFailedAuthMetrics(TestTelemetryPlugin telemetryPlugin) {
+        List<Measurement> failuresMetrics = telemetryPlugin.getLongCounterMeasurement(
+            SecurityMetricType.AUTHC_API_KEY.failuresMetricInfo().name()
+        );
+        assertThat(failuresMetrics.size(), equalTo(0));
+    }
+
+    private static ApiKeyCredentials randomApiKeyCredentials() {
+        return new ApiKeyCredentials(
+            randomAlphaOfLength(12),
+            new SecureString(randomAlphaOfLength(20).toCharArray()),
+            randomFrom(ApiKey.Type.values())
+        );
+    }
+
+    private static ApiKeyAuthenticator createApiKeyAuthenticator(
+        ApiKeyService apiKeyService,
+        TestTelemetryPlugin telemetryPlugin,
+        LongSupplier nanoTimeSupplier
+    ) {
+        return new ApiKeyAuthenticator(
+            apiKeyService,
+            randomAlphaOfLengthBetween(3, 8),
+            telemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry(),
+            nanoTimeSupplier
+        );
+    }
+
+    private static Authenticator.Context mockApiKeyAuthenticationContext(ApiKeyCredentials apiKeyCredentials) {
+        final Authenticator.Context context = mock(Authenticator.Context.class);
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        when(context.getMostRecentAuthenticationToken()).thenReturn(apiKeyCredentials);
+        when(context.getThreadContext()).thenReturn(threadContext);
+        final AuditableRequest auditableRequest = mock(AuditableRequest.class);
+        when(context.getRequest()).thenReturn(auditableRequest);
+        return context;
+    }
+
+    private static class TestNanoTimeSupplier implements LongSupplier {
+
+        private long currentTime;
+
+        TestNanoTimeSupplier(long initialTime) {
+            this.currentTime = initialTime;
+        }
+
+        public void advanceTime(long timeToAdd) {
+            this.currentTime += timeToAdd;
+        }
+
+        @Override
+        public long getAsLong() {
+            return currentTime;
+        }
+    }
+
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

@@ -53,6 +53,7 @@ import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
@@ -364,7 +365,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
     }
 
@@ -660,7 +662,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
         User user = new User("_username", "r1");
         when(firstRealm.supports(token)).thenReturn(true);
@@ -1040,7 +1043,8 @@ public class AuthenticationServiceTests extends ESTestCase {
                 tokenService,
                 apiKeyService,
                 serviceAccountService,
-                operatorPrivilegesService
+                operatorPrivilegesService,
+                MeterRegistry.NOOP
             );
             boolean requestIdAlreadyPresent = randomBoolean();
             SetOnce<String> reqId = new SetOnce<>();
@@ -1090,7 +1094,8 @@ public class AuthenticationServiceTests extends ESTestCase {
                     tokenService,
                     apiKeyService,
                     serviceAccountService,
-                    operatorPrivilegesService
+                    operatorPrivilegesService,
+                    MeterRegistry.NOOP
                 );
                 threadContext2.putHeader(AuthenticationField.AUTHENTICATION_KEY, authHeaderRef.get());
 
@@ -1113,7 +1118,8 @@ public class AuthenticationServiceTests extends ESTestCase {
                 tokenService,
                 apiKeyService,
                 serviceAccountService,
-                operatorPrivilegesService
+                operatorPrivilegesService,
+                MeterRegistry.NOOP
             );
             service.authenticate("_action", new InternalRequest(), InternalUsers.SYSTEM_USER, ActionListener.wrap(result -> {
                 if (requestIdAlreadyPresent) {
@@ -1175,7 +1181,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
 
         try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
@@ -1219,7 +1226,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
         doAnswer(invocationOnMock -> {
             final GetRequest request = (GetRequest) invocationOnMock.getArguments()[0];
@@ -1283,7 +1291,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
         RestRequest request = new FakeRestRequest();
 
@@ -1319,7 +1328,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
         RestRequest request = new FakeRestRequest();
 
@@ -1350,7 +1360,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
         InternalRequest message = new InternalRequest();
         boolean requestIdAlreadyPresent = randomBoolean();
@@ -1385,7 +1396,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            operatorPrivilegesService
+            operatorPrivilegesService,
+            MeterRegistry.NOOP
         );
 
         InternalRequest message = new InternalRequest();

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.license.License;
 import org.elasticsearch.license.TestUtils;
 import org.elasticsearch.license.internal.XPackLicenseStatus;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestMatchers;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -156,7 +157,8 @@ public class SecondaryAuthenticatorTests extends ESTestCase {
             tokenService,
             apiKeyService,
             serviceAccountService,
-            OperatorPrivileges.NOOP_OPERATOR_PRIVILEGES_SERVICE
+            OperatorPrivileges.NOOP_OPERATOR_PRIVILEGES_SERVICE,
+            MeterRegistry.NOOP
         );
         authenticator = new SecondaryAuthenticator(securityContext, authenticationService, auditTrail);
     }