Add analytics plugin usage stats to _xpack/usage (#54911)

Adds analytics plugin usage stats to _xpack/usage. 

Closes #54847

docs/reference/rest-api/usage.asciidoc

@@ -16,7 +16,7 @@ Provides usage information about the installed {xpack} features.
 === {api-description-title}
 
 This API provides information about which features are currently enabled and
-available under the current license and some usage statistics. 
+available under the current license and some usage statistics.
 
 [discrete]
 [[usage-api-query-parms]]
@@ -263,7 +263,14 @@ GET /_xpack/usage
   },
   "analytics" : {
     "available" : true,
-    "enabled" : true
+    "enabled" : true,
+    "stats": {
+      "boxplot_usage" : 0,
+      "top_metrics_usage" : 0,
+      "cumulative_cardinality_usage" : 0,
+      "t_test_usage" : 0,
+      "string_stats_usage" : 0
+    }
   }
 }
 ------------------------------------------------------------

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java

@@ -75,7 +75,7 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
                 CumulativeCardinalityPipelineAggregationBuilder.NAME,
                 CumulativeCardinalityPipelineAggregationBuilder::new,
                 CumulativeCardinalityPipelineAggregator::new,
-                usage.track(AnalyticsUsage.Item.CUMULATIVE_CARDINALITY,
+                usage.track(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY,
                         checkLicense(CumulativeCardinalityPipelineAggregationBuilder.PARSER)))
         );
     }
@@ -86,24 +86,24 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
             new AggregationSpec(
                 StringStatsAggregationBuilder.NAME,
                 StringStatsAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
                 .addResultReader(InternalStringStats::new)
                 .setAggregatorRegistrar(StringStatsAggregationBuilder::registerAggregators),
             new AggregationSpec(
                 BoxplotAggregationBuilder.NAME,
                 BoxplotAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
                 .addResultReader(InternalBoxplot::new)
                 .setAggregatorRegistrar(BoxplotAggregationBuilder::registerAggregators),
             new AggregationSpec(
                 TopMetricsAggregationBuilder.NAME,
                 TopMetricsAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
                 .addResultReader(InternalTopMetrics::new),
             new AggregationSpec(
                 TTestAggregationBuilder.NAME,
                 TTestAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
                 .addResultReader(InternalTTest::new)
         );
     }
@@ -137,7 +137,7 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
             ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
             Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
             IndexNameExpressionResolver indexNameExpressionResolver) {
-        return singletonList(new AnalyticsUsage());
+        return singletonList(usage);
     }
 
     @Override

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java

@@ -8,54 +8,32 @@ package org.elasticsearch.xpack.analytics;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
 import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
 
-import java.util.EnumMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
 /**
  * Tracks usage of the Analytics aggregations.
  */
 public class AnalyticsUsage {
-    /**
-     * Items to track.
-     */
-    public enum Item {
-        BOXPLOT,
-        CUMULATIVE_CARDINALITY,
-        STRING_STATS,
-        TOP_METRICS,
-        T_TEST;
-    }
 
-    private final Map<Item, AtomicLong> trackers = new EnumMap<>(Item.class);
+    private final EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
 
     public AnalyticsUsage() {
-        for (Item item: Item.values()) {
-            trackers.put(item, new AtomicLong(0));
-        }
     }
 
     /**
      * Track successful parsing.
      */
-    public <C, T> ContextParser<C, T> track(Item item, ContextParser<C, T> realParser) {
-        AtomicLong usage = trackers.get(item);
+    public <C, T> ContextParser<C, T> track(AnalyticsStatsAction.Item item, ContextParser<C, T> realParser) {
         return (parser, context) -> {
             T value = realParser.parse(parser, context);
             // Intentionally doesn't count unless the parser returns cleanly.
-            usage.incrementAndGet();
+            counters.inc(item);
             return value;
         };
     }
 
     public AnalyticsStatsAction.NodeResponse stats(DiscoveryNode node) {
-        return new AnalyticsStatsAction.NodeResponse(node,
-                trackers.get(Item.BOXPLOT).get(),
-                trackers.get(Item.CUMULATIVE_CARDINALITY).get(),
-                trackers.get(Item.STRING_STATS).get(),
-                trackers.get(Item.TOP_METRICS).get(),
-                trackers.get(Item.T_TEST).get());
+        return new AnalyticsStatsAction.NodeResponse(node, counters);
     }
 }

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java

@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.analytics.action;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -20,26 +21,49 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
+import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public class AnalyticsUsageTransportAction extends XPackUsageFeatureTransportAction {
     private final XPackLicenseState licenseState;
+    private final Client client;
 
     @Inject
     public AnalyticsUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
                                          ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                         XPackLicenseState licenseState) {
+                                         XPackLicenseState licenseState, Client client) {
         super(XPackUsageFeatureAction.ANALYTICS.name(), transportService, clusterService,
             threadPool, actionFilters, indexNameExpressionResolver);
         this.licenseState = licenseState;
+        this.client = client;
     }
 
     @Override
     protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
                                    ActionListener<XPackUsageFeatureResponse> listener) {
         boolean available = licenseState.isDataScienceAllowed();
+        if (available) {
+            AnalyticsStatsAction.Request statsRequest = new AnalyticsStatsAction.Request();
+            statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+            client.execute(AnalyticsStatsAction.INSTANCE, statsRequest, ActionListener.wrap(r ->
+                    listener.onResponse(new XPackUsageFeatureResponse(usageFeatureResponse(true, true, r))),
+                listener::onFailure));
+        } else {
+            AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(false, true, Collections.emptyMap());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
 
-        AnalyticsFeatureSetUsage usage =
-            new AnalyticsFeatureSetUsage(available, true);
-        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    static AnalyticsFeatureSetUsage usageFeatureResponse(boolean available, boolean enabled, AnalyticsStatsAction.Response r) {
+        List<EnumCounters<AnalyticsStatsAction.Item>> countersPerNode = r.getNodes()
+            .stream()
+            .map(AnalyticsStatsAction.NodeResponse::getStats)
+            .collect(Collectors.toList());
+        EnumCounters<AnalyticsStatsAction.Item> mergedCounters = EnumCounters.merge(AnalyticsStatsAction.Item.class, countersPerNode);
+        return new AnalyticsFeatureSetUsage(available, enabled, mergedCounters.toMap());
     }
 }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java

@@ -5,28 +5,54 @@
  */
 package org.elasticsearch.xpack.analytics.action;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
+import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
 import org.junit.Before;
+import org.mockito.stubbing.Answer;
+
+import java.util.Collections;
 
 import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 public class AnalyticsInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
+    private Task task;
+    private ClusterService clusterService;
+    private ClusterName clusterName;
 
     @Before
     public void init() {
         licenseState = mock(XPackLicenseState.class);
+        task = mock(Task.class);
+        when(task.getId()).thenReturn(randomLong());
+        clusterService = mock(ClusterService.class);
+        DiscoveryNode discoveryNode = mock(DiscoveryNode.class);
+        when(discoveryNode.getId()).thenReturn(randomAlphaOfLength(10));
+        when(clusterService.localNode()).thenReturn(discoveryNode);
+        clusterName = mock(ClusterName.class);
     }
 
     public void testAvailable() throws Exception {
@@ -35,11 +61,11 @@ public class AnalyticsInfoTransportActionTests extends ESTestCase {
         boolean available = randomBoolean();
         when(licenseState.isDataScienceAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
-
-        AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), null, null,
-            mock(ActionFilters.class), null, licenseState);
+        Client client = mockClient();
+        AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), clusterService, null,
+            mock(ActionFilters.class), null, licenseState, client);
         PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
-        usageAction.masterOperation(null, null, null, future);
+        usageAction.masterOperation(task, null, null, future);
         XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.available(), is(available));
 
@@ -47,6 +73,10 @@ public class AnalyticsInfoTransportActionTests extends ESTestCase {
         usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new AnalyticsFeatureSetUsage(out.bytes().streamInput());
         assertThat(serializedUsage.available(), is(available));
+        if (available) {
+            verify(client, times(1)).execute(any(), any(), any());
+        }
+        verifyNoMoreInteractions(client);
     }
 
     public void testEnabled() throws Exception {
@@ -54,11 +84,13 @@ public class AnalyticsInfoTransportActionTests extends ESTestCase {
             mock(TransportService.class), mock(ActionFilters.class), licenseState);
         assertThat(featureSet.enabled(), is(true));
         assertTrue(featureSet.enabled());
-
+        boolean available = randomBoolean();
+        when(licenseState.isDataScienceAllowed()).thenReturn(available);
+        Client client = mockClient();
         AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class),
-            null, null, mock(ActionFilters.class), null, licenseState);
+            clusterService, null, mock(ActionFilters.class), null, licenseState, client);
         PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
-        usageAction.masterOperation(null, null, null, future);
+        usageAction.masterOperation(task, null, null, future);
         XPackFeatureSet.Usage usage = future.get().getUsage();
         assertTrue(usage.enabled());
 
@@ -66,6 +98,22 @@ public class AnalyticsInfoTransportActionTests extends ESTestCase {
         usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new AnalyticsFeatureSetUsage(out.bytes().streamInput());
         assertTrue(serializedUsage.enabled());
+        if (available) {
+            verify(client, times(1)).execute(any(), any(), any());
+        }
+        verifyNoMoreInteractions(client);
+    }
+
+    private Client mockClient() {
+        Client client = mock(Client.class);
+        doAnswer((Answer<Void>) invocation -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<AnalyticsStatsAction.Response> listener =
+                (ActionListener<AnalyticsStatsAction.Response>) invocation.getArguments()[2];
+            listener.onResponse(new AnalyticsStatsAction.Response(clusterName, Collections.emptyList(), Collections.emptyList()));
+            return null;
+        }).when(client).execute(eq(AnalyticsStatsAction.INSTANCE), any(), any());
+        return client;
     }
 
 }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.analytics.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
+import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AnalyticsStatsActionNodeResponseTests extends AbstractWireSerializingTestCase<AnalyticsStatsAction.NodeResponse> {
+
+    @Override
+    protected Writeable.Reader<AnalyticsStatsAction.NodeResponse> instanceReader() {
+        return AnalyticsStatsAction.NodeResponse::new;
+    }
+
+    @Override
+    protected AnalyticsStatsAction.NodeResponse createTestInstance() {
+        String nodeName = randomAlphaOfLength(10);
+        DiscoveryNode node = new DiscoveryNode(nodeName, buildNewFakeTransportAddress(), Version.CURRENT);
+        EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
+        for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
+            if (randomBoolean()) {
+                counters.inc(item, randomLongBetween(0, 1000));
+            }
+        }
+        return new AnalyticsStatsAction.NodeResponse(node, counters);
+    }
+
+    public void testItemEnum() {
+        int i = 0;
+        // We rely on the ordinals for serialization, so they shouldn't change between version
+        assertThat(AnalyticsStatsAction.Item.BOXPLOT.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.STRING_STATS.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.TOP_METRICS.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.T_TEST.ordinal(), equalTo(i++));
+        // Please add tests for newly added items here
+        assertThat(AnalyticsStatsAction.Item.values().length, equalTo(i));
+    }
+}

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java

@@ -22,6 +22,7 @@ import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.analytics.AnalyticsUsage;
+import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
 import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
 
 import java.io.IOException;
@@ -33,6 +34,7 @@ import java.util.Locale;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction.usageFeatureResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -58,20 +60,18 @@ public class TransportAnalyticsStatsActionTests extends ESTestCase {
     }
 
     public void test() throws IOException {
-        for (AnalyticsUsage.Item item : AnalyticsUsage.Item.values()) {
+        for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
             AnalyticsUsage realUsage = new AnalyticsUsage();
             AnalyticsUsage emptyUsage = new AnalyticsUsage();
             ContextParser<Void, Void> parser = realUsage.track(item, (p, c) -> c);
             ObjectPath unused = run(realUsage, emptyUsage);
-            assertThat(unused.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
-            assertThat(unused.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
+            assertThat(unused.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
             int count = between(1, 10000);
             for (int i = 0; i < count; i++) {
                 assertNull(parser.parse(null, null));
             }
             ObjectPath used = run(realUsage, emptyUsage);
-            assertThat(item.name(), used.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
-            assertThat(item.name(), used.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
+            assertThat(item.name(), used.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
         }
     }
 
@@ -83,11 +83,9 @@ public class TransportAnalyticsStatsActionTests extends ESTestCase {
         AnalyticsStatsAction.Response response = new AnalyticsStatsAction.Response(
                 new ClusterName("cluster_name"), nodeResponses, emptyList());
 
+        AnalyticsFeatureSetUsage usage = usageFeatureResponse(true, true, response);
         try (XContentBuilder builder = jsonBuilder()) {
-            builder.startObject();
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-
+            usage.toXContent(builder, ToXContent.EMPTY_PARAMS);
             return ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder));
         }
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java

@@ -34,4 +34,4 @@ public class XPackUsageFeatureResponse extends ActionResponse {
         out.writeNamedWriteable(usage);
     }
 
-    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java

@@ -43,4 +43,4 @@ public class XPackUsageResponse extends ActionResponse {
         }
     }
 
-    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java

@@ -6,26 +6,47 @@
 
 package org.elasticsearch.xpack.core.analytics;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 public class AnalyticsFeatureSetUsage extends XPackFeatureSet.Usage {
 
-    public AnalyticsFeatureSetUsage(boolean available, boolean enabled) {
+    private final Map<String, Object> stats;
+
+    public AnalyticsFeatureSetUsage(boolean available, boolean enabled, Map<String, Object> stats) {
         super(XPackField.ANALYTICS, available, enabled);
+        this.stats = stats;
     }
 
     public AnalyticsFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
+        if (input.getVersion().onOrAfter(Version.V_7_8_0)) {
+            stats = input.readMap();
+        } else {
+            stats = Collections.emptyMap();
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+            out.writeMap(stats);
+        }
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(available, enabled);
+        return Objects.hash(available, enabled, stats);
     }
 
     @Override
@@ -38,6 +59,19 @@ public class AnalyticsFeatureSetUsage extends XPackFeatureSet.Usage {
         }
         AnalyticsFeatureSetUsage other = (AnalyticsFeatureSetUsage) obj;
         return Objects.equals(available, other.available) &&
-            Objects.equals(enabled, other.enabled);
+            Objects.equals(enabled, other.enabled) &&
+            Objects.equals(stats, other.stats);
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        if (enabled) {
+            builder.startObject("stats");
+            for (Map.Entry<String, Object> entry : stats.entrySet()) {
+                builder.field(entry.getKey() + "_usage", entry.getValue());
+            }
+            builder.endObject();
+        }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.analytics;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+/**
+ * Utility class similar to org.elasticsearch.xpack.core.watcher.common.stats.Counters, but it is using Enum instead
+ * of string to identify the counter. The serialization happens using enum ordinals similar to
+ * {@link StreamOutput#writeEnum(Enum)}, which means that ordinal for existing enums should remain the same for backward
+ * and forward compatibility of the serialization protocol.
+ */
+public class EnumCounters<E extends Enum<E>> implements Writeable {
+    private final AtomicLongArray counters;
+    private final E[] enums;
+
+    public EnumCounters(Class<E> enumClass) {
+        counters = new AtomicLongArray(enumClass.getEnumConstants().length);
+        enums = enumClass.getEnumConstants();
+    }
+
+    public EnumCounters(StreamInput in, Class<E> enumClass) throws IOException {
+        int size = in.readVInt();
+        enums = enumClass.getEnumConstants();
+        long[] vals = new long[enums.length];
+        for (int i = 0; i < size; i++) {
+            long val = in.readVLong();
+            if (i < vals.length) {
+                vals[i] = val;
+            }
+        }
+        counters = new AtomicLongArray(vals);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(counters.length());
+        for (int i = 0; i < counters.length(); i++) {
+            out.writeVLong(counters.get(i));
+        }
+    }
+
+    public void set(E name) {
+        counters.set(name.ordinal(), 0);
+    }
+
+    public void inc(E name) {
+        counters.incrementAndGet(name.ordinal());
+    }
+
+    public void inc(E name, long count) {
+        counters.addAndGet(name.ordinal(), count);
+    }
+
+    public long get(E name) {
+        return counters.get(name.ordinal());
+    }
+
+    public long size() {
+        return counters.length();
+    }
+
+    public boolean hasCounters() {
+        return size() > 0;
+    }
+
+    public Map<String, Object> toMap() {
+        Map<String, Object> map = new HashMap<>();
+        for (E e : enums) {
+            map.put(e.name().toLowerCase(Locale.ROOT), counters.get(e.ordinal()));
+        }
+        return map;
+    }
+
+    public static <E extends Enum<E>> EnumCounters<E> merge(Class<E> enumClass, List<EnumCounters<E>> counters) {
+        EnumCounters<E> result = new EnumCounters<>(enumClass);
+        E[] enums = enumClass.getEnumConstants();
+        for (EnumCounters<E> c : counters) {
+            for (E e : enums) {
+                result.inc(e, c.get(e));
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EnumCounters<?> that = (EnumCounters<?>) o;
+        return Arrays.equals(toArray(), that.toArray()) &&
+            Arrays.equals(enums, that.enums);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(toArray());
+        result = 31 * result + Arrays.hashCode(enums);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder("[");
+        boolean first = true;
+        for (E e : enums) {
+            buf.append(e.name().toLowerCase(Locale.ROOT)).append(": ").append(get(e));
+            if (first) {
+                buf.append(", ");
+                first = false;
+            }
+        }
+        buf.append("]");
+        return buf.toString();
+    }
+
+    private long[] toArray() {
+        long[] res = new long[enums.length];
+        for (int i = 0; i < res.length; i++) {
+            res[i] = counters.get(i);
+        }
+        return res;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java

@@ -13,13 +13,13 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
 
 import java.io.IOException;
 import java.util.List;
@@ -33,6 +33,17 @@ public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Respon
         super(NAME, Response::new);
     }
 
+    /**
+     * Items to track. Serialized by ordinals. Append only, don't remove or change order of items in this list.
+     */
+    public enum Item {
+        BOXPLOT,
+        CUMULATIVE_CARDINALITY,
+        STRING_STATS,
+        TOP_METRICS,
+        T_TEST;
+    }
+
     public static class Request extends BaseNodesRequest<Request> implements ToXContentObject {
 
         public Request() {
@@ -78,7 +89,7 @@ public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Respon
         }
     }
 
-    public static class Response extends BaseNodesResponse<NodeResponse> implements Writeable, ToXContentObject {
+    public static class Response extends BaseNodesResponse<NodeResponse> implements Writeable {
         public Response(StreamInput in) throws IOException {
             super(in);
         }
@@ -96,110 +107,66 @@ public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Respon
         protected void writeNodesTo(StreamOutput out, List<NodeResponse> nodes) throws IOException {
             out.writeList(nodes);
         }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startArray("stats");
-            for (NodeResponse node : getNodes()) {
-                node.toXContent(builder, params);
-            }
-            builder.endArray();
-
-            return builder;
-        }
     }
 
-    public static class NodeResponse extends BaseNodeResponse implements ToXContentObject {
-        static final ParseField BOXPLOT_USAGE = new ParseField("boxplot_usage");
-        static final ParseField CUMULATIVE_CARDINALITY_USAGE = new ParseField("cumulative_cardinality_usage");
-        static final ParseField STRING_STATS_USAGE = new ParseField("string_stats_usage");
-        static final ParseField TOP_METRICS_USAGE = new ParseField("top_metrics_usage");
-        static final ParseField T_TEST_USAGE = new ParseField("t_test_usage");
-
-        private final long boxplotUsage;
-        private final long cumulativeCardinalityUsage;
-        private final long stringStatsUsage;
-        private final long topMetricsUsage;
-        private final long ttestUsage;
-
-        public NodeResponse(DiscoveryNode node, long boxplotUsage, long cumulativeCardinalityUsage, long stringStatsUsage,
-                long topMetricsUsage, long ttestUsage) {
+    public static class NodeResponse extends BaseNodeResponse {
+        private final EnumCounters<Item> counters;
+
+        public NodeResponse(DiscoveryNode node, EnumCounters<Item> counters) {
             super(node);
-            this.boxplotUsage = boxplotUsage;
-            this.cumulativeCardinalityUsage = cumulativeCardinalityUsage;
-            this.stringStatsUsage = stringStatsUsage;
-            this.topMetricsUsage = topMetricsUsage;
-            this.ttestUsage = ttestUsage;
+            this.counters = counters;
         }
 
         public NodeResponse(StreamInput in) throws IOException {
             super(in);
-            if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-                boxplotUsage = in.readVLong();
-            } else {
-                boxplotUsage = 0;
-            }
-            cumulativeCardinalityUsage = in.readZLong();
-            if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-                stringStatsUsage = in.readVLong();
-                topMetricsUsage = in.readVLong();
-            } else {
-                stringStatsUsage = 0;
-                topMetricsUsage = 0;
-            }
             if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
-                ttestUsage = in.readVLong();
+                counters = new EnumCounters<>(in, Item.class);
             } else {
-                ttestUsage = 0;
+                counters = new EnumCounters<>(Item.class);
+                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    counters.inc(Item.BOXPLOT, in.readVLong());
+                }
+                counters.inc(Item.CUMULATIVE_CARDINALITY, in.readZLong());
+                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    counters.inc(Item.STRING_STATS, in.readVLong());
+                    counters.inc(Item.TOP_METRICS, in.readVLong());
+                }
             }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-                out.writeVLong(boxplotUsage);
-            }
-            out.writeVLong(cumulativeCardinalityUsage);
-            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-                out.writeVLong(stringStatsUsage);
-                out.writeVLong(topMetricsUsage);
-            }
             if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
-                out.writeVLong(ttestUsage);
+                counters.writeTo(out);
+            } else {
+                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    out.writeVLong(counters.get(Item.BOXPLOT));
+                }
+                out.writeZLong(counters.get(Item.CUMULATIVE_CARDINALITY));
+                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    out.writeVLong(counters.get(Item.STRING_STATS));
+                    out.writeVLong(counters.get(Item.TOP_METRICS));
+                }
             }
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(BOXPLOT_USAGE.getPreferredName(), boxplotUsage);
-            builder.field(CUMULATIVE_CARDINALITY_USAGE.getPreferredName(), cumulativeCardinalityUsage);
-            builder.field(STRING_STATS_USAGE.getPreferredName(), stringStatsUsage);
-            builder.field(TOP_METRICS_USAGE.getPreferredName(), topMetricsUsage);
-            builder.field(T_TEST_USAGE.getPreferredName(), ttestUsage);
-            builder.endObject();
-            return builder;
-        }
-
-        public long getBoxplotUsage() {
-            return boxplotUsage;
+        public EnumCounters<Item> getStats() {
+            return counters;
         }
 
-        public long getCumulativeCardinalityUsage() {
-            return cumulativeCardinalityUsage;
-        }
-
-        public long getStringStatsUsage() {
-            return stringStatsUsage;
-        }
-
-        public long getTopMetricsUsage() {
-            return topMetricsUsage;
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeResponse that = (NodeResponse) o;
+            return counters.equals(that.counters) &&
+                getNode().equals(that.getNode());
         }
 
-        public long getTTestUsage() {
-            return topMetricsUsage;
+        @Override
+        public int hashCode() {
+            return Objects.hash(counters, getNode());
         }
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/analytics/EnumCountersTests.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.analytics;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireTestCase;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class EnumCountersTests extends AbstractWireTestCase<EnumCounters<EnumCountersTests.TestV2>> {
+
+    enum TestV1 {A, B, C}
+
+    enum TestV2 {A, B, C, D}
+
+    @Override
+    protected EnumCounters<TestV2> createTestInstance() {
+        EnumCounters<TestV2> inst = new EnumCounters<>(TestV2.class);
+        inst.inc(TestV2.A, randomNonNegativeLong());
+        inst.inc(TestV2.B, randomNonNegativeLong());
+        inst.inc(TestV2.C, randomNonNegativeLong());
+        inst.inc(TestV2.D, randomNonNegativeLong());
+        return inst;
+    }
+
+    @Override
+    protected EnumCounters<TestV2> copyInstance(EnumCounters<TestV2> instance, Version version) throws IOException {
+        return serialize(instance, in -> new EnumCounters<>(in, TestV2.class));
+    }
+
+    public void testIncrements() {
+        EnumCounters<TestV1> counters = new EnumCounters<>(TestV1.class);
+        int a = randomIntBetween(0, 100);
+        int b = randomIntBetween(0, 100);
+        int c = randomIntBetween(0, 100);
+        incrementRandomly(counters, TestV1.A, a);
+        incrementRandomly(counters, TestV1.B, b);
+        incrementRandomly(counters, TestV1.C, c);
+        assertEquals(a, counters.get(TestV1.A));
+        assertEquals(b, counters.get(TestV1.B));
+        assertEquals(c, counters.get(TestV1.C));
+        Map<String, Object> map = counters.toMap();
+        assertThat(map.keySet(), hasSize(3));
+        assertThat(map.get("a"), equalTo((long) a));
+        assertThat(map.get("b"), equalTo((long) b));
+        assertThat(map.get("c"), equalTo((long) c));
+    }
+
+    public void testBackwardCompatibility() throws Exception {
+        EnumCounters<TestV2> counters = new EnumCounters<>(TestV2.class);
+        counters.inc(TestV2.A, 1);
+        counters.inc(TestV2.B, 2);
+        counters.inc(TestV2.C, 3);
+        counters.inc(TestV2.D, 4);
+        EnumCounters<TestV1> oldCounters = serialize(counters, in -> new EnumCounters<>(in, TestV1.class));
+        assertEquals(counters.get(TestV2.A), oldCounters.get(TestV1.A));
+        assertEquals(counters.get(TestV2.B), oldCounters.get(TestV1.B));
+        assertEquals(counters.get(TestV2.C), oldCounters.get(TestV1.C));
+    }
+
+
+    public void testForwardCompatibility() throws Exception {
+        EnumCounters<TestV1> counters = new EnumCounters<>(TestV1.class);
+        counters.inc(TestV1.A, 1);
+        counters.inc(TestV1.B, 2);
+        counters.inc(TestV1.C, 3);
+        EnumCounters<TestV2> newCounters = serialize(counters, in -> new EnumCounters<>(in, TestV2.class));
+        assertEquals(counters.get(TestV1.A), newCounters.get(TestV2.A));
+        assertEquals(counters.get(TestV1.B), newCounters.get(TestV2.B));
+        assertEquals(counters.get(TestV1.C), newCounters.get(TestV2.C));
+        assertEquals(0, newCounters.get(TestV2.D));
+    }
+
+    private <E1 extends Enum<E1>, E2 extends Enum<E2>> EnumCounters<E2> serialize(
+        EnumCounters<E1> source, Writeable.Reader<EnumCounters<E2>> targetReader) throws IOException {
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            source.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                return targetReader.read(in);
+            }
+        }
+    }
+
+    private <E extends Enum<E>> void incrementRandomly(EnumCounters<E> counters, E e, int inc) {
+        int single = randomIntBetween(0, inc);
+        if (randomBoolean()) {
+            for (int i = 0; i < single; i++) {
+                counters.inc(e);
+            }
+            counters.inc(e, inc - single);
+        } else {
+            counters.inc(e, inc - single);
+            for (int i = 0; i < single; i++) {
+                counters.inc(e);
+            }
+        }
+    }
+
+}

x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/usage.yml

@@ -0,0 +1,158 @@
+---
+setup:
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"timestamp": "2017-01-01T05:00:00Z", "s": 1, "v1": 3.1415, "v2": 2.1415, "str": "a"}'
+          - '{"index": {}}'
+          - '{"timestamp": "2017-01-01T05:00:00Z", "s": 2, "v1": 1.0, "v2": 2.0, "str": "a"}'
+          - '{"index": {}}'
+          - '{"timestamp": "2017-01-01T05:00:00Z", "s": 3, "v1": 2.71828, "v2": 3.71828, "str": "b"}'
+
+---
+"Usage stats on analytics indices":
+  - skip:
+      version: " - 7.99.99"
+      reason:  "stats is not working in earlier versions at the moment"
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - set: {analytics.stats.boxplot_usage: boxplot_usage}
+  - set: {analytics.stats.top_metrics_usage: top_metrics_usage}
+  - set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage}
+  - set: {analytics.stats.t_test_usage: t_test_usage}
+  - set: {analytics.stats.string_stats_usage: string_stats_usage}
+
+  # use boxplot agg
+  - do:
+      search:
+        index: "test"
+        body:
+          size: 0
+          aggs:
+            plot:
+              boxplot:
+                field: "s"
+
+  - match: { aggregations.plot.q2: 2.0 }
+
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - gt: { analytics.stats.boxplot_usage: $boxplot_usage }
+  - set: {analytics.stats.boxplot_usage: boxplot_usage}
+  - match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
+  - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
+  - match: {analytics.stats.t_test_usage: $t_test_usage}
+  - match: {analytics.stats.string_stats_usage: $string_stats_usage}
+
+
+  # use top_metrics agg
+  - do:
+      search:
+        index: "test"
+        size: 0
+        body:
+          aggs:
+            tm:
+              top_metrics:
+                metrics:
+                  field: v1
+                sort:
+                  s: desc
+  - match: { aggregations.tm.top.0.metrics.v1: 2.718280076980591 }
+  - match: { aggregations.tm.top.0.sort: [3] }
+
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - match: {analytics.stats.boxplot_usage: $boxplot_usage}
+  - gt: { analytics.stats.top_metrics_usage: $top_metrics_usage }
+  - set: {analytics.stats.top_metrics_usage: top_metrics_usage}
+  - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
+  - match: {analytics.stats.t_test_usage: $t_test_usage}
+  - match: {analytics.stats.string_stats_usage: $string_stats_usage}
+
+
+  # use cumulative_cardinality agg
+  - do:
+      search:
+        index: "test"
+        body:
+          size: 0
+          aggs:
+            histo:
+              date_histogram:
+                field: "timestamp"
+                calendar_interval: "day"
+              aggs:
+                distinct_s:
+                  cardinality:
+                    field: "s"
+                total_users:
+                  cumulative_cardinality:
+                    buckets_path: "distinct_s"
+
+  - length: { aggregations.histo.buckets: 1 }
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - match: {analytics.stats.boxplot_usage: $boxplot_usage}
+  - match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
+  - gt: { analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage }
+  - set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage}
+  - match: {analytics.stats.t_test_usage: $t_test_usage}
+  - match: {analytics.stats.string_stats_usage: $string_stats_usage}
+
+  # use t-test agg
+  - do:
+      search:
+        size: 0
+        index: "test"
+        body:
+          aggs:
+            ttest:
+              t_test:
+                a:
+                  field: v1
+                b:
+                  field: v2
+  - match: { aggregations.ttest.value: 0.7172402682151968 }
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - match: {analytics.stats.boxplot_usage: $boxplot_usage}
+  - match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
+  - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
+  - gt: { analytics.stats.t_test_usage: $t_test_usage }
+  - set: {analytics.stats.t_test_usage: t_test_usage}
+  - match: {analytics.stats.string_stats_usage: $string_stats_usage}
+
+  - do:
+      search:
+        size: 0
+        index: "test"
+        body:
+          aggs:
+            my_agg:
+              string_stats:
+                field: str.keyword
+  - match: { aggregations.my_agg.count: 3 }
+
+  - do: {xpack.usage: {}}
+  - match: { analytics.available: true }
+  - match: { analytics.enabled: true }
+  - match: {analytics.stats.boxplot_usage: $boxplot_usage}
+  - match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
+  - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
+  - match: {analytics.stats.t_test_usage: $t_test_usage}
+  - gt: { analytics.stats.string_stats_usage: $string_stats_usage }
+  - set: {analytics.stats.string_stats_usage: string_stats_usage}