Added ccr to xpack usage infrastructure (#37256)

* Added ccr to xpack usage infrastructure

Closes #37221

docs/reference/rest-api/info.asciidoc

@@ -63,6 +63,11 @@ Example response:
       "expiry_date_in_millis" : 1542665112332
    },
    "features" : {
+      "ccr" : {
+        "description" : "Cross Cluster Replication",
+        "available" : true,
+        "enabled" : true
+      },
       "graph" : {
          "description" : "Graph Data Exploration for the Elastic Stack",
          "available" : true,

x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class XPackUsageIT extends ESCCRRestTestCase {
+
+    public void testXPackCcrUsage() throws Exception {
+        if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]" );
+            return;
+        }
+
+        Map<?, ?> previousUsage = getCcrUsage();
+        putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*");
+
+        // This index should be auto followed:
+        createLeaderIndex("messages-20200101");
+        // This index will be followed manually
+        createLeaderIndex("my_index");
+        followIndex("my_index", "my_index");
+
+        int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count");
+        int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count");
+        assertBusy(() -> {
+            Map<?, ?> ccrUsage = getCcrUsage();
+            assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2));
+            assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1));
+            assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
+        });
+
+        deleteAutoFollowPattern("my_pattern");
+        pauseFollow("messages-20200101");
+        closeIndex("messages-20200101");
+        unfollow("messages-20200101");
+
+        pauseFollow("my_index");
+        closeIndex("my_index");
+        unfollow("my_index");
+
+        assertBusy(() -> {
+            Map<?, ?> ccrUsage = getCcrUsage();
+            assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount));
+            assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount));
+            if (previousFollowerIndicesCount == 0) {
+                assertThat(ccrUsage.get("last_follow_time_in_millis"), nullValue());
+            } else {
+                assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
+            }
+        });
+    }
+
+    private void createLeaderIndex(String indexName) throws IOException {
+        try (RestClient leaderClient = buildLeaderClient()) {
+            Settings settings = Settings.builder()
+                .put("index.soft_deletes.enabled", true)
+                .build();
+            Request request = new Request("PUT", "/" + indexName);
+            request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}");
+            assertOK(leaderClient.performRequest(request));
+        }
+    }
+
+    private Map<?, ?> getCcrUsage() throws IOException {
+        Request request = new Request("GET", "/_xpack/usage");
+        Map<String, ?> response = toMap(client().performRequest(request));
+        logger.info("xpack usage response={}", response);
+        return  (Map<?, ?>) response.get("ccr");
+    }
+
+}

x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java

@@ -87,6 +87,22 @@ public class ESCCRRestTestCase extends ESRestTestCase {
         assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow")));
     }
 
+    protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException {
+        Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName);
+        putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" +
+            remoteCluster + "\"}");
+        assertOK(client().performRequest(putPatternRequest));
+    }
+
+    protected static void deleteAutoFollowPattern(String patternName) throws IOException {
+        Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName);
+        assertOK(client().performRequest(putPatternRequest));
+    }
+
+    protected static void unfollow(String followIndex) throws IOException {
+        assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow")));
+    }
+
     protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException {
         verifyDocuments(index, expectedNumDocs, query, adminClient());
     }

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java

@@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -81,6 +82,7 @@ import org.elasticsearch.xpack.ccr.rest.RestPutFollowAction;
 import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction;
 import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
@@ -126,6 +128,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
     private final SetOnce<CcrRestoreSourceService> restoreSourceService = new SetOnce<>();
     private final SetOnce<CcrSettings> ccrSettings = new SetOnce<>();
     private Client client;
+    private final boolean transportClientMode;
 
     /**
      * Construct an instance of the CCR container with the specified settings.
@@ -147,6 +150,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
         this.settings = settings;
         this.enabled = CCR_ENABLED_SETTING.get(settings);
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
+        this.transportClientMode = XPackPlugin.transportClientMode(settings);
     }
 
     @Override
@@ -314,6 +318,15 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
         }
     }
 
+    @Override
+    public Collection<Module> createGuiceModules() {
+        if (transportClientMode) {
+            return Collections.emptyList();
+        }
+
+        return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class));
+    }
+
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
 
     @Override

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CCRFeatureSetTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+    private ClusterService clusterService;
+
+    @Before
+    public void init() throws Exception {
+        licenseState = mock(XPackLicenseState.class);
+        clusterService = mock(ClusterService.class);
+    }
+
+    public void testAvailable() {
+        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+
+        when(licenseState.isCcrAllowed()).thenReturn(false);
+        assertThat(featureSet.available(), equalTo(false));
+
+        when(licenseState.isCcrAllowed()).thenReturn(true);
+        assertThat(featureSet.available(), equalTo(true));
+
+        featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService);
+        assertThat(featureSet.available(), equalTo(false));
+    }
+
+    public void testEnabled() {
+        Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false);
+        CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(false));
+
+        settings = Settings.builder().put("xpack.ccr.enabled", true);
+        featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(true));
+    }
+
+    public void testName() {
+        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        assertThat(featureSet.name(), equalTo("ccr"));
+    }
+
+    public void testNativeCodeInfo() {
+        CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService);
+        assertNull(featureSet.nativeCodeInfo());
+    }
+
+    public void testUsageStats() throws Exception {
+        MetaData.Builder metaData = MetaData.builder();
+
+        int numFollowerIndices = randomIntBetween(0, 32);
+        for (int i = 0; i < numFollowerIndices; i++) {
+            IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i)
+                .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(i)
+                .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
+            metaData.put(followerIndex);
+        }
+
+        // Add a regular index, to check that we do not take that one into account:
+        IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .creationDate(numFollowerIndices);
+        metaData.put(regularIndex);
+
+        int numAutoFollowPatterns = randomIntBetween(0, 32);
+        Map<String, AutoFollowMetadata.AutoFollowPattern> patterns = new HashMap<>(numAutoFollowPatterns);
+        for (int i = 0; i < numAutoFollowPatterns; i++) {
+            AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser",
+                Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null);
+            patterns.put("pattern" + i, pattern);
+        }
+        metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build();
+        Mockito.when(clusterService.state()).thenReturn(clusterState);
+
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        ccrFeatureSet.usage(future);
+        CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get();
+        assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled()));
+        assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available()));
+
+        assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices));
+        assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L));
+        assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns));
+    }
+
+}

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
+
+public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRFeatureSet.Usage> {
+
+    @Override
+    protected CCRFeatureSet.Usage createTestInstance() {
+        return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong());
+    }
+
+    @Override
+    protected Writeable.Reader<CCRFeatureSet.Usage> instanceReader() {
+        return CCRFeatureSet.Usage::new;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
@@ -412,6 +413,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
                     in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new),
                 // ILM
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
                     IndexLifecycleFeatureSetUsage::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -33,6 +33,8 @@ public final class XPackField {
     public static final String ROLLUP = "rollup";
     /** Name constant for the index lifecycle feature. */
     public static final String INDEX_LIFECYCLE = "ilm";
+    /** Name constant for the CCR feature. */
+    public static final String CCR = "ccr";
 
     private XPackField() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java

@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ccr;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Map;
+import java.util.Objects;
+
+public class CCRFeatureSet implements XPackFeatureSet {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final ClusterService clusterService;
+
+    @Inject
+    public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
+        this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings);
+        this.licenseState = licenseState;
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.CCR;
+    }
+
+    @Override
+    public String description() {
+        return "Cross Cluster Replication";
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState != null && licenseState.isCcrAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @Override
+    public Map<String, Object> nativeCodeInfo() {
+        return null;
+    }
+
+    @Override
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
+        MetaData metaData = clusterService.state().metaData();
+
+        int numberOfFollowerIndices = 0;
+        long lastFollowerIndexCreationDate = 0L;
+        for (IndexMetaData imd : metaData) {
+            if (imd.getCustomData("ccr") != null) {
+                numberOfFollowerIndices++;
+                if (lastFollowerIndexCreationDate < imd.getCreationDate()) {
+                    lastFollowerIndexCreationDate = imd.getCreationDate();
+                }
+            }
+        }
+        AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE);
+        int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0;
+
+        Long lastFollowTimeInMillis;
+        if (numberOfFollowerIndices == 0) {
+            // Otherwise we would return a value that makes no sense.
+            lastFollowTimeInMillis = null;
+        } else {
+            lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate);
+        }
+
+        Usage usage =
+            new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
+        listener.onResponse(usage);
+    }
+
+    public static class Usage extends XPackFeatureSet.Usage {
+
+        private final int numberOfFollowerIndices;
+        private final int numberOfAutoFollowPatterns;
+        private final Long lastFollowTimeInMillis;
+
+        public Usage(boolean available,
+                     boolean enabled,
+                     int numberOfFollowerIndices,
+                     int numberOfAutoFollowPatterns,
+                     Long lastFollowTimeInMillis) {
+            super(XPackField.CCR, available, enabled);
+            this.numberOfFollowerIndices = numberOfFollowerIndices;
+            this.numberOfAutoFollowPatterns = numberOfAutoFollowPatterns;
+            this.lastFollowTimeInMillis = lastFollowTimeInMillis;
+        }
+
+        public Usage(StreamInput in) throws IOException {
+            super(in);
+            numberOfFollowerIndices = in.readVInt();
+            numberOfAutoFollowPatterns = in.readVInt();
+            if (in.readBoolean()) {
+                lastFollowTimeInMillis = in.readVLong();
+            } else {
+                lastFollowTimeInMillis = null;
+            }
+        }
+
+        public int getNumberOfFollowerIndices() {
+            return numberOfFollowerIndices;
+        }
+
+        public int getNumberOfAutoFollowPatterns() {
+            return numberOfAutoFollowPatterns;
+        }
+
+        public Long getLastFollowTimeInMillis() {
+            return lastFollowTimeInMillis;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeVInt(numberOfFollowerIndices);
+            out.writeVInt(numberOfAutoFollowPatterns);
+            if (lastFollowTimeInMillis != null) {
+                out.writeBoolean(true);
+                out.writeVLong(lastFollowTimeInMillis);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
+
+        @Override
+        protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+            super.innerXContent(builder, params);
+            builder.field("follower_indices_count", numberOfFollowerIndices);
+            builder.field("auto_follow_patterns_count", numberOfAutoFollowPatterns);
+            if (lastFollowTimeInMillis != null) {
+                builder.field("last_follow_time_in_millis", lastFollowTimeInMillis);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Usage usage = (Usage) o;
+            return numberOfFollowerIndices == usage.numberOfFollowerIndices &&
+                numberOfAutoFollowPatterns == usage.numberOfAutoFollowPatterns &&
+                Objects.equals(lastFollowTimeInMillis, usage.lastFollowTimeInMillis);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
+        }
+    }
+}