[8.x] Deprecate dot-prefixed indices and composable template index patterns (#112571) (#113146)

* Deprecate dot-prefixed indices and composable template index patterns (#112571)

This commit adds a module emitting a deprecation warning when a
dot-prefixed index is manually or automatically created, or when a
composable index template with an index pattern that uses a dot-prefix
is created. This pattern warns that in the future these indices will not
be allowed. In a future breaking change (10.0.0 maybe?) the deprecation
can then be changed to an exception.

These deprecations are only displayed when a non-operator user is using
the API (one that does not set the `X-elastic-product-origin` header).

* Attempt to fix build for V7 gradle stuff

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/changelog/112571.yaml

@@ -0,0 +1,17 @@
+pr: 112571
+summary: Deprecate dot-prefixed indices and composable template index patterns
+area: CRUD
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dot-prefixed indices and composable template index patterns
+  area: CRUD
+  details: "Indices beginning with a dot '.' are reserved for system and internal\
+    \ indices, and should not be used by and end-user. Additionally, composable index\
+    \ templates that contain patterns for dot-prefixed indices should also be avoided,\
+    \ as these patterns are meant for internal use only. In a future Elasticsearch\
+    \ version, creation of these dot-prefixed indices will no longer be allowed."
+  impact: "Requests performing an action that would create an index beginning with\
+    \ a dot (indexing a document, manual creation, reindex), or creating an index\
+    \ template with index patterns beginning with a dot, will contain a deprecation\
+    \ header warning about dot-prefixed indices in the response."

modules/data-streams/build.gradle

@@ -40,3 +40,7 @@ if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.failure_store_feature_flag_enabled', 'true'
   }
 }
+
+tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
+  task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility")
+})

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.datastreams;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -71,10 +72,12 @@ public class DataStreamsRestIT extends DisabledSecurityDataStreamTestCase {
         // Create a template
         Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden");
         putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\".hidden\"], \"data_stream\": {\"hidden\": true}}");
+        putComposableIndexTemplateRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
         assertOK(client().performRequest(putComposableIndexTemplateRequest));
 
         Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true");
         createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }");
+        createDocRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
 
         assertOK(client().performRequest(createDocRequest));
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -115,8 +115,10 @@ setup:
 "Create hidden data stream":
   - requires:
       cluster_features: ["gte_v7.11.0"]
+      test_runner_features: ["warnings", "headers"]
       reason: "hidden data streams only available in 7.11"
   - do:
+      headers: { X-elastic-product-origin: elastic }
       allowed_warnings:
         - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
       indices.put_index_template:

modules/dot-prefix-validation/build.gradle

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  description 'Validation for dot-prefixed indices for non-operator users'
+  classname 'org.elasticsearch.validation.DotPrefixValidationPlugin'
+}
+
+restResources {
+  restApi {
+    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex'
+  }
+}
+
+tasks.named("yamlRestTestV7CompatTest").configure { enabled = false };
+
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}

modules/dot-prefix-validation/src/main/java/module-info.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module org.elasticsearch.validation {
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.base;
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class AutoCreateDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return AutoCreateAction.NAME;
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class CreateIndexDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return TransportCreateIndexAction.TYPE.name();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin {
+    private final AtomicReference<List<MappedActionFilter>> actionFilters = new AtomicReference<>();
+
+    public DotPrefixValidationPlugin() {}
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        ThreadContext context = services.threadPool().getThreadContext();
+        ClusterService clusterService = services.clusterService();
+
+        actionFilters.set(
+            List.of(
+                new CreateIndexDotValidator(context, clusterService),
+                new AutoCreateDotValidator(context, clusterService),
+                new IndexTemplateDotValidator(context, clusterService)
+            )
+        );
+
+        return Set.of();
+    }
+
+    @Override
+    public Collection<MappedActionFilter> getMappedActionFilters() {
+        return actionFilters.get();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ActionFilterChain;
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.tasks.Task;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * DotPrefixValidator provides an abstract class implementing a mapped action filter.
+ *
+ * This class then implements the {@link #apply(Task, String, ActionRequest, ActionListener, ActionFilterChain)}
+ * method which checks for indices in the request that begin with a dot, emitting a deprecation
+ * warning if they do. If the request is performed by a non-external user (operator, internal product, etc.)
+ * as defined by {@link #isInternalRequest()} then the deprecation is emitted. Otherwise, it is skipped.
+ *
+ * The indices for consideration are returned by the abstract {@link #getIndicesFromRequest(Object)}
+ * method, which subclasses must implement.
+ *
+ * Some built-in index names and patterns are also elided from the check, as defined in
+ * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS}.
+ */
+public abstract class DotPrefixValidator<RequestType> implements MappedActionFilter {
+    public static final Setting<Boolean> VALIDATE_DOT_PREFIXES = Setting.boolSetting(
+        "cluster.indices.validate_dot_prefixes",
+        true,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Names and patterns for indexes where no deprecation should be emitted.
+     * Normally we would want to transition these to either system indices, or
+     * to use an internal origin for the client. These are shorter-term
+     * workarounds until that work can be completed.
+     *
+     * .elastic-connectors-* is used by enterprise search
+     * .ml-* is used by ML
+     */
+    private static Set<String> IGNORED_INDEX_NAMES = Set.of(
+        ".elastic-connectors-v1",
+        ".elastic-connectors-sync-jobs-v1",
+        ".ml-state",
+        ".ml-anomalies-unrelated"
+    );
+    private static Set<Pattern> IGNORED_INDEX_PATTERNS = Set.of(Pattern.compile("\\.ml-state-\\d+"));
+
+    DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
+
+    private final ThreadContext threadContext;
+    private final boolean isEnabled;
+
+    public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) {
+        this.threadContext = threadContext;
+        this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings());
+    }
+
+    protected abstract Set<String> getIndicesFromRequest(RequestType request);
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void apply(
+        Task task,
+        String action,
+        Request request,
+        ActionListener<Response> listener,
+        ActionFilterChain<Request, Response> chain
+    ) {
+        Set<String> indices = getIndicesFromRequest((RequestType) request);
+        if (isEnabled) {
+            validateIndices(indices);
+        }
+        chain.proceed(task, action, request, listener);
+    }
+
+    void validateIndices(@Nullable Set<String> indices) {
+        if (indices != null && isInternalRequest() == false) {
+            for (String index : indices) {
+                if (Strings.hasLength(index)) {
+                    char c = getFirstChar(index);
+                    if (c == '.') {
+                        if (IGNORED_INDEX_NAMES.contains(index)) {
+                            return;
+                        }
+                        if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(index).matches())) {
+                            return;
+                        }
+                        deprecationLogger.warn(
+                            DeprecationCategory.INDICES,
+                            "dot-prefix",
+                            "Index [{}] name begins with a dot (.), which is deprecated, "
+                                + "and will not be allowed in a future Elasticsearch version.",
+                            index
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    private static char getFirstChar(String index) {
+        char c = index.charAt(0);
+        if (c == '<') {
+            // Date-math is being used for the index, we need to
+            // consider it by stripping the first '<' before we
+            // check for a dot-prefix
+            String strippedLeading = index.substring(1);
+            if (Strings.hasLength(strippedLeading)) {
+                c = strippedLeading.charAt(0);
+            }
+        }
+        return c;
+    }
+
+    private boolean isInternalRequest() {
+        final String actionOrigin = threadContext.getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME);
+        final boolean isSystemContext = threadContext.isSystemContext();
+        final boolean isInternalOrigin = Optional.ofNullable(actionOrigin).map(Strings::hasText).orElse(false);
+        final boolean hasElasticOriginHeader = Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER))
+            .map(Strings::hasText)
+            .orElse(false);
+        return isSystemContext || isInternalOrigin || hasElasticOriginHeader;
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class IndexTemplateDotValidator extends DotPrefixValidator<TransportPutComposableIndexTemplateAction.Request> {
+    public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) {
+        return new HashSet<>(Arrays.asList(request.indices()));
+    }
+
+    @Override
+    public String actionName() {
+        return TransportPutComposableIndexTemplateAction.TYPE.name();
+    }
+}

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+
+import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
+
+public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public DotPrefixClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password"));
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build();
+    }
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = createCluster();
+
+    private static ElasticsearchCluster createCluster() {
+        LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
+            .distribution(DistributionType.DEFAULT)
+            .feature(FAILURE_STORE_ENABLED)
+            .setting("xpack.security.enabled", "true")
+            .keystore("bootstrap.password", "x-pack-test-password")
+            .user("x_pack_rest_user", "x-pack-test-password");
+        boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true"));
+        if (setNodes) {
+            clusterBuilder.nodes(2);
+        }
+        return clusterBuilder.build();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+}

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -0,0 +1,193 @@
+---
+teardown:
+  - do:
+      indices.delete:
+        index: .*
+
+---
+"Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set":
+  - requires:
+      test_runner_features: ["warnings", "warnings_regex", "headers"]
+
+  - do:
+      warnings:
+        - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+      indices.create:
+        index: .myindex
+        body:
+          settings:
+            index.hidden: true
+
+  - do:
+      headers: { X-elastic-product-origin: kibana }
+      indices.create:
+        index: .myindex2
+        body:
+          settings:
+            index.hidden: true
+
+  - do:
+      warnings_regex:
+        - "Index \\[.*\\] name begins with a dot \\(\\.\\), which is deprecated, and will not be allowed in a future Elasticsearch version\\."
+      indices.create:
+        index: <.myindex-{now/d}>
+        body:
+          settings:
+            index.hidden: true
+
+  - do:
+      headers: { X-elastic-product-origin: kibana }
+      indices.create:
+        index: <.myindex2-{now/d}>
+        body:
+          settings:
+            index.hidden: true
+
+---
+"Deprecated auto-creation of dot-prefixed indices":
+  - requires:
+      test_runner_features: ["warnings", "headers"]
+
+  - do:
+      warnings:
+        - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.myindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      index:
+        index: .myindex
+        id: "1"
+        body: {foo: bar}
+
+  - do:
+      headers: { X-elastic-product-origin: kibana }
+      warnings:
+        - "index name [.myindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      index:
+        index: .myindex2
+        id: "1"
+        body: {foo: bar}
+
+  - do:
+      warnings:
+        - "Index [.myindex3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.myindex3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      bulk:
+        body:
+          - index:
+              _index: .myindex3
+          - message: foo
+
+---
+"Deprecated auto-creation of dot-prefixed indices through pipelines":
+  - requires:
+      test_runner_features: ["warnings", "headers"]
+
+  - do:
+      ingest.put_pipeline:
+        id: mypipeline
+        body:  >
+          {
+            "processors": [
+              {
+                "set" : {
+                  "field" : "_index",
+                  "value": "{{redirect_to}}"
+                }
+              }
+            ]
+          }
+  - do:
+      warnings:
+        - "Index [.other] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.other] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      index:
+        index: myindex
+        id: "1"
+        body: {redirect_to: ".other"}
+        pipeline: mypipeline
+
+  - do:
+      warnings:
+        - "Index [.other2] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.other2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      bulk:
+        body:
+          - index:
+              _index: myindex
+              pipeline: mypipeline
+          - redirect_to: .other2
+
+  - do:
+      index:
+        index: original
+        id: "1"
+        body: { "redirect_to": ".other3" }
+  - do:
+      indices.refresh: {}
+  - do:
+      warnings:
+        - "Index [.other3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.other3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      reindex:
+        body:
+          source:
+            index: original
+          dest:
+            index: newindex
+            pipeline: mypipeline
+
+  - do:
+      warnings:
+        - "Index [.reindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        - "index name [.reindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      reindex:
+        body:
+          source:
+            index: original
+          dest:
+            index: .reindex
+
+  - do:
+      headers: { X-elastic-product-origin: kibana }
+      warnings:
+        - "index name [.reindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices"
+      reindex:
+        body:
+          source:
+            index: original
+          dest:
+            index: .reindex2
+
+  - do:
+      ingest.delete_pipeline:
+        id: mypipeline
+
+---
+"Deprecated index template with a dot prefix index pattern":
+  - requires:
+      test_runner_features: ["warnings", "headers"]
+
+  - do:
+      warnings:
+        - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: [regular, .data-*]
+          data_stream: {}
+
+  - do:
+      headers: { X-elastic-product-origin: kibana }
+      warnings:
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [other, .data2-*]
+          data_stream: {}
+
+  - do:
+      indices.delete_index_template:
+        name: my-template
+
+  - do:
+      indices.delete_index_template:
+        name: my-template2

qa/mixed-cluster/build.gradle

@@ -58,6 +58,9 @@ excludeList.add('cluster.desired_nodes/11_old_format/Test node version must have
 excludeList.add('cluster.desired_nodes/11_old_format/Test node version can not be null')
 excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry run updates')
 
+// Excluded because they create dot-prefixed indices on older versions
+excludeList.add('indices.resolve_index/20_resolve_system_index/*')
+
 BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
 
   if (bwcVersion != VersionProperties.getElasticsearchVersion()) {

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.system.indices;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -43,13 +44,17 @@ public class FeatureUpgradeApiIT extends ESRestTestCase {
     }
 
     public void testCreatingSystemIndex() throws Exception {
-        Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        var request = new Request("PUT", "/_net_new_sys_index/_create");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
+        Response response = client().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
     @SuppressWarnings("unchecked")
     public void testGetFeatureUpgradedStatuses() throws Exception {
-        client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        var request = new Request("PUT", "/_net_new_sys_index/_create");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
+        client().performRequest(request);
         Response response = client().performRequest(new Request("GET", "/_migration/system_features"));
         assertThat(response.getStatusLine().getStatusCode(), is(200));
         XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent());

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java

@@ -12,6 +12,7 @@ package org.elasticsearch.system.indices;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
@@ -40,7 +41,9 @@ public class NetNewSystemIndicesIT extends ESRestTestCase {
         );
         assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system"));
 
-        Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        var request = new Request("PUT", "/_net_new_sys_index/_create");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
+        Response response = client().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
@@ -56,6 +59,7 @@ public class NetNewSystemIndicesIT extends ESRestTestCase {
 
         Request request = new Request("PUT", "/_net_new_sys_index/" + id);
         request.setJsonEntity("{}");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
         Response response = client().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
@@ -74,6 +78,7 @@ public class NetNewSystemIndicesIT extends ESRestTestCase {
         Request request = new Request("PUT", "/_net_new_sys_index/" + id);
         request.setJsonEntity("{}");
         request.addParameter("refresh", "true");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
         Response response = client().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
 

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.system.indices;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -42,6 +43,7 @@ public class SystemAliasIT extends ESRestTestCase {
         {
             Request request = new Request("PUT", "/.internal-unmanaged-index-8");
             request.setJsonEntity("{\"aliases\": {\".internal-unmanaged-alias\": {}}}");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             Response response = client().performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
@@ -68,6 +70,7 @@ public class SystemAliasIT extends ESRestTestCase {
 
         {
             Request request = new Request("PUT", "/.internal-unmanaged-index-8");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             Response response = client().performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
@@ -79,6 +82,7 @@ public class SystemAliasIT extends ESRestTestCase {
     public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception {
         {
             Request request = new Request("PUT", "/.internal-unmanaged-index-8");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             Response response = client().performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
@@ -102,6 +106,7 @@ public class SystemAliasIT extends ESRestTestCase {
     public void testCreatingSystemIndexWithAliasEndpoint() throws Exception {
         {
             Request request = new Request("PUT", "/.internal-unmanaged-index-8");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             Response response = client().performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
@@ -126,6 +131,7 @@ public class SystemAliasIT extends ESRestTestCase {
     public void testCreatingSystemIndexWithAliasesEndpoint() throws Exception {
         {
             Request request = new Request("PUT", "/.internal-unmanaged-index-8");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             Response response = client().performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml

@@ -44,7 +44,9 @@
   - requires:
       cluster_features: ["gte_v8.11.0"]
       reason: "dataset size was added in 8.11.0"
+      test_runner_features: ["headers"]
   - do:
+      headers: { X-elastic-product-origin: elastic }
       indices.create:
         index: .index1
         body:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml

@@ -3,9 +3,10 @@ setup:
   - requires:
       cluster_features: ["gte_v8.13.0"]
       reason: "resolve cluster introduced in 8.13"
-      test_runner_features: "warnings"
+      test_runner_features: ["warnings", "headers"]
 
   - do:
+      headers: { X-elastic-product-origin: elastic }
       indices.create:
         index: ".tasks"
 ---

x-pack/plugin/build.gradle

@@ -147,6 +147,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("spatial/70_script_doc_values/geoshape value", "error message changed in 8.9.0")
   task.skipTest("security/authz/14_cat_indices/Test empty request while single authorized index", "not supported for compatibility")
   task.skipTestsByFilePattern("**/rollup/**", "The rollup yaml tests in the 7.x branch don't know how to fake a cluster with rollup usage")
+  task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")

x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java

@@ -204,7 +204,7 @@ public class ProfileIT extends ESRestTestCase {
                 expectWarnings(
                     "this request accesses system indices: [.security-profile-8], but in a future major version, "
                         + "direct access to system indices will be prevented by default"
-                )
+                ).toBuilder().addHeader("X-elastic-product-origin", "elastic")
             );
             assertOK(adminClient().performRequest(indexRequest));
         }
@@ -515,7 +515,7 @@ public class ProfileIT extends ESRestTestCase {
             expectWarnings(
                 "this request accesses system indices: [.security-profile-8], but in a future major version, "
                     + "direct access to system indices will be prevented by default"
-            )
+            ).toBuilder().addHeader("X-elastic-product-origin", "elastic")
         );
         assertOK(adminClient().performRequest(indexRequest));
 

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.test.rest;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -41,6 +42,7 @@ public class CatIndicesWithSecurityIT extends SecurityOnTrialLicenseRestTestCase
             createRequest.setJsonEntity(
                 "{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, \"aliases\": {\"index_allowed\": {}}}"
             );
+            createRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             final Response createResponse = adminClient().performRequest(createRequest);
             assertOK(createResponse);
             ensureGreen("index_allowed");
@@ -73,6 +75,7 @@ public class CatIndicesWithSecurityIT extends SecurityOnTrialLicenseRestTestCase
                     }
                   }
                 }""");
+            createRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
             final Response createResponse = adminClient().performRequest(createRequest);
             assertOK(createResponse);
             ensureGreen("index_allowed");

x-pack/plugin/stack/qa/rest/build.gradle

@@ -17,3 +17,7 @@ testClusters.configureEach {
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }
+
+tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
+  task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility")
+})

x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml

@@ -223,7 +223,11 @@ setup:
 
 ---
 "Test kibana reporting index auto creation":
+  - requires:
+      test_runner_features: ["headers"]
+
   - do:
+      headers: { X-elastic-product-origin: kibana }
       index:
         index: .kibana-reporting-foo
         body:

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java

@@ -103,7 +103,7 @@ public class TransformAuditorIT extends TransformRestTestCase {
                 + "] starts "
                 + "with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices "
                 + "and system indices"
-        );
+        ).toBuilder().addHeader("X-elastic-product-origin", "elastic").build();
         Request request = new Request("PUT", "/" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED);
         String entity = "{\"settings\": " + Strings.toString(settings.build()) + "}";
         request.setJsonEntity(entity);

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformConfigurationIndexIT.java

@@ -40,7 +40,7 @@ public class TransformConfigurationIndexIT extends TransformRestTestCase {
                 + TransformInternalIndexConstants.LATEST_INDEX_NAME
                 + "], but in a future major version, direct access to system indices will "
                 + "be prevented by default"
-        );
+        ).toBuilder().addHeader("X-elastic-product-origin", "elastic").build();
 
         try (XContentBuilder builder = jsonBuilder()) {
             builder.startObject();