Settings: Remove file based index templates

As a follow up to #10870, this removes support for
index templates on disk. It also removes a missed
place still allowing disk based mappings.

closes #11052

docs/reference/indices/templates.asciidoc

@@ -68,7 +68,7 @@ curl -XDELETE localhost:9200/_template/template_1
 
 [float]
 [[getting]]
-=== GETting templates
+=== Getting templates
 
 Index templates are identified by a name (in the above case
 `template_1`) and can be retrieved using the following:
@@ -157,39 +157,3 @@ for indices of that start with `te*`, source will still be enabled.
 Note, for mappings, the merging is "deep", meaning that specific
 object/property based mappings can easily be added/overridden on higher
 order templates, with lower order templates providing the basis.
-
-[float]
-[[config]]
-=== Config
-
-Index templates can also be placed within the config location
-(`path.conf`) under the `templates` directory (note, make sure to place
-them on all master eligible nodes). For example, a file called
-`template_1.json` can be placed under `config/templates` and it will be
-added if it matches an index. Here is a sample of the mentioned file:
-
-[source,js]
---------------------------------------------------
-{
-    "template_1" : {
-        "template" : "*",
-        "settings" : {
-            "index.number_of_shards" : 2
-        },
-        "mappings" : {
-            "_default_" : {
-                "_source" : {
-                    "enabled" : false
-                }
-            },
-            "type1" : {
-                "_all" : {
-                    "enabled" : false
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-
-*Please note that templates added this way will not appear in the `/_template/*` API request.*

docs/reference/migration/migrate_2_0.asciidoc

@@ -519,4 +519,7 @@ Log messages are now truncated at 10,000 characters. This can be changed in the
 
 The `top_children` query has been removed in favour of the `has_child` query. The `top_children` query wasn't always faster
 than the `has_child` query and the `top_children` query was often inaccurate. The total hits and any aggregations in the
-same search request will likely be off if `top_children` was used.
+same search request will likely be off if `top_children` was used.
+
+=== Removed file based index templates
+Index templates can no longer be configured on disk. Use the `_template` API instead.

src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -300,22 +300,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                         }
                     }
 
-                    // now add config level mappings
-                    Path mappingsDir = environment.configFile().resolve("mappings");
-                    if (Files.isDirectory(mappingsDir)) {
-                        // first index level
-                        Path indexMappingsDir = mappingsDir.resolve(request.index());
-                        if (Files.isDirectory(indexMappingsDir)) {
-                            addMappings(mappings, indexMappingsDir);
-                        }
-
-                        // second is the _default mapping
-                        Path defaultMappingsDir = mappingsDir.resolve("_default");
-                        if (Files.isDirectory(defaultMappingsDir)) {
-                            addMappings(mappings, defaultMappingsDir);
-                        }
-                    }
-
                     ImmutableSettings.Builder indexSettingsBuilder = settingsBuilder();
                     // apply templates, here, in reverse order, since first ones are better matching
                     for (int i = templates.size() - 1; i >= 0; i--) {
@@ -517,30 +501,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
             }
         }
 
-        // see if we have templates defined under config
-        final Path templatesDir = environment.configFile().resolve("templates");
-        if (Files.isDirectory(templatesDir)) {
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(templatesDir)) {
-                for (Path templatesFile : stream) {
-                    if (Files.isRegularFile(templatesFile)) {
-                        XContentParser parser = null;
-                        try {
-                            final byte[] templatesData = Files.readAllBytes(templatesFile);
-                            parser = XContentHelper.createParser(templatesData, 0, templatesData.length);
-                            IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, templatesFile.getFileName().toString());
-                            if (indexTemplateFilter.apply(request, template)) {
-                                templates.add(template);
-                            }
-                        } catch (Exception e) {
-                            logger.warn("[{}] failed to read template [{}] from config", e, request.index(), templatesFile.toAbsolutePath());
-                        } finally {
-                            Releasables.closeWhileHandlingException(parser);
-                        }
-                    }
-                }
-            }
-        }
-
         CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
             @Override
             public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {

src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java

@@ -1,111 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
-import org.elasticsearch.test.ElasticsearchTestCase;
-
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Map;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-
-public class FileBasedMappingsTests extends ElasticsearchTestCase {
-
-    private static final String NAME = FileBasedMappingsTests.class.getSimpleName();
-
-    public void testFileBasedMappings() throws Exception {
-        Path configDir = createTempDir();
-        Path mappingsDir = configDir.resolve("mappings");
-        Path indexMappings = mappingsDir.resolve("index").resolve("type.json");
-        Path defaultMappings = mappingsDir.resolve("_default").resolve("type.json");
-        try {
-            Files.createDirectories(indexMappings.getParent());
-            Files.createDirectories(defaultMappings.getParent());
-
-            try (OutputStream stream = Files.newOutputStream(indexMappings);
-                 XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) {
-                builder.startObject()
-                        .startObject("type")
-                            .startObject("properties")
-                                .startObject("f")
-                                    .field("type", "string")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject();
-            }
-
-            try (OutputStream stream = Files.newOutputStream(defaultMappings);
-                 XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) {
-                builder.startObject()
-                        .startObject("type")
-                            .startObject("properties")
-                                .startObject("g")
-                                    .field("type", "string")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject();
-            }
-
-            Settings settings = ImmutableSettings.builder()
-                    .put(ClusterName.SETTING, NAME)
-                    .put("node.name", NAME)
-                    .put("path.home", createTempDir())
-                    .put("path.conf", configDir.toAbsolutePath())
-                    .put("http.enabled", false)
-                    .build();
-
-            try (Node node = NodeBuilder.nodeBuilder().local(true).data(true).settings(settings).node()) {
-
-                assertAcked(node.client().admin().indices().prepareCreate("index").addMapping("type", "h", "type=string").get());
-                try {
-                    final GetMappingsResponse response = node.client().admin().indices().prepareGetMappings("index").get();
-                    assertTrue(response.mappings().toString(), response.mappings().containsKey("index"));
-                    MappingMetaData mappings = response.mappings().get("index").get("type");
-                    assertNotNull(mappings);
-                    Map<?, ?> properties = (Map<?, ?>) (mappings.getSourceAsMap().get("properties"));
-                    assertNotNull(properties);
-                    assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet());
-                } finally {
-                    // remove the index...
-                    assertAcked(node.client().admin().indices().prepareDelete("index"));
-                }
-            }
-        } finally {
-            IOUtils.rm(configDir);
-        }
-    }
-
-}

src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java

@@ -1,104 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.indices.template;
-
-import com.carrotsearch.randomizedtesting.LifecycleScope;
-import com.google.common.base.Charsets;
-
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
-import org.junit.Test;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Set;
-
-import static org.elasticsearch.test.ElasticsearchIntegrationTest.*;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-@ClusterScope(scope= Scope.TEST, numDataNodes =1)
-public class IndexTemplateFileLoadingTests extends ElasticsearchIntegrationTest {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder();
-        settingsBuilder.put(super.nodeSettings(nodeOrdinal));
-
-        try {
-            Path directory = createTempDir();
-            settingsBuilder.put("path.conf", directory.toAbsolutePath());
-
-            Path templatesDir = directory.resolve("templates");
-            Files.createDirectory(templatesDir);
-
-            Path dst = templatesDir.resolve("template.json");
-            String templatePath = "/org/elasticsearch/indices/template/template" + randomInt(5) + ".json";
-            logger.info("Picking template path [{}]", templatePath);
-            // random template, one uses the 'setting.index.number_of_shards', the other 'settings.number_of_shards'
-            String template = Streams.copyToStringFromClasspath(templatePath);
-            Files.write(dst, template.getBytes(StandardCharsets.UTF_8));
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-
-        return settingsBuilder.build();
-    }
-
-    @Override
-    protected int numberOfShards() {
-        //number of shards won't be set through index settings, the one from the index templates needs to be used
-        return -1;
-    }
-
-    @Override
-    protected int numberOfReplicas() {
-        //number of replicas won't be set through index settings, the one from the index templates needs to be used
-        return -1;
-    }
-
-    @Test
-    public void testThatLoadingTemplateFromFileWorks() throws Exception {
-        final int iters = scaledRandomIntBetween(1, 5);
-        Set<String> indices = new HashSet<>();
-        for (int i = 0; i < iters; i++) {
-            String indexName = "foo" + randomAsciiOfLengthBetween(0, 5).toLowerCase(Locale.ROOT);
-            if (indices.contains(indexName)) {
-                continue;
-            }
-            indices.add(indexName);
-            createIndex(indexName);
-            ensureYellow(); // ensuring yellow so the test fails faster if the template cannot be loaded
-
-            ClusterStateResponse stateResponse = client().admin().cluster().prepareState().setIndices(indexName).get();
-            assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfShards(), is(10));
-            assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfReplicas(), is(0));
-            assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().size(), equalTo(1));
-            String aliasName = indexName + "-alias";
-            assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().get(aliasName).alias(), equalTo(aliasName));
-        }
-    }
-}