Core: Store index creation time in index metadata

This change stores the index creation time in the index metadata when an index is created.  The creation time cannot be changed but can be set as part of the create index request to allow for correct creation times for historical data.

Closes #7119

docs/reference/indices/create-index.asciidoc

@@ -129,3 +129,21 @@ curl -XPUT localhost:9200/test -d '{
     }
 }'
 --------------------------------------------------
+
+[float]
+=== Creation Date
+
+coming[1.4.0]
+
+When an index is created, a timestamp is stored in the index metadata for the creation date.  By 
+default this it is automatically generated but it can also be specified using the 
+`creation_date` parameter on the create index API:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT localhost:9200/test -d '{
+    "creation_date" : 1407751337000 <1>
+}'
+--------------------------------------------------
+
+<1> `creation_date` is set using epoch time in milliseconds.

src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

@@ -45,9 +45,9 @@ import java.util.Set;
 
 import static com.google.common.collect.Maps.newHashMap;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream;
+import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
 
 /**
  * A request to create an index. Best created with {@link org.elasticsearch.client.Requests#createIndexRequest(String)}.

src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -160,6 +160,7 @@ public class IndexMetaData {
     public static final String SETTING_BLOCKS_WRITE = "index.blocks.write";
     public static final String SETTING_BLOCKS_METADATA = "index.blocks.metadata";
     public static final String SETTING_VERSION_CREATED = "index.version.created";
+    public static final String SETTING_CREATION_DATE = "index.creation_date";
     public static final String SETTING_UUID = "index.uuid";
     public static final String INDEX_UUID_NA_VALUE = "_na_";
 
@@ -251,6 +252,14 @@ public class IndexMetaData {
         return this.version;
     }
 
+    public long creationDate() {
+        return settings.getAsLong(SETTING_CREATION_DATE, -1l);
+    }
+
+    public long getCreationDate() {
+        return creationDate();
+    }
+
     public State state() {
         return this.state;
     }
@@ -457,6 +466,15 @@ public class IndexMetaData {
         public int numberOfReplicas() {
             return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
         }
+        
+        public Builder creationDate(long creationDate) {
+            settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
+            return this;
+        }
+
+        public long creationDate() {
+            return settings.getAsLong(SETTING_CREATION_DATE, -1l);
+        }
 
         public Builder settings(Settings.Builder settings) {
             this.settings = settings.build();

src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -36,6 +36,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData.Custom;
+import org.elasticsearch.cluster.metadata.IndexMetaData.State;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -324,6 +326,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                         final Version createdVersion = Version.smallest(version, nodes.smallestNonClientNodeVersion());
                         indexSettingsBuilder.put(SETTING_VERSION_CREATED, createdVersion);
                     }
+
+                    if (indexSettingsBuilder.get(SETTING_CREATION_DATE) == null) {
+                        indexSettingsBuilder.put(SETTING_CREATION_DATE, System.currentTimeMillis());
+                    }
+
                     indexSettingsBuilder.put(SETTING_UUID, Strings.randomBase64UUID());
 
                     Settings actualIndexSettings = indexSettingsBuilder.build();

src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexTests.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.*;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+@ClusterScope(scope = Scope.TEST)
+public class CreateIndexTests extends ElasticsearchIntegrationTest{
+
+    @Test
+    public void testCreationDate_Given() {
+        prepareCreate("test").setSettings(ImmutableSettings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4l)).get();
+        ClusterStateResponse response = client().admin().cluster().prepareState().get();
+        ClusterState state = response.getState();
+        assertThat(state, notNullValue());
+        MetaData metadata = state.getMetaData();
+        assertThat(metadata, notNullValue());
+        ImmutableOpenMap<String, IndexMetaData> indices = metadata.getIndices();
+        assertThat(indices, notNullValue());
+        assertThat(indices.size(), equalTo(1));
+        IndexMetaData index = indices.get("test");
+        assertThat(index, notNullValue());
+        assertThat(index.creationDate(), equalTo(4l));
+    }
+    
+    @Test
+    public void testCreationDate_Generated() {
+        long timeBeforeRequest = System.currentTimeMillis();
+        prepareCreate("test").get();
+        long timeAfterRequest = System.currentTimeMillis();
+        ClusterStateResponse response = client().admin().cluster().prepareState().get();
+        ClusterState state = response.getState();
+        assertThat(state, notNullValue());
+        MetaData metadata = state.getMetaData();
+        assertThat(metadata, notNullValue());
+        ImmutableOpenMap<String, IndexMetaData> indices = metadata.getIndices();
+        assertThat(indices, notNullValue());
+        assertThat(indices.size(), equalTo(1));
+        IndexMetaData index = indices.get("test");
+        assertThat(index, notNullValue());
+        assertThat(index.creationDate(), allOf(lessThanOrEqualTo(timeAfterRequest), greaterThanOrEqualTo(timeBeforeRequest)));
+    }
+
+}

src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

@@ -28,9 +28,7 @@ import java.io.IOException;
 
 import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder;
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.*;
 
 /**
  *
@@ -52,12 +50,27 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
                         .numberOfReplicas(2)
                         .putMapping("mapping1", MAPPING_SOURCE1))
                 .put(IndexMetaData.builder("test4")
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .creationDate(2l))
+                .put(IndexMetaData.builder("test5")
                         .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
                         .numberOfShards(1)
                         .numberOfReplicas(2)
                         .putMapping("mapping1", MAPPING_SOURCE1)
                         .putMapping("mapping2", MAPPING_SOURCE2))
-                .put(IndexMetaData.builder("test5")
+                .put(IndexMetaData.builder("test6")
+                        .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .creationDate(2l))
+                .put(IndexMetaData.builder("test7")
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .creationDate(2l)
+                        .putMapping("mapping1", MAPPING_SOURCE1)
+                        .putMapping("mapping2", MAPPING_SOURCE2))
+                .put(IndexMetaData.builder("test8")
                         .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
                         .numberOfShards(1)
                         .numberOfReplicas(2)
@@ -65,7 +78,16 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
                         .putMapping("mapping2", MAPPING_SOURCE2)
                         .putAlias(newAliasMetaDataBuilder("alias1"))
                         .putAlias(newAliasMetaDataBuilder("alias2")))
-                .put(IndexMetaData.builder("test6")
+                .put(IndexMetaData.builder("test9")
+                        .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
+                        .creationDate(2l)
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .putMapping("mapping1", MAPPING_SOURCE1)
+                        .putMapping("mapping2", MAPPING_SOURCE2)
+                        .putAlias(newAliasMetaDataBuilder("alias1"))
+                        .putAlias(newAliasMetaDataBuilder("alias2")))
+                .put(IndexMetaData.builder("test10")
                         .settings(settingsBuilder()
                                 .put("setting1", "value1")
                                 .put("setting2", "value2"))
@@ -75,7 +97,7 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
                         .putMapping("mapping2", MAPPING_SOURCE2)
                         .putAlias(newAliasMetaDataBuilder("alias1"))
                         .putAlias(newAliasMetaDataBuilder("alias2")))
-                .put(IndexMetaData.builder("test7")
+                .put(IndexMetaData.builder("test11")
                         .settings(settingsBuilder()
                                 .put("setting1", "value1")
                                 .put("setting2", "value2"))
@@ -95,6 +117,27 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
                                 .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                                 .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                                 .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put(IndexMetaData.builder("test12")
+                        .settings(settingsBuilder()
+                                .put("setting1", "value1")
+                                .put("setting2", "value2"))
+                                .creationDate(2l)
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .putMapping("mapping1", MAPPING_SOURCE1)
+                        .putMapping("mapping2", MAPPING_SOURCE2)
+                        .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1))
+                        .putAlias(newAliasMetaDataBuilder("alias2"))
+                        .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2)))
+                        .put(IndexTemplateMetaData.builder("foo")
+                                .template("bar")
+                                .order(1)
+                                .settings(settingsBuilder()
+                                        .put("setting1", "value1")
+                                        .put("setting2", "value2"))
+                                .putAlias(newAliasMetaDataBuilder("alias-bar1"))
+                                .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
+                                .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
                 .build();
 
         String metaDataSource = MetaData.Builder.toXContent(metaData);
@@ -105,12 +148,14 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         IndexMetaData indexMetaData = parsedMetaData.index("test1");
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
         assertThat(indexMetaData.mappings().size(), equalTo(0));
 
         indexMetaData = parsedMetaData.index("test2");
         assertThat(indexMetaData.numberOfShards(), equalTo(2));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(3));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
         assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
@@ -119,13 +164,22 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         indexMetaData = parsedMetaData.index("test3");
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
         assertThat(indexMetaData.mappings().size(), equalTo(1));
         assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
 
         indexMetaData = parsedMetaData.index("test4");
+        assertThat(indexMetaData.creationDate(), equalTo(2l));
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.mappings().size(), equalTo(0));
+
+        indexMetaData = parsedMetaData.index("test5");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
         assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
@@ -133,9 +187,28 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
         assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
 
-        indexMetaData = parsedMetaData.index("test5");
+        indexMetaData = parsedMetaData.index("test6");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(2l));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
+        assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
+        assertThat(indexMetaData.mappings().size(), equalTo(0));
+
+        indexMetaData = parsedMetaData.index("test7");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(2l));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.mappings().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
+        assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
+
+        indexMetaData = parsedMetaData.index("test8");
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
         assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
@@ -146,9 +219,24 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1"));
         assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2"));
 
-        indexMetaData = parsedMetaData.index("test6");
+        indexMetaData = parsedMetaData.index("test9");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(2l));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
+        assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
+        assertThat(indexMetaData.mappings().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
+        assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
+        assertThat(indexMetaData.aliases().size(), equalTo(2));
+        assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1"));
+        assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2"));
+
+        indexMetaData = parsedMetaData.index("test10");
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
         assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
@@ -159,9 +247,10 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1"));
         assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2"));
 
-        indexMetaData = parsedMetaData.index("test7");
+        indexMetaData = parsedMetaData.index("test11");
         assertThat(indexMetaData.numberOfShards(), equalTo(1));
         assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(-1l));
         assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
         assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
@@ -176,6 +265,24 @@ public class ToAndFromJsonMetaDataTests extends ElasticsearchTestCase {
         assertThat(indexMetaData.aliases().get("alias4").alias(), equalTo("alias4"));
         assertThat(indexMetaData.aliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2));
 
+        indexMetaData = parsedMetaData.index("test12");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.creationDate(), equalTo(2l));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
+        assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
+        assertThat(indexMetaData.mappings().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
+        assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
+        assertThat(indexMetaData.aliases().size(), equalTo(3));
+        assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1"));
+        assertThat(indexMetaData.aliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1));
+        assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2"));
+        assertThat(indexMetaData.aliases().get("alias2").filter(), nullValue());
+        assertThat(indexMetaData.aliases().get("alias4").alias(), equalTo("alias4"));
+        assertThat(indexMetaData.aliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2));
+
         // templates
         assertThat(parsedMetaData.templates().get("foo").name(), is("foo"));
         assertThat(parsedMetaData.templates().get("foo").template(), is("bar"));