Expose the index age in ILM explain output. (#81273)

* Expose the index age in ILM explain output.

 ILM already exposes the `age` that ILM will use to transition to the next phase, based on that phase's `min_age`. The `index_age` is based only on the index creation date and it's used to trigger a rollover.

 Resolves #64429

docs/reference/ilm/apis/explain.asciidoc

@@ -111,30 +111,36 @@ that the index is managed and in the `new` phase:
   "indices": {
     "my-index-000001": {
       "index": "my-index-000001",
-      "managed": true, <1>
-      "policy": "my_policy", <2>
-      "lifecycle_date_millis": 1538475653281, <3>
-      "age": "15s", <4>
+      "index_creation_date_millis": 1538475653281,  <1>
+      "time_since_index_creation": "15s",           <2>
+      "managed": true,                              <3>
+      "policy": "my_policy",                        <4>
+      "lifecycle_date_millis": 1538475653281,       <5>
+      "age": "15s",                                 <6>
       "phase": "new",
-      "phase_time_millis": 1538475653317, <5>
-      "action": "complete",
-      "action_time_millis": 1538475653317, <6>
+      "phase_time_millis": 1538475653317,           <7>
+      "action": "complete"
+      "action_time_millis": 1538475653317,          <8>
       "step": "complete",
-      "step_time_millis": 1538475653317 <7>
+      "step_time_millis": 1538475653317             <9>
     }
   }
 }
 --------------------------------------------------
 // TESTRESPONSE[skip:no way to know if we will get this response immediately]
 
-<1> Shows if the index is being managed by {ilm-init}. If the index is not managed by
+<1> When the index was created, this timestamp is used to determine when to
+rollover
+<2> The time since the index creation (used for calculating when to rollover
+the index via the `max_age`)
+<3> Shows if the index is being managed by {ilm-init}. If the index is not managed by
 {ilm-init} the other fields will not be shown
-<2> The name of the policy which {ilm-init} is using for this index
-<3> The timestamp used for the `min_age`
-<4> The age of the index (used for calculating when to enter the next phase)
-<5> When the index entered the current phase
-<6> When the index entered the current action
-<7> When the index entered the current step
+<4> The name of the policy which {ilm-init} is using for this index
+<5> The timestamp used for the `min_age`
+<6> The age of the index (used for calculating when to enter the next phase)
+<7> When the index entered the current phase
+<8> When the index entered the current action
+<9> When the index entered the current step
 
 Once the policy is running on the index, the response includes a
 `phase_execution` object that shows the definition of the current phase.
@@ -147,6 +153,8 @@ phase completes.
   "indices": {
     "test-000069": {
       "index": "test-000069",
+      "index_creation_date_millis": 1538475653281,
+      "time_since_index_creation": "25.14s",
       "managed": true,
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,
@@ -196,6 +204,8 @@ information for the step that's being performed on the index.
   "indices": {
     "test-000020": {
       "index": "test-000020",
+      "index_creation_date_millis": 1538475653281,
+      "time_since_index_creation": "4.12m",
       "managed": true,
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,
@@ -260,6 +270,8 @@ the case.
   "indices": {
     "test-000056": {
       "index": "test-000056",
+      "index_creation_date_millis": 1538475653281,
+      "time_since_index_creation": "50.1d",
       "managed": true,
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,

docs/reference/ilm/error-handling.asciidoc

@@ -68,6 +68,8 @@ Which returns the following information:
     "my-index-000001" : {
       "index" : "my-index-000001",
       "managed" : true,                         
+      "index_creation_date_millis" : 1541717265865,
+      "time_since_index_creation": "5.1d",
       "policy" : "shrink-index",                <1>
       "lifecycle_date_millis" : 1541717265865,
       "age": "5.1d",                            <2>

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -215,19 +215,21 @@ is met.
   "indices": {
     ".ds-timeseries-2099.03.07-000001": {
       "index": ".ds-timeseries-2099.03.07-000001",
+      "index_creation_date_millis": 1538475653281,
+      "time_since_index_creation": "30s",        <1>
       "managed": true,
-      "policy": "timeseries_policy",             <1>
+      "policy": "timeseries_policy",             <2>
       "lifecycle_date_millis": 1538475653281,
-      "age": "30s",                              <2>
+      "age": "30s",                              <3>
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "action": "rollover",
       "action_time_millis": 1538475653317,
-      "step": "check-rollover-ready",            <3>
+      "step": "check-rollover-ready",            <4>
       "step_time_millis": 1538475653317,
       "phase_execution": {
         "policy": "timeseries_policy",
-        "phase_definition": {                    <4>
+        "phase_definition": {                    <5>
           "min_age": "0ms",
           "actions": {
             "rollover": {
@@ -245,10 +247,11 @@ is met.
 --------------------------------------------------
 // TESTRESPONSE[skip:no way to know if we will get this response immediately]
 
-<1> The policy used to manage the index
-<2> The age of the index
-<3> The step {ilm-init} is performing on the index
-<4> The definition of the current phase (the `hot` phase)
+<1> The age of the index used for calculating when to rollover the index via the `max_age`
+<2> The policy used to manage the index
+<3> The age of the indexed used to transition to the next phase (in this case it is the same with the age of the index).
+<4> The step {ilm-init} is performing on the index
+<5> The definition of the current phase (the `hot` phase)
 
 //////////////////////////
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.ilm;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -22,11 +23,14 @@ import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 public class IndexLifecycleExplainResponse implements ToXContentObject, Writeable {
 
     private static final ParseField INDEX_FIELD = new ParseField("index");
+    private static final ParseField INDEX_CREATION_DATE_MILLIS_FIELD = new ParseField("index_creation_date_millis");
+    private static final ParseField INDEX_CREATION_DATE_FIELD = new ParseField("index_creation_date");
     private static final ParseField MANAGED_BY_ILM_FIELD = new ParseField("managed");
     private static final ParseField POLICY_NAME_FIELD = new ParseField("policy");
     private static final ParseField LIFECYCLE_DATE_MILLIS_FIELD = new ParseField("lifecycle_date_millis");
@@ -46,6 +50,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
     private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
     private static final ParseField AGE_FIELD = new ParseField("age");
+    private static final ParseField TIME_SINCE_INDEX_CREATION_FIELD = new ParseField("time_since_index_creation");
     private static final ParseField REPOSITORY_NAME = new ParseField("repository_name");
     private static final ParseField SHRINK_INDEX_NAME = new ParseField("shrink_index_name");
     private static final ParseField SNAPSHOT_NAME = new ParseField("snapshot_name");
@@ -54,6 +59,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         "index_lifecycle_explain_response",
         a -> new IndexLifecycleExplainResponse(
             (String) a[0],
+            (Long) (a[19]),
             (boolean) a[1],
             (String) a[2],
             (Long) (a[3]),
@@ -72,6 +78,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             (BytesReference) a[11],
             (PhaseExecutionInfo) a[12]
             // a[13] == "age"
+            // a[20] == "time_since_index_creation"
         )
     );
     static {
@@ -102,9 +109,12 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), REPOSITORY_NAME);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SNAPSHOT_NAME);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHRINK_INDEX_NAME);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), INDEX_CREATION_DATE_MILLIS_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SINCE_INDEX_CREATION_FIELD);
     }
 
     private final String index;
+    private final Long indexCreationDate;
     private final String policyName;
     private final String phase;
     private final String action;
@@ -125,6 +135,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
 
     public static IndexLifecycleExplainResponse newManagedIndexResponse(
         String index,
+        Long indexCreationDate,
         String policyName,
         Long lifecycleDate,
         String phase,
@@ -144,6 +155,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     ) {
         return new IndexLifecycleExplainResponse(
             index,
+            indexCreationDate,
             true,
             policyName,
             lifecycleDate,
@@ -167,6 +179,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String index) {
         return new IndexLifecycleExplainResponse(
             index,
+            null,
             false,
             null,
             null,
@@ -189,6 +202,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
 
     private IndexLifecycleExplainResponse(
         String index,
+        Long indexCreationDate,
         boolean managedByILM,
         String policyName,
         Long lifecycleDate,
@@ -232,6 +246,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             }
         } else {
             if (policyName != null
+                || indexCreationDate != null
                 || lifecycleDate != null
                 || phase != null
                 || action != null
@@ -248,6 +263,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             }
         }
         this.index = index;
+        this.indexCreationDate = indexCreationDate;
         this.policyName = policyName;
         this.managedByILM = managedByILM;
         this.lifecycleDate = lifecycleDate;
@@ -287,6 +303,11 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             repositoryName = in.readOptionalString();
             snapshotName = in.readOptionalString();
             shrinkIndexName = in.readOptionalString();
+            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+                indexCreationDate = in.readOptionalLong();
+            } else {
+                indexCreationDate = null;
+            }
         } else {
             policyName = null;
             lifecycleDate = null;
@@ -304,6 +325,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             repositoryName = null;
             snapshotName = null;
             shrinkIndexName = null;
+            indexCreationDate = null;
         }
     }
 
@@ -328,6 +350,9 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             out.writeOptionalString(repositoryName);
             out.writeOptionalString(snapshotName);
             out.writeOptionalString(shrinkIndexName);
+            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+                out.writeOptionalLong(indexCreationDate);
+            }
         }
     }
 
@@ -335,6 +360,18 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return index;
     }
 
+    public Long getIndexCreationDate() {
+        return indexCreationDate;
+    }
+
+    public TimeValue getTimeSinceIndexCreation(Supplier<Long> now) {
+        if (indexCreationDate == null) {
+            return null;
+        } else {
+            return TimeValue.timeValueMillis(Math.max(0L, now.get() - indexCreationDate));
+        }
+    }
+
     public boolean managedByILM() {
         return managedByILM;
     }
@@ -391,11 +428,11 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return failedStepRetryCount;
     }
 
-    public TimeValue getAge() {
+    public TimeValue getAge(Supplier<Long> now) {
         if (lifecycleDate == null) {
             return TimeValue.MINUS_ONE;
         } else {
-            return TimeValue.timeValueMillis(System.currentTimeMillis() - lifecycleDate);
+            return TimeValue.timeValueMillis(Math.max(0L, now.get() - lifecycleDate));
         }
     }
 
@@ -418,9 +455,20 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         builder.field(MANAGED_BY_ILM_FIELD.getPreferredName(), managedByILM);
         if (managedByILM) {
             builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
+            if (indexCreationDate != null) {
+                builder.timeField(
+                    INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(),
+                    INDEX_CREATION_DATE_FIELD.getPreferredName(),
+                    indexCreationDate
+                );
+                builder.field(
+                    TIME_SINCE_INDEX_CREATION_FIELD.getPreferredName(),
+                    getTimeSinceIndexCreation(System::currentTimeMillis).toHumanReadableString(2)
+                );
+            }
             if (lifecycleDate != null) {
                 builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
-                builder.field(AGE_FIELD.getPreferredName(), getAge().toHumanReadableString(2));
+                builder.field(AGE_FIELD.getPreferredName(), getAge(System::currentTimeMillis).toHumanReadableString(2));
             }
             if (phase != null) {
                 builder.field(PHASE_FIELD.getPreferredName(), phase);
@@ -473,6 +521,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     public int hashCode() {
         return Objects.hash(
             index,
+            indexCreationDate,
             managedByILM,
             policyName,
             lifecycleDate,
@@ -503,6 +552,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         }
         IndexLifecycleExplainResponse other = (IndexLifecycleExplainResponse) obj;
         return Objects.equals(index, other.index)
+            && Objects.equals(indexCreationDate, other.indexCreationDate)
             && Objects.equals(managedByILM, other.managedByILM)
             && Objects.equals(policyName, other.policyName)
             && Objects.equals(lifecycleDate, other.lifecycleDate)

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
@@ -27,6 +28,10 @@ import java.util.Objects;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
 public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestCase<IndexLifecycleExplainResponse> {
@@ -47,6 +52,7 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
         boolean stepNull = randomBoolean();
         return IndexLifecycleExplainResponse.newManagedIndexResponse(
             randomAlphaOfLength(10),
+            randomBoolean() ? null : randomLongBetween(0, System.currentTimeMillis()),
             randomAlphaOfLength(10),
             randomBoolean() ? null : randomLongBetween(0, System.currentTimeMillis()),
             stepNull ? null : randomAlphaOfLength(10),
@@ -72,6 +78,7 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
             IllegalArgumentException.class,
             () -> IndexLifecycleExplainResponse.newManagedIndexResponse(
                 randomAlphaOfLength(10),
+                randomNonNegativeLong(),
                 randomAlphaOfLength(10),
                 randomBoolean() ? null : randomNonNegativeLong(),
                 (numNull == 1) ? null : randomAlphaOfLength(10),
@@ -94,6 +101,51 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
         assertThat(exception.getMessage(), containsString("=null"));
     }
 
+    public void testIndexAges() {
+        IndexLifecycleExplainResponse unmanagedExplainResponse = randomUnmanagedIndexExplainResponse();
+        assertThat(unmanagedExplainResponse.getLifecycleDate(), is(nullValue()));
+        assertThat(unmanagedExplainResponse.getAge(System::currentTimeMillis), is(TimeValue.MINUS_ONE));
+
+        assertThat(unmanagedExplainResponse.getIndexCreationDate(), is(nullValue()));
+        assertThat(unmanagedExplainResponse.getTimeSinceIndexCreation(System::currentTimeMillis), is(nullValue()));
+
+        IndexLifecycleExplainResponse managedExplainResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(
+            "indexName",
+            12345L,
+            "policy",
+            5678L,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        assertThat(managedExplainResponse.getLifecycleDate(), is(notNullValue()));
+        Long now = 1_000_000L;
+        assertThat(managedExplainResponse.getAge(() -> now), is(notNullValue()));
+        assertThat(
+            managedExplainResponse.getAge(() -> now),
+            is(equalTo(TimeValue.timeValueMillis(now - managedExplainResponse.getLifecycleDate())))
+        );
+        assertThat(managedExplainResponse.getAge(() -> 0L), is(equalTo(TimeValue.ZERO)));
+        assertThat(managedExplainResponse.getIndexCreationDate(), is(notNullValue()));
+        assertThat(managedExplainResponse.getTimeSinceIndexCreation(() -> now), is(notNullValue()));
+        assertThat(
+            managedExplainResponse.getTimeSinceIndexCreation(() -> now),
+            is(equalTo(TimeValue.timeValueMillis(now - managedExplainResponse.getIndexCreationDate())))
+        );
+        assertThat(managedExplainResponse.getTimeSinceIndexCreation(() -> 0L), is(equalTo(TimeValue.ZERO)));
+    }
+
     @Override
     protected IndexLifecycleExplainResponse createTestInstance() {
         return randomIndexExplainResponse();
@@ -117,6 +169,7 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
     @Override
     protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResponse instance) throws IOException {
         String index = instance.getIndex();
+        Long indexCreationDate = instance.getIndexCreationDate();
         String policy = instance.getPolicyName();
         String phase = instance.getPhase();
         String action = instance.getAction();
@@ -206,6 +259,7 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
             }
             return IndexLifecycleExplainResponse.newManagedIndexResponse(
                 index,
+                indexCreationDate,
                 policy,
                 policyTime,
                 phase,

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java

@@ -209,6 +209,8 @@ public class ExplainLifecycleIT extends ESRestTestCase {
 
     private void assertUnmanagedIndex(Map<String, Object> explainIndexMap) {
         assertThat(explainIndexMap.get("managed"), is(false));
+        assertThat(explainIndexMap.get("time_since_index_creation"), is(nullValue()));
+        assertThat(explainIndexMap.get("index_creation_date_millis"), is(nullValue()));
         assertThat(explainIndexMap.get("policy"), is(nullValue()));
         assertThat(explainIndexMap.get("phase"), is(nullValue()));
         assertThat(explainIndexMap.get("action"), is(nullValue()));
@@ -220,6 +222,8 @@ public class ExplainLifecycleIT extends ESRestTestCase {
 
     private void assertManagedIndex(Map<String, Object> explainIndexMap) {
         assertThat(explainIndexMap.get("managed"), is(true));
+        assertThat(explainIndexMap.get("time_since_index_creation"), is(notNullValue()));
+        assertThat(explainIndexMap.get("index_creation_date_millis"), is(notNullValue()));
         assertThat(explainIndexMap.get("policy"), is(policy));
         assertThat(explainIndexMap.get("phase"), is("new"));
         assertThat(explainIndexMap.get("action"), is("complete"));

x-pack/plugin/ilm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ilm/40_explain_lifecycle.yml

@@ -108,6 +108,8 @@ teardown:
         index: "unmanaged_index"
 
   - match: { indices.unmanaged_index.index: "unmanaged_index" }
+  - is_false: indices.unmanaged_index.index_creation_date_millis
+  - is_false: indices.unmanaged_index.time_since_index_creation
   - is_false: indices.unmanaged_index.managed
   - is_false: indices.unmanaged_index.policy
   - is_false: indices.unmanaged_index.phase

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java

@@ -122,6 +122,7 @@ public class TransportExplainLifecycleAction extends TransportClusterInfoAction<
             stepInfoBytes = new BytesArray(stepInfo);
         }
         String indexName = indexMetadata.getIndex().getName();
+        Long indexCreationDate = indexMetadata.getCreationDate();
 
         // parse existing phase steps from the phase definition in the index settings
         String phaseDef = lifecycleState.getPhaseDefinition();
@@ -146,6 +147,7 @@ public class TransportExplainLifecycleAction extends TransportClusterInfoAction<
                 Long originationDate = idxSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
                 indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(
                     indexName,
+                    indexCreationDate,
                     policyName,
                     originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate(),
                     lifecycleState.getPhase(),