Expose index age in ILM explain output (#44457)

* Expose index age in ILM explain output

This adds the index's age to the ILM explain output, for example:

```
{
  "indices" : {
    "ilm-000001" : {
      "index" : "ilm-000001",
      "managed" : true,
      "policy" : "full-lifecycle",
      "lifecycle_date" : "2019-07-16T19:48:22.294Z",
      "lifecycle_date_millis" : 1563306502294,
      "age" : "1.34m",
      "phase" : "hot",
      "phase_time" : "2019-07-16T19:48:22.487Z",
      ... etc ...
    }
  }
}
```

This age can be used to tell when ILM will transition the index to the
next phase, based on that phase's `min_age`.

Resolves #38988

* Expose age in getters and in HLRC

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java

@@ -23,6 +23,7 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -52,6 +53,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject {
     private static final ParseField STEP_TIME_FIELD = new ParseField("step_time");
     private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
     private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
+    private static final ParseField AGE_FIELD = new ParseField("age");
 
     public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
         "index_lifecycle_explain_response", true,
@@ -205,6 +207,14 @@ public class IndexLifecycleExplainResponse implements ToXContentObject {
         return phaseExecutionInfo;
     }
 
+    public TimeValue getAge() {
+        if (lifecycleDate == null) {
+            return TimeValue.MINUS_ONE;
+        } else {
+            return TimeValue.timeValueMillis(System.currentTimeMillis() - lifecycleDate);
+        }
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -214,6 +224,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject {
             builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
             if (lifecycleDate != null) {
                 builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
+                builder.field(AGE_FIELD.getPreferredName(), getAge().toHumanReadableString(2));
             }
             if (phase != null) {
                 builder.field(PHASE_FIELD.getPreferredName(), phase);

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponseTests.java

@@ -52,6 +52,11 @@ public class ExplainLifecycleResponseTests extends AbstractXContentTestCase<Expl
         return false;
     }
 
+    @Override
+    protected boolean assertToXContentEquivalence() {
+        return false;
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java

@@ -103,6 +103,11 @@ public class IndexLifecycleExplainResponseTests extends AbstractXContentTestCase
         return true;
     }
 
+    @Override
+    protected boolean assertToXContentEquivalence() {
+        return false;
+    }
+
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
         return (field) ->

docs/reference/ilm/apis/explain.asciidoc

@@ -98,12 +98,13 @@ that the index is managed and in the `new` phase:
       "managed": true, <1>
       "policy": "my_policy", <2>
       "lifecycle_date_millis": 1538475653281, <3>
+      "age": "15s", <4>
       "phase": "new",
-      "phase_time_millis": 1538475653317, <4>
+      "phase_time_millis": 1538475653317, <5>
       "action": "complete",
-      "action_time_millis": 1538475653317, <5>
+      "action_time_millis": 1538475653317, <6>
       "step": "complete",
-      "step_time_millis": 1538475653317 <6>
+      "step_time_millis": 1538475653317 <7>
     }
   }
 }
@@ -114,9 +115,10 @@ that the index is managed and in the `new` phase:
 ILM the other fields will not be shown
 <2> The name of the policy which ILM is using for this index
 <3> The timestamp used for the `min_age`
-<4> When the index entered the current phase
-<5> When the index entered the current action
-<6> When the index entered the current step
+<4> The age of the index (used for calculating when to enter the next phase)
+<5> When the index entered the current phase
+<6> When the index entered the current action
+<7> When the index entered the current step
 
 Once the policy is running on the index, the response includes a
 `phase_execution` object that shows the definition of the current phase.
@@ -133,6 +135,7 @@ phase completes.
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,
       "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "age": "25.14s",
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "phase_time": "2018-10-15T13:45:22.577Z",
@@ -181,6 +184,7 @@ information for the step that's being performed on the index.
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,
       "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "age": "4.12m",
       "phase": "warm",
       "phase_time_millis": 1538475653317,
       "phase_time": "2018-10-15T13:45:22.577Z",
@@ -241,6 +245,7 @@ the step that failed and the step info provides information about the error.
       "policy": "my_lifecycle3",
       "lifecycle_date_millis": 1538475653281,
       "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "age": "50.1d",
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "phase_time": "2018-10-15T13:45:22.577Z",

docs/reference/ilm/error-handling.asciidoc

@@ -76,20 +76,21 @@ Which returns the following information:
       "managed" : true,                         <1>
       "policy" : "shrink-the-index",            <2>
       "lifecycle_date_millis" : 1541717265865,
-      "phase" : "warm",                         <3>
+      "age": "5.1d",                            <3>
+      "phase" : "warm",                         <4>
       "phase_time_millis" : 1541717272601,
-      "action" : "shrink",                      <4>
+      "action" : "shrink",                      <5>
       "action_time_millis" : 1541717272601,
-      "step" : "ERROR",                         <5>
+      "step" : "ERROR",                         <6>
       "step_time_millis" : 1541717272688,
-      "failed_step" : "shrink",                 <6>
+      "failed_step" : "shrink",                 <7>
       "step_info" : {
-        "type" : "illegal_argument_exception",   <7>
-        "reason" : "the number of target shards [4] must be less that the number of source shards [2]" <8>
+        "type" : "illegal_argument_exception",   <8>
+        "reason" : "the number of target shards [4] must be less that the number of source shards [2]" <9>
       },
       "phase_execution" : {
         "policy" : "shrink-the-index",
-        "phase_definition" : {                   <9>
+        "phase_definition" : {                   <10>
           "min_age" : "5d",
           "actions" : {
             "shrink" : {
@@ -108,13 +109,14 @@ Which returns the following information:
 // TESTRESPONSE[skip:no way to know if we will get this response immediately]
 <1> this index is managed by ILM
 <2> the policy in question, in this case, "shrink-the-index"
-<3> what phase the index is currently in
-<4> what action the index is currently on
-<5> what step the index is currently on, in this case, because there is an error, the index is in the "ERROR" step
-<6> the name of the step that failed to execute, in this case "shrink"
-<7> the error class that occurred during this step
-<8> the error message that occurred during the execution failure
-<9> the definition of the phase (in this case, the "warm" phase) that the index is currently on
+<3> the current age for the index
+<4> what phase the index is currently in
+<5> what action the index is currently on
+<6> what step the index is currently on, in this case, because there is an error, the index is in the "ERROR" step
+<7> the name of the step that failed to execute, in this case "shrink"
+<8> the error class that occurred during this step
+<9> the error message that occurred during the execution failure
+<10> the definition of the phase (in this case, the "warm" phase) that the index is currently on
 
 The index here has been moved to the error step because the shrink definition in
 the policy is using an incorrect number of shards. So rectifying that in the

docs/reference/ilm/getting-started-ilm.asciidoc

@@ -173,15 +173,16 @@ managed indices.
       "managed": true,                           <1>
       "policy": "datastream_policy",             <2>
       "lifecycle_date_millis": 1538475653281,
-      "phase": "hot",                            <3>
+      "age": "30s",                              <3>
+      "phase": "hot",                            <4>
       "phase_time_millis": 1538475653317,
-      "action": "rollover",                      <4>
+      "action": "rollover",                      <5>
       "action_time_millis": 1538475653317,
-      "step": "attempt-rollover",                <5>
+      "step": "attempt-rollover",                <6>
       "step_time_millis": 1538475653317,
       "phase_execution": {
         "policy": "datastream_policy",
-        "phase_definition": {                    <6>
+        "phase_definition": {                    <7>
           "min_age": "0ms",
           "actions": {
             "rollover": {
@@ -190,7 +191,7 @@ managed indices.
             }
           }
         },
-        "version": 1,                            <7>
+        "version": 1,                            <8>
         "modified_date_in_millis": 1539609701576
       }
     }
@@ -201,12 +202,13 @@ managed indices.
 // TESTRESPONSE[skip:no way to know if we will get this response immediately]
 <1> this index is managed by ILM
 <2> the policy in question, in this case, "datastream_policy"
-<3> what phase the index is currently in
-<4> what action the index is currently on
-<5> what step the index is currently on
-<6> the definition of the phase
+<3> the current age of the index
+<4> what phase the index is currently in
+<5> what action the index is currently on
+<6> what step the index is currently on
+<7> the definition of the phase
     (in this case, the "hot" phase) that the index is currently on
-<7> the version of the policy being used to execute the current phase
+<8> the version of the policy being used to execute the current phase
 
 You can read about the full details of this response in the
 <<ilm-explain-lifecycle, explain API docs>>. For now, let's focus on how

docs/reference/ilm/update-lifecycle-policy.asciidoc

@@ -198,6 +198,7 @@ GET my_index/_ilm/explain
       "managed": true,
       "policy": "my_executing_policy",
       "lifecycle_date_millis": 1538475653281,
+      "age": "30s",
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "action": "rollover",
@@ -275,6 +276,7 @@ GET my_index/_ilm/explain
       "managed": true,
       "policy": "my_executing_policy",
       "lifecycle_date_millis": 1538475653281,
+      "age": "30s",
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "action": "rollover",
@@ -354,6 +356,7 @@ GET my_index/_ilm/explain
       "managed": true,
       "policy": "my_executing_policy",
       "lifecycle_date_millis": 1538475653281,
+      "age": "30s",
       "phase": "hot",
       "phase_time_millis": 1538475653317,
       "action": "rollover",
@@ -408,6 +411,7 @@ GET my_index/_ilm/explain
       "managed": true,
       "policy": "my_executing_policy",
       "lifecycle_date_millis": 1538475653281,
+      "age": "30s",
       "phase": "warm",
       "phase_time_millis": 1538475653317,
       "action": "forcemerge",

test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java

@@ -43,7 +43,7 @@ public abstract class AbstractStreamableXContentTestCase<T extends ToXContent &
             .shuffleFieldsExceptions(getShuffleFieldsExceptions())
             .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter())
             .assertEqualsConsumer(this::assertEqualInstances)
-            .assertToXContentEquivalence(true)
+            .assertToXContentEquivalence(assertToXContentEquivalence())
             .test();
     }
 
@@ -83,6 +83,16 @@ public abstract class AbstractStreamableXContentTestCase<T extends ToXContent &
         return Strings.EMPTY_ARRAY;
     }
 
+    /**
+     * Whether or not to assert equivalence of the {@link org.elasticsearch.common.xcontent.XContent} of the test instance and the instance
+     * parsed from the {@link org.elasticsearch.common.xcontent.XContent} of the test instance.
+     *
+     * @return true if equivalence should be asserted, otherwise false
+     */
+    protected boolean assertToXContentEquivalence() {
+        return true;
+    }
+
     /**
      * Params that have to be provided when calling calling {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)}
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -42,6 +43,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private static final ParseField STEP_TIME_FIELD = new ParseField("step_time");
     private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
     private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
+    private static final ParseField AGE_FIELD = new ParseField("age");
 
     public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
             "index_lifecycle_explain_response",
@@ -58,7 +60,9 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
                     (Long) (a[9]),
                     (Long) (a[10]),
                     (BytesReference) a[11],
-                    (PhaseExecutionInfo) a[12]));
+                    (PhaseExecutionInfo) a[12]
+                // a[13] == "age"
+            ));
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD);
         PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), MANAGED_BY_ILM_FIELD);
@@ -78,6 +82,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         }, STEP_INFO_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> PhaseExecutionInfo.parse(p, ""),
             PHASE_EXECUTION_INFO);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), AGE_FIELD);
     }
 
     private final String index;
@@ -243,6 +248,14 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return phaseExecutionInfo;
     }
 
+    public TimeValue getAge() {
+        if (lifecycleDate == null) {
+            return TimeValue.MINUS_ONE;
+        } else {
+            return TimeValue.timeValueMillis(System.currentTimeMillis() - lifecycleDate);
+        }
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -252,6 +265,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
             if (lifecycleDate != null) {
                 builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
+                builder.field(AGE_FIELD.getPreferredName(), getAge().toHumanReadableString(2));
             }
             if (phase != null) {
                 builder.field(PHASE_FIELD.getPreferredName(), phase);

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponseTests.java

@@ -56,6 +56,11 @@ public class ExplainLifecycleResponseTests extends AbstractSerializingTestCase<E
         return false;
     }
 
+    @Override
+    protected boolean assertToXContentEquivalence() {
+        return false;
+    }
+
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(Arrays
             .asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new)));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponseTests.java

@@ -93,6 +93,11 @@ public class IndexLifecycleExplainResponseTests extends AbstractSerializingTestC
         return IndexLifecycleExplainResponse.PARSER.apply(parser, null);
     }
 
+    @Override
+    protected boolean assertToXContentEquivalence() {
+        return false;
+    }
+
     @Override
     protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResponse instance) throws IOException {
         String index = instance.getIndex();

x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/40_explain_lifecycle.yml

@@ -104,6 +104,7 @@ teardown:
   - match: { indices.my_index.action: "complete" }
   - match: { indices.my_index.step: "complete" }
   - is_true: indices.my_index.phase_time_millis
+  - is_true: indices.my_index.age
   - is_false: indices.my_index.failed_step
   - is_false: indices.my_index.step_info
   - is_false: indices.my_index.phase_execution
@@ -126,6 +127,7 @@ teardown:
   - match: { indices.my_index.action: "complete" }
   - match: { indices.my_index.step: "complete" }
   - is_true: indices.my_index.phase_time_millis
+  - is_true: indices.my_index.age
   - is_false: indices.my_index.failed_step
   - is_false: indices.my_index.step_info
   - is_false: indices.my_index.phase_execution
@@ -137,6 +139,7 @@ teardown:
   - match: { indices.my_index2.action: "complete" }
   - match: { indices.my_index2.step: "complete" }
   - is_true: indices.my_index2.phase_time_millis
+  - is_true: indices.my_index2.age
   - is_false: indices.my_index2.failed_step
   - is_false: indices.my_index2.step_info
   - is_false: indices.my_index2.phase_execution
@@ -159,6 +162,7 @@ teardown:
   - match: { indices.my_index.action: "complete" }
   - match: { indices.my_index.step: "complete" }
   - is_true: indices.my_index.phase_time_millis
+  - is_true: indices.my_index.age
   - is_false: indices.my_index.failed_step
   - is_false: indices.my_index.step_info
   - is_false: indices.my_index.phase_execution
@@ -170,6 +174,7 @@ teardown:
   - match: { indices.my_index2.action: "complete" }
   - match: { indices.my_index2.step: "complete" }
   - is_true: indices.my_index2.phase_time_millis
+  - is_true: indices.my_index2.age
   - is_false: indices.my_index2.failed_step
   - is_false: indices.my_index2.step_info
   - is_false: indices.my_index2.phase_execution
@@ -181,6 +186,7 @@ teardown:
   - match: { indices.another_index.action: "complete" }
   - match: { indices.another_index.step: "complete" }
   - is_true: indices.another_index.phase_time_millis
+  - is_true: indices.another_index.age
   - is_false: indices.another_index.failed_step
   - is_false: indices.another_index.step_info
   - is_false: indices.another_index.phase_execution
@@ -191,6 +197,7 @@ teardown:
   - is_false: indices.unmanaged_index.phase
   - is_false: indices.unmanaged_index.action
   - is_false: indices.unmanaged_index.step
+  - is_false: indices.unmanaged.age
   - is_false: indices.another_index.failed_step
   - is_false: indices.another_index.step_info
 
@@ -208,6 +215,7 @@ teardown:
   - is_false: indices.unmanaged_index.action
   - is_false: indices.unmanaged_index.step
   - is_false: indices.unmanaged_index.phase_execution
+  - is_false: indices.unmanaged.age
   - is_false: indices.another_index.failed_step
   - is_false: indices.another_index.step_info
   - is_false: indices.my_index