Add _meta field to ilm policy (#73515)

Relates to #70755.

The main changes of this PR are:

    Add an optional _meta field to ILM policy.
    Add some test code about the change.
    Update the doc of Create or update lifecycle policy API.

docs/reference/ilm/apis/put-lifecycle.asciidoc

@@ -47,13 +47,23 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
 [[ilm-put-lifecycle-example]]
 ==== {api-examples-title}
 
-The following example creates a new policy named `my_policy`:
+The following example creates a new policy named `my_policy`. In addition, you can use the
+`_meta` parameter to add arbitrary metadata to the policy, the `_meta` parameter is optional
+and not automatically generated or used by Elasticsearch. To unset `_meta`, replace the policy
+without specifying one. To check the `_meta`, you can use the <<ilm-get-lifecycle,Get lifecycle policy>> API.
 
 [source,console]
 --------------------------------------------------
 PUT _ilm/policy/my_policy
 {
   "policy": {
+    "_meta": {
+      "description": "used for nginx log",
+      "project": {
+        "name": "myProject",
+        "department": "myDepartment"
+      }
+    },
     "phases": {
       "warm": {
         "min_age": "10d",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java

@@ -6,9 +6,11 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -43,23 +45,27 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
     private static final int MAX_INDEX_NAME_BYTES = 255;
 
     public static final ParseField PHASES_FIELD = new ParseField("phases");
+    private static final ParseField METADATA = new ParseField("_meta");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
             (a, name) -> {
                 List<Phase> phases = (List<Phase>) a[0];
                 Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));
-                return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phaseMap);
+                return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phaseMap, (Map<String, Object>) a[1]);
             });
     static {
         PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> Phase.parse(p, n), v -> {
             throw new IllegalArgumentException("ordered " + PHASES_FIELD.getPreferredName() + " are not supported");
         }, PHASES_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
     }
 
     private final String name;
     private final LifecycleType type;
     private final Map<String, Phase> phases;
+    @Nullable
+    private final Map<String, Object> metadata;
 
     /**
      * @param name
@@ -67,9 +73,23 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
      * @param phases
      *            a {@link Map} of {@link Phase}s which make up this
      *            {@link LifecyclePolicy}.
+     *
      */
     public LifecyclePolicy(String name, Map<String, Phase> phases) {
-        this(TimeseriesLifecycleType.INSTANCE, name, phases);
+        this(TimeseriesLifecycleType.INSTANCE, name, phases, null);
+    }
+
+    /**
+     * @param name
+     *            the name of this {@link LifecyclePolicy}
+     * @param phases
+     *            a {@link Map} of {@link Phase}s which make up this
+     *            {@link LifecyclePolicy}.
+     * @param metadata
+     *            the custom metadata of this {@link LifecyclePolicy}
+     */
+    public LifecyclePolicy(String name, Map<String, Phase> phases, @Nullable Map<String, Object> metadata) {
+        this(TimeseriesLifecycleType.INSTANCE, name, phases, metadata);
     }
 
     /**
@@ -79,6 +99,11 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
         type = in.readNamedWriteable(LifecycleType.class);
         name = in.readString();
         phases = Collections.unmodifiableMap(in.readMap(StreamInput::readString, Phase::new));
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            this.metadata = in.readMap();
+        } else {
+            this.metadata = null;
+        }
     }
 
     /**
@@ -89,11 +114,14 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
      * @param phases
      *            a {@link Map} of {@link Phase}s which make up this
      *            {@link LifecyclePolicy}.
+     * @param metadata
+     *            the custom metadata of this {@link LifecyclePolicy}
      */
-    public LifecyclePolicy(LifecycleType type, String name, Map<String, Phase> phases) {
+    public LifecyclePolicy(LifecycleType type, String name, Map<String, Phase> phases, @Nullable Map<String, Object> metadata) {
         this.name = name;
         this.phases = phases;
         this.type = type;
+        this.metadata = metadata;
         this.type.validate(phases.values());
     }
 
@@ -106,6 +134,9 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
         out.writeNamedWriteable(type);
         out.writeString(name);
         out.writeMap(phases, StreamOutput::writeString, (o, val) -> val.writeTo(o));
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeMap(this.metadata);
+        }
     }
 
     /**
@@ -130,6 +161,13 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
         return phases;
     }
 
+    /**
+     * @return the custom metadata of this {@link LifecyclePolicy}
+     */
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -138,6 +176,9 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
                     builder.field(phase.getName(), phase);
                 }
             builder.endObject();
+        if (this.metadata != null) {
+            builder.field(METADATA.getPreferredName(), this.metadata);
+        }
         builder.endObject();
         return builder;
     }
@@ -264,7 +305,7 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, phases);
+        return Objects.hash(name, phases, metadata);
     }
 
     @Override
@@ -277,7 +318,8 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
         }
         LifecyclePolicy other = (LifecyclePolicy) obj;
         return Objects.equals(name, other.name) &&
-                Objects.equals(phases, other.phases);
+            Objects.equals(phases, other.phases) &&
+            Objects.equals(metadata, other.metadata);
     }
 
     @Override

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java

@@ -23,6 +23,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.ilm.LifecyclePolicyTests.randomMeta;
+
 public class LifecyclePolicyMetadataTests extends AbstractSerializingTestCase<LifecyclePolicyMetadata> {
 
     private String lifecycleName;
@@ -110,7 +112,7 @@ public class LifecyclePolicyMetadataTests extends AbstractSerializingTestCase<Li
         switch (between(0, 3)) {
         case 0:
             policy = new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, policy.getName() + randomAlphaOfLengthBetween(1, 5),
-                    policy.getPhases());
+                    policy.getPhases(), randomMeta());
             break;
         case 1:
             headers = new HashMap<>(headers);

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java

@@ -123,7 +123,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
             }
             phases.put(phase, new Phase(phase, after, actions));
         }
-        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases);
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
     }
 
     public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String lifecycleName) {
@@ -203,7 +203,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
         } else {
             phases.remove(TimeseriesLifecycleType.FROZEN_PHASE);
         }
-        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases);
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
     }
 
     private static Function<String, Set<String>> getPhaseToValidActions() {
@@ -271,7 +271,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
             String phaseName = randomAlphaOfLength(10);
             phases.put(phaseName, new Phase(phaseName, after, actions));
         }
-        return new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        return new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
     }
 
     @Override
@@ -299,7 +299,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
             default:
                 throw new AssertionError("Illegal randomisation branch");
         }
-        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phases);
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phases, randomMeta());
     }
 
     @Override
@@ -311,7 +311,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
         Client client = mock(Client.class);
         lifecycleName = randomAlphaOfLengthBetween(1, 20);
         Map<String, Phase> phases = new LinkedHashMap<>();
-        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
         List<Step> steps = policy.toSteps(client);
         assertThat(steps.size(), equalTo(2));
         assertThat(steps.get(0), instanceOf(InitializePolicyContextStep.class));
@@ -331,7 +331,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
         Map<String, LifecycleAction> actions = Collections.singletonMap(MockAction.NAME, firstAction);
         Phase firstPhase = new Phase("test", TimeValue.ZERO, actions);
         phases.put(firstPhase.getName(), firstPhase);
-        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
         StepKey firstStepKey = InitializePolicyContextStep.KEY;
         StepKey secondStepKey = PhaseCompleteStep.finalStep("new").getKey();
         List<Step> steps = policy.toSteps(client);
@@ -366,7 +366,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
         Phase secondPhase = new Phase("second_phase", TimeValue.ZERO, secondActions);
         phases.put(firstPhase.getName(), firstPhase);
         phases.put(secondPhase.getName(), secondPhase);
-        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
 
         List<Step> steps = policy.toSteps(client);
         assertThat(steps.size(), equalTo(7));
@@ -395,7 +395,7 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
         Phase secondPhase = new Phase("second_phase", TimeValue.ZERO, secondActions);
         phases.put(firstPhase.getName(), firstPhase);
         phases.put(secondPhase.getName(), secondPhase);
-        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta());
 
         assertTrue(policy.isActionSafe(new StepKey("first_phase", MockAction.NAME, randomAlphaOfLength(10))));
 
@@ -428,4 +428,17 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
 
         LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(1, 255));
     }
+
+    public static Map<String, Object> randomMeta() {
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                return Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4));
+            } else {
+                return Collections.singletonMap(randomAlphaOfLength(5),
+                    Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4)));
+            }
+        } else {
+            return null;
+        }
+    }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java

@@ -162,7 +162,7 @@ public class PolicyStepsRegistry {
             if (phaseExecutionInfo.getPhase() != null) {
                 phaseMap.put(currentPhase, phaseExecutionInfo.getPhase());
             }
-            policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap);
+            policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap, currentPolicy.getMetadata());
         }
         LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client,
             ClientHelper.INDEX_LIFECYCLE_ORIGIN, lifecyclePolicyMap.get(policy).getHeaders());

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecyclePolicyTestsUtils.java

@@ -15,6 +15,8 @@ import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
 
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.ilm.LifecyclePolicyTests.randomMeta;
+
 /**
  * This class is here for constructing instances of {@link LifecyclePolicy} that differs from
  * the main {@link TimeseriesLifecycleType} one. Since the more generic constructor is package-private so
@@ -24,11 +26,11 @@ import java.util.Map;
 public class LifecyclePolicyTestsUtils {
 
     public static LifecyclePolicy newTestLifecyclePolicy(String policyName, Map<String, Phase> phases) {
-        return new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName, phases);
+        return new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName, phases, randomMeta());
     }
 
     public static LifecyclePolicy newLockableLifecyclePolicy(String policyName, Map<String, Phase> phases) {
-        return new LifecyclePolicy(LockableLifecycleType.INSTANCE, policyName, phases);
+        return new LifecyclePolicy(LockableLifecycleType.INSTANCE, policyName, phases, randomMeta());
     }
 
     public static LifecyclePolicy randomTimeseriesLifecyclePolicy(String policyName) {