Remove managing ds by default for now (#100149)

This removes the DSL functionality that would automatically
configure the lifecycle to all new data streams in preparation
for marking Data stream lifecycle as ready for Technical Preview.

docs/changelog/98644.yaml

@@ -1,20 +0,0 @@
-pr: 98644
-summary: GA the data stream lifecycle
-area: Data streams
-type: feature
-issues: []
-highlight:
-  title: The data stream lifecycle is now generally available
-  body: |-
-    This makes the data stream lifecycle generaly available. Data streams will 
-    be able to take advantage of a built-in simplified and resilient
-    lifecycle implementation. Data streams with a configured lifecycle will
-    be automatically rolled over and tail merged (a forcemerge implementation 
-    that's lightweight and only merges the long tail of small segments instead 
-    of the whole shard). With the shard and index maintenance tasks being 
-    handled automatically to ensure optimum performance, and trade-off between
-    indexing and searching, you'll be able to focus on the business related 
-    lifecycle aspects like data retention. The automatic maintenance tasks will
-    automatically be available for any new data stream that's not managed by 
-    ILM.
-  notable: true

docs/reference/data-streams/downsampling-manual.asciidoc

@@ -360,19 +360,16 @@ This returns:
           "index_name": ".ds-my-data-stream-2023.07.26-000001", <1>
           "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
           "prefer_ilm": true,
-          "managed_by": "Data stream lifecycle"
+          "managed_by": "Unmanaged"
         }
       ],
       "generation": 1,
       "status": "GREEN",
-      "next_generation_managed_by": "Data stream lifecycle",
+      "next_generation_managed_by": "Unmanaged",
       "prefer_ilm": true,
       "template": "my-data-stream-template",
       "hidden": false,
       "system": false,
-      "lifecycle": {
-        "enabled": true
-      },
       "allow_custom_routing": false,
       "replicated": false,
       "time_series": {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java

@@ -43,7 +43,7 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
     public void testGetLifecycle() throws Exception {
         DataStreamLifecycle lifecycle = randomLifecycle();
         putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
-        putComposableIndexTemplate("id2", null, List.of("with-default-lifecycle*"), null, null, null);
+        putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
         {
             String dataStreamName = "with-lifecycle-1";
             CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
@@ -55,7 +55,7 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         {
-            String dataStreamName = "with-default-lifecycle";
+            String dataStreamName = "without-lifecycle";
             CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
@@ -68,12 +68,12 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
                 getDataLifecycleRequest
             ).get();
             assertThat(response.getDataStreamLifecycles().size(), equalTo(3));
-            assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-default-lifecycle"));
-            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
-            assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-1"));
+            assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
+            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+            assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
             assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
-            assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("with-lifecycle-2"));
-            assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), equalTo(lifecycle));
+            assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
+            assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
             assertThat(response.getRolloverConfiguration(), nullValue());
         }
 
@@ -90,7 +90,7 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
             assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
             assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
             assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
-            assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
+            assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle));
             assertThat(response.getRolloverConfiguration(), nullValue());
         }
 
@@ -118,12 +118,12 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
             getDataLifecycleRequestWithDefaults
         ).get();
         assertThat(responseWithRollover.getDataStreamLifecycles().size(), equalTo(3));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-default-lifecycle"));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-1"));
+        assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
+        assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+        assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
         assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("with-lifecycle-2"));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), equalTo(lifecycle));
+        assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
+        assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
         assertThat(responseWithRollover.getRolloverConfiguration(), notNullValue());
     }
 
@@ -145,7 +145,7 @@ public class CrudDataStreamLifecycleIT extends ESIntegTestCase {
             assertThat(response.getDataStreamLifecycles().isEmpty(), equalTo(false));
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle = response.getDataStreamLifecycles().get(0);
             assertThat(dataStreamLifecycle.dataStreamName(), is(dataStreamName));
-            assertThat(dataStreamLifecycle.lifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
+            assertThat(dataStreamLifecycle.lifecycle(), is(nullValue()));
         }
 
         // Set lifecycle

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -304,9 +304,6 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea
 
         Settings settings = Settings.builder().put(templateSettings).put(additionalSettings.build()).build();
         DataStreamLifecycle lifecycle = resolveLifecycle(simulatedState.metadata(), matchingTemplate);
-        if (template.getDataStreamTemplate() != null && lifecycle == null) {
-            lifecycle = DataStreamLifecycle.DEFAULT;
-        }
         return new Template(settings, mergedMapping, aliasesByName, lifecycle);
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java

@@ -57,8 +57,6 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
         Setting.Property.NodeScope
     );
 
-    public static final DataStreamLifecycle DEFAULT = new DataStreamLifecycle();
-
     public static final String DATA_STREAM_LIFECYCLE_ORIGIN = "data_stream_lifecycle";
 
     public static final ParseField ENABLED_FIELD = new ParseField("enabled");

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

@@ -277,7 +277,7 @@ public class MetadataCreateDataStreamService {
             isSystem,
             template.getDataStreamTemplate().isAllowCustomRouting(),
             indexMode,
-            lifecycle == null ? DataStreamLifecycle.DEFAULT : lifecycle
+            lifecycle
         );
         Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream);
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java

@@ -67,7 +67,6 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
         assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false));
         assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false));
         assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false));
-        assertThat(newState.metadata().dataStreams().get(dataStreamName).getLifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
         assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue());
         assertThat(
             newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"),

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java

@@ -171,8 +171,7 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
 
         indexDocs(dataStreamName, 2);
 
-        // data stream was rolled over and has 3 indices, two managed by ILM and one will be managed by data stream lifecycle with infinite
-        // retention
+        // data stream was rolled over and has 3 indices, two managed by ILM and the write index will be unmanaged
         assertBusy(() -> {
             GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
@@ -208,16 +207,22 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
             IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(writeIndex);
             assertThat(thirdGenerationExplain.managedByILM(), is(false));
 
+            // the write index is not managed by DSL either
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
                 new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             ExplainIndexDataStreamLifecycle writeIndexDataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0);
-            assertThat(writeIndexDataStreamLifecycleExplain.isManagedByLifecycle(), is(true));
-            assertThat(writeIndexDataStreamLifecycleExplain.getLifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
+            assertThat(writeIndexDataStreamLifecycleExplain.isManagedByLifecycle(), is(false));
         });
 
+        // let's make the write index of the data stream managed by DSL
+        client().execute(
+            PutDataStreamLifecycleAction.INSTANCE,
+            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, new DataStreamLifecycle())
+        ).actionGet();
+
         // at this point we should be able to rollover the data stream by indexing only one document (as the data stream lifecycle is
         // configured to)
         indexDocs(dataStreamName, 1);
@@ -237,7 +242,7 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
             new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, customLifecycle.getEffectiveDataRetention())
-        );
+        ).actionGet();
 
         assertBusy(() -> {
             List<String> backingIndices = getBackingIndices(dataStreamName);
@@ -555,18 +560,22 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
             assertThat(secondGenerationExplain.getPhase(), is("hot"));
             assertThat(secondGenerationExplain.getStep(), is(PhaseCompleteStep.NAME));
 
-            // the write index is managed by the data stream lifecycle
+            // the write index is unmanaged
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
                 new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             ExplainIndexDataStreamLifecycle dataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0);
-            assertThat(dataStreamLifecycleExplain.isManagedByLifecycle(), is(true));
-            assertThat(dataStreamLifecycleExplain.getLifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
-            assertThat(dataStreamLifecycleExplain.getIndex(), is(writeIndex));
+            assertThat(dataStreamLifecycleExplain.isManagedByLifecycle(), is(false));
         });
 
+        // let's make the write index of the data stream managed by DSL
+        client().execute(
+            PutDataStreamLifecycleAction.INSTANCE,
+            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, new DataStreamLifecycle())
+        ).actionGet();
+
         // at this point, the write index of the data stream is managed by data stream lifecycle and not by ILM anymore so we can just index
         // one document to trigger the rollover
         indexDocs(dataStreamName, 1);
@@ -575,7 +584,7 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
             new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, customLifecycle.getEffectiveDataRetention())
-        );
+        ).actionGet();
 
         // data stream was rolled over and has 4 indices, 2 managed by ILM, and 2 managed by the custom data stream lifecycle
         assertBusy(() -> {
@@ -635,7 +644,7 @@ public class DataStreamAndIndexLifecycleMixingTests extends ESIntegTestCase {
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
             new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, TimeValue.timeValueDays(90), false)
-        );
+        ).actionGet();
 
         // At this point the data stream is not managed by the data stream lifecycle and all the indices fallback to ILM.
         assertBusy(() -> {