Correctly handling data stream settings when component templates are used (#130394)

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml

@@ -327,8 +327,6 @@ setup:
         index: "my-data-stream-1"
         wait_for_status: green
 
-
-
   - do:
       indices.get_data_stream:
         name: my-data-stream-1
@@ -398,3 +396,96 @@ setup:
   - match: { .$idx0name.settings.index.number_of_shards: "1" }
   - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" }
   - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: "true" }
+
+---
+"Test null out settings component templates only":
+  - requires:
+      cluster_features: [ "logs_stream" ]
+      reason: requires setting 'logs_stream' to get or set data stream settings
+
+  - do:
+      cluster.put_component_template:
+        name: settings-template
+        body:
+          template:
+            settings:
+              lifecycle.name: my-policy
+
+  - do:
+      allowed_warnings:
+        - "index template [my-component-only-template] has index patterns [my-component-only-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-component-only-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-component-only-template
+        body:
+          index_patterns: [ my-component-only-data-stream-* ]
+          data_stream: { }
+          composed_of:
+            - settings-template
+
+  - do:
+      indices.create_data_stream:
+        name: my-component-only-data-stream-1
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.settings: {} }
+  - match: { data_streams.0.effective_settings: null }
+
+  - do:
+      indices.put_data_stream_settings:
+        name: my-component-only-data-stream-1
+        body:
+          index:
+            lifecycle:
+              name: my-new-policy
+              prefer_ilm: true
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
+  - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
+
+  - do:
+      indices.put_data_stream_settings:
+        name: my-component-only-data-stream-1
+        body:
+          index:
+            lifecycle:
+              name: null
+              prefer_ilm: null
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
+  - match: { data_streams.0.settings.index.lifecycle.name: null }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: null }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: null }
+
+  - do:
+      indices.get_data_stream_settings:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.settings.index.lifecycle.name: null }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: null }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: null }
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_settings:
+        index: my-component-only-data-stream-1
+  - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" }
+  - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: null }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java

@@ -480,10 +480,9 @@ public class MetadataDataStreamsService {
         ProjectMetadata projectMetadata = clusterState.metadata().getProject(projectId);
         Map<String, DataStream> dataStreamMap = projectMetadata.dataStreams();
         DataStream dataStream = dataStreamMap.get(dataStreamName);
-        Settings existingSettings = dataStream.getSettings();
+        Settings existingDataStreamSettings = dataStream.getSettings();
 
-        Template.Builder templateBuilder = Template.builder();
-        Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(settingsOverrides);
+        Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingDataStreamSettings).put(settingsOverrides);
         /*
          * A null value for a setting override means that we remove it from the data stream, and let the value from the template (if any)
          * be used.
@@ -493,18 +492,18 @@ public class MetadataDataStreamsService {
                 mergedSettingsBuilder.remove(key);
             }
         });
-        Settings mergedSettings = mergedSettingsBuilder.build();
+        Settings mergedDataStreamSettings = mergedSettingsBuilder.build();
 
         final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, projectMetadata);
-        ComposableIndexTemplate mergedTemplate = template.mergeSettings(mergedSettings);
+        Settings templateSettings = MetadataIndexTemplateService.resolveSettings(template, projectMetadata.componentTemplates());
+        Settings mergedEffectiveSettings = templateSettings.merge(mergedDataStreamSettings);
         MetadataIndexTemplateService.validateTemplate(
-            mergedTemplate.template().settings(),
+            mergedEffectiveSettings,
             dataStream.getEffectiveMappings(projectMetadata),
             indicesService
         );
 
-        templateBuilder.settings(mergedSettingsBuilder);
-        return dataStream.copy().setSettings(mergedSettings).build();
+        return dataStream.copy().setSettings(mergedDataStreamSettings).build();
     }
 
     private DataStream createDataStreamForUpdatedDataStreamMappings(