Exclude internal data streams from global retention (#112100)

With #111972 we enable users to set up global retention for data streams that are managed by the data stream lifecycle. This will allow users of elasticsearch to have a more control over their data retention, and consequently better resource management of their clusters.

However, there is a small number of data streams that are necessary for the good operation of elasticsearch and should not follow user defined retention to avoid surprises.

For this reason, we put forth the following definition of internal data streams.

A data stream is internal if it's either a system index (system flag is true) or if its name starts with a dot.

This PR adds the `isInternalDataStream` param in the effective retention calculation making explicit that this is also used to determine the effective retention.

docs/changelog/112100.yaml

@@ -0,0 +1,5 @@
+pr: 112100
+summary: Exclude internal data streams from global retention
+area: Data streams
+type: bug
+issues: []

docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc

@@ -96,6 +96,9 @@ will exceed this time period. This can be set via the <<cluster-update-settings,
 Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is
 calculated as it is described <<effective-retention-calculation,here>>.
 
+NOTE: Global default and max retention do not apply to data streams internal to elastic. Internal data streams are recognised
+ either by having the `system` flag set to `true` or if their name is prefixed with a dot (`.`).
+
 [discrete]
 [[retention-configuration]]
 ==== How to configure retention?

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -819,7 +819,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
                 RolloverRequest rolloverRequest = getDefaultRolloverRequest(
                     rolloverConfiguration,
                     dataStream.getName(),
-                    dataStream.getLifecycle().getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetentionSettings.get()),
+                    dataStream.getLifecycle().getEffectiveDataRetention(globalRetentionSettings.get(), dataStream.isInternal()),
                     rolloverFailureStore
                 );
                 transportActionsDeduplicator.executeOnce(
@@ -879,7 +879,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
         Set<Index> indicesToBeRemoved = new HashSet<>();
         // We know that there is lifecycle and retention because there are indices to be deleted
         assert dataStream.getLifecycle() != null;
-        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention);
+        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention, dataStream.isInternal());
         for (Index index : backingIndicesOlderThanRetention) {
             if (indicesToExcludeForRemainingRun.contains(index) == false) {
                 IndexMetadata backingIndex = metadata.index(index);

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java

@@ -103,7 +103,7 @@ public class TransportExplainDataStreamLifecycleAction extends TransportMasterNo
             ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle(
                 index,
                 true,
-                parentDataStream.isSystem(),
+                parentDataStream.isInternal(),
                 idxMetadata.getCreationDate(),
                 rolloverInfo == null ? null : rolloverInfo.getTime(),
                 generationDate,

server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java

@@ -319,8 +319,7 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
                 }
                 if (dataStream.getLifecycle() != null) {
                     builder.field(LIFECYCLE_FIELD.getPreferredName());
-                    dataStream.getLifecycle()
-                        .toXContent(builder, params, rolloverConfiguration, dataStream.isSystem() ? null : globalRetention);
+                    dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
                 }
                 if (ilmPolicyName != null) {
                     builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName);

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java

@@ -45,7 +45,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
 
     private final String index;
     private final boolean managedByLifecycle;
-    private final boolean isSystemDataStream;
+    private final boolean isInternalDataStream;
     @Nullable
     private final Long indexCreationDate;
     @Nullable
@@ -61,7 +61,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
     public ExplainIndexDataStreamLifecycle(
         String index,
         boolean managedByLifecycle,
-        boolean isSystemDataStream,
+        boolean isInternalDataStream,
         @Nullable Long indexCreationDate,
         @Nullable Long rolloverDate,
         @Nullable TimeValue generationDate,
@@ -70,7 +70,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
     ) {
         this.index = index;
         this.managedByLifecycle = managedByLifecycle;
-        this.isSystemDataStream = isSystemDataStream;
+        this.isInternalDataStream = isInternalDataStream;
         this.indexCreationDate = indexCreationDate;
         this.rolloverDate = rolloverDate;
         this.generationDateMillis = generationDate == null ? null : generationDate.millis();
@@ -82,9 +82,9 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
         this.index = in.readString();
         this.managedByLifecycle = in.readBoolean();
         if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            this.isSystemDataStream = in.readBoolean();
+            this.isInternalDataStream = in.readBoolean();
         } else {
-            this.isSystemDataStream = false;
+            this.isInternalDataStream = false;
         }
         if (managedByLifecycle) {
             this.indexCreationDate = in.readOptionalLong();
@@ -141,7 +141,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
             }
             if (this.lifecycle != null) {
                 builder.field(LIFECYCLE_FIELD.getPreferredName());
-                lifecycle.toXContent(builder, params, rolloverConfiguration, isSystemDataStream ? null : globalRetention);
+                lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternalDataStream);
             }
             if (this.error != null) {
                 if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) {
@@ -161,7 +161,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
         out.writeString(index);
         out.writeBoolean(managedByLifecycle);
         if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            out.writeBoolean(isSystemDataStream);
+            out.writeBoolean(isInternalDataStream);
         }
         if (managedByLifecycle) {
             out.writeOptionalLong(indexCreationDate);

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java

@@ -143,7 +143,7 @@ public class GetDataStreamLifecycleAction {
         public record DataStreamLifecycle(
             String dataStreamName,
             @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle,
-            boolean isSystemDataStream
+            boolean isInternalDataStream
         ) implements Writeable, ToXContentObject {
 
             public static final ParseField NAME_FIELD = new ParseField("name");
@@ -162,7 +162,7 @@ public class GetDataStreamLifecycleAction {
                 out.writeString(dataStreamName);
                 out.writeOptionalWriteable(lifecycle);
                 if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-                    out.writeBoolean(isSystemDataStream);
+                    out.writeBoolean(isInternalDataStream);
                 }
             }
 
@@ -189,7 +189,8 @@ public class GetDataStreamLifecycleAction {
                         builder,
                         org.elasticsearch.cluster.metadata.DataStreamLifecycle.addEffectiveRetentionParams(params),
                         rolloverConfiguration,
-                        isSystemDataStream ? null : globalRetention
+                        globalRetention,
+                        isInternalDataStream
                     );
                 }
                 builder.endObject();

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -277,6 +277,18 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
         return backingIndices.rolloverOnWrite;
     }
 
+    /**
+     * We define that a data stream is considered internal either if it is a system index or if
+     * its name starts with a dot.
+     *
+     * Note: Dot-prefixed internal data streams is a naming convention for internal data streams,
+     * but it's not yet enforced.
+     * @return true if it's a system index or has a dot-prefixed name.
+     */
+    public boolean isInternal() {
+        return isSystem() || name.charAt(0) == '.';
+    }
+
     /**
      * @param timestamp The timestamp used to select a backing index based on its start and end time.
      * @param metadata  The metadata that is used to fetch the start and end times for backing indices of this data stream.
@@ -796,12 +808,12 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
     ) {
         if (lifecycle == null
             || lifecycle.isEnabled() == false
-            || lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention) == null) {
+            || lifecycle.getEffectiveDataRetention(globalRetention, isInternal()) == null) {
             return List.of();
         }
 
         List<Index> indicesPastRetention = getNonWriteIndicesOlderThan(
-            lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention),
+            lifecycle.getEffectiveDataRetention(globalRetention, isInternal()),
             indexMetadataSupplier,
             this::isIndexManagedByDataStreamLifecycle,
             nowSupplier
@@ -1202,7 +1214,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
         }
         if (lifecycle != null) {
             builder.field(LIFECYCLE.getPreferredName());
-            lifecycle.toXContent(builder, params, rolloverConfiguration, isSystem() ? null : globalRetention);
+            lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternal());
         }
         builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), backingIndices.rolloverOnWrite);
         if (backingIndices.autoShardingEvent != null) {

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java

@@ -65,6 +65,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
         DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME,
         "true"
     );
+    public static final Tuple<TimeValue, RetentionSource> INFINITE_RETENTION = Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
 
     /**
      * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that
@@ -145,32 +146,37 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
-    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention) {
-        return getEffectiveDataRetentionWithSource(globalRetention).v1();
+    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention, boolean isInternalDataStream) {
+        return getEffectiveDataRetentionWithSource(globalRetention, isInternalDataStream).v1();
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch.. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return A tuple containing the time period or null as v1 (where null represents that data should never be deleted), and the non-null
      * retention source as v2.
      */
-    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(@Nullable DataStreamGlobalRetention globalRetention) {
+    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
         // If lifecycle is disabled there is no effective retention
         if (enabled == false) {
-            return Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
+            return INFINITE_RETENTION;
         }
         var dataStreamRetention = getDataStreamRetention();
-        if (globalRetention == null) {
+        if (globalRetention == null || isInternalDataStream) {
             return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION);
         }
         if (dataStreamRetention == null) {
@@ -187,7 +193,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
 
     /**
      * The least amount of time data the data stream is requesting es to keep the data.
-     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention)}.
+     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention,boolean)}.
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
@@ -199,12 +205,16 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
      * This method checks if the effective retention is matching what the user has configured; if the effective retention
      * does not match then it adds a warning informing the user about the effective retention and the source.
      */
-    public void addWarningHeaderIfDataRetentionNotEffective(@Nullable DataStreamGlobalRetention globalRetention) {
-        if (globalRetention == null) {
+    public void addWarningHeaderIfDataRetentionNotEffective(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
+        if (globalRetention == null || isInternalDataStream) {
             return;
         }
         Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
-            globalRetention
+            globalRetention,
+            isInternalDataStream
         );
         if (effectiveDataRetentionWithSource.v1() == null) {
             return;
@@ -318,7 +328,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return toXContent(builder, params, null, null);
+        return toXContent(builder, params, null, null, false);
     }
 
     /**
@@ -326,12 +336,14 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
      * and injects the RolloverConditions if they exist.
      * In order to request the effective retention you need to set {@link #INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME} to true
      * in the XContent params.
+     * NOTE: this is used for serialising user output and the result is never deserialised in elasticsearch.
      */
     public XContentBuilder toXContent(
         XContentBuilder builder,
         Params params,
         @Nullable RolloverConfiguration rolloverConfiguration,
-        @Nullable DataStreamGlobalRetention globalRetention
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
     ) throws IOException {
         builder.startObject();
         builder.field(ENABLED_FIELD.getPreferredName(), enabled);
@@ -342,11 +354,14 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
                 builder.field(DATA_RETENTION_FIELD.getPreferredName(), dataRetention.value().getStringRep());
             }
         }
+        Tuple<TimeValue, RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
+            globalRetention,
+            isInternalDataStream
+        );
         if (params.paramAsBoolean(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false)) {
-            Tuple<TimeValue, RetentionSource> effectiveRetention = getEffectiveDataRetentionWithSource(globalRetention);
-            if (effectiveRetention.v1() != null) {
-                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveRetention.v1().getStringRep());
-                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveRetention.v2().displayName());
+            if (effectiveDataRetentionWithSource.v1() != null) {
+                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v1().getStringRep());
+                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v2().displayName());
             }
         }
 
@@ -356,12 +371,18 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
         }
         if (rolloverConfiguration != null) {
             builder.field(ROLLOVER_FIELD.getPreferredName());
-            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention(globalRetention));
+            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1());
         }
         builder.endObject();
         return builder;
     }
 
+    /**
+     * This method deserialises XContent format as it was generated ONLY by {@link DataStreamLifecycle#toXContent(XContentBuilder, Params)}.
+     * It does not support the output of
+     * {@link DataStreamLifecycle#toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention, boolean)} because
+     * this output is enriched with derived fields we do not handle in this deserialisation.
+     */
     public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java

@@ -214,17 +214,15 @@ public class MetadataDataStreamsService {
     ClusterState updateDataLifecycle(ClusterState currentState, List<String> dataStreamNames, @Nullable DataStreamLifecycle lifecycle) {
         Metadata metadata = currentState.metadata();
         Metadata.Builder builder = Metadata.builder(metadata);
-        boolean atLeastOneDataStreamIsNotSystem = false;
+        boolean onlyInternalDataStreams = true;
         for (var dataStreamName : dataStreamNames) {
             var dataStream = validateDataStream(metadata, dataStreamName);
             builder.put(dataStream.copy().setLifecycle(lifecycle).build());
-            atLeastOneDataStreamIsNotSystem = atLeastOneDataStreamIsNotSystem || dataStream.isSystem() == false;
+            onlyInternalDataStreams = onlyInternalDataStreams && dataStream.isInternal();
         }
         if (lifecycle != null) {
-            if (atLeastOneDataStreamIsNotSystem) {
-                // We don't issue any warnings if all data streams are system data streams
-                lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get());
-            }
+            // We don't issue any warnings if all data streams are internal data streams
+            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), onlyInternalDataStreams);
         }
         return ClusterState.builder(currentState).metadata(builder.build()).build();
     }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -369,7 +369,8 @@ public class MetadataIndexTemplateService {
         }
 
         if (finalComponentTemplate.template().lifecycle() != null) {
-            finalComponentTemplate.template().lifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get());
+            // We do not know if this lifecycle will belong to an internal data stream, so we fall back to a non internal.
+            finalComponentTemplate.template().lifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), false);
         }
 
         logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
@@ -815,7 +816,12 @@ public class MetadataIndexTemplateService {
                         + "] specifies lifecycle configuration that can only be used in combination with a data stream"
                 );
             }
-            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+            if (globalRetention != null) {
+                // We cannot know for sure if the template will apply to internal data streams, so we use a simpler heuristic:
+                // If all the index patterns start with a dot, we consider that all the connected data streams are internal.
+                boolean isInternalDataStream = template.indexPatterns().stream().allMatch(indexPattern -> indexPattern.charAt(0) == '.');
+                lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, isInternalDataStream);
+            }
         }
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/Template.java

@@ -254,7 +254,7 @@ public class Template implements SimpleDiffable<Template>, ToXContentObject {
         }
         if (this.lifecycle != null) {
             builder.field(LIFECYCLE.getPreferredName());
-            lifecycle.toXContent(builder, params, rolloverConfiguration, null);
+            lifecycle.toXContent(builder, params, rolloverConfiguration, null, false);
         }
         builder.endObject();
         return builder;

server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java

@@ -88,7 +88,7 @@ public class GetComponentTemplateResponseTests extends AbstractWireSerializingTe
             response.toXContent(builder, EMPTY_PARAMS);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(null))
+            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(null, randomBoolean()))
                 .getConditions()
                 .keySet()) {
                 assertThat(serialized, containsString(label));

server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java

@@ -34,10 +34,10 @@ public class GetDataStreamLifecycleActionTests extends ESTestCase {
         TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
         DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null);
         {
-            boolean isSystemDataStream = true;
+            boolean isInternalDataStream = true;
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
                 lifecycle,
-                isSystemDataStream
+                isInternalDataStream
             );
             Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
@@ -46,10 +46,10 @@ public class GetDataStreamLifecycleActionTests extends ESTestCase {
             assertThat(lifecycleResult.get("retention_determined_by"), equalTo("data_stream_configuration"));
         }
         {
-            boolean isSystemDataStream = false;
+            boolean isInternalDataStream = false;
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
                 lifecycle,
-                isSystemDataStream
+                isInternalDataStream
             );
             Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
@@ -61,9 +61,9 @@ public class GetDataStreamLifecycleActionTests extends ESTestCase {
 
     private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStreamLifecycle(
         DataStreamLifecycle lifecycle,
-        boolean isSystemDataStream
+        boolean isInternalDataStream
     ) {
-        return new GetDataStreamLifecycleAction.Response.DataStreamLifecycle(randomAlphaOfLength(50), lifecycle, isSystemDataStream);
+        return new GetDataStreamLifecycleAction.Response.DataStreamLifecycle(randomAlphaOfLength(50), lifecycle, isInternalDataStream);
     }
 
     /*

server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java

@@ -295,9 +295,9 @@ public class ComponentTemplateTests extends SimpleDiffableSerializationTestCase<
             template.toXContent(builder, withEffectiveRetention, rolloverConfiguration);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, randomBoolean())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             /*

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -243,9 +243,9 @@ public class ComposableIndexTemplateTests extends SimpleDiffableSerializationTes
             template.toXContent(builder, withEffectiveRetention, rolloverConfiguration);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, randomBoolean())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             /*

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java

@@ -116,10 +116,10 @@ public class DataStreamLifecycleTests extends AbstractXContentSerializingTestCas
             RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions();
             DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionTests.randomGlobalRetention();
             ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS);
-            lifecycle.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention);
+            lifecycle.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention, false);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
+            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention, false))
                 .getConditions()
                 .keySet()) {
                 assertThat(serialized, containsString(label));
@@ -282,24 +282,27 @@ public class DataStreamLifecycleTests extends AbstractXContentSerializingTestCas
             TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100));
             TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, 50));
             Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = noRetentionLifecycle
-                .getEffectiveDataRetentionWithSource(null);
+                .getEffectiveDataRetentionWithSource(null, randomBoolean());
             assertThat(effectiveDataRetentionWithSource.v1(), nullValue());
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(null, maxRetention)
+                new DataStreamGlobalRetention(null, maxRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, null)
+                new DataStreamGlobalRetention(defaultRetention, null),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, maxRetention)
+                new DataStreamGlobalRetention(defaultRetention, maxRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
@@ -315,19 +318,21 @@ public class DataStreamLifecycleTests extends AbstractXContentSerializingTestCas
             TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, (int) dataStreamRetention.getDays() - 1));
 
             Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = lifecycleRetention
-                .getEffectiveDataRetentionWithSource(null);
+                .getEffectiveDataRetentionWithSource(null, randomBoolean());
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, null)
+                new DataStreamGlobalRetention(defaultRetention, null),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             TimeValue maxGlobalRetention = randomBoolean() ? dataStreamRetention : TimeValue.timeValueDays(dataStreamRetention.days() + 1);
             effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, maxGlobalRetention)
+                new DataStreamGlobalRetention(defaultRetention, maxGlobalRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
@@ -339,11 +344,44 @@ public class DataStreamLifecycleTests extends AbstractXContentSerializingTestCas
                         ? null
                         : TimeValue.timeValueDays(randomIntBetween(1, (int) (maxRetentionLessThanDataStream.days() - 1))),
                     maxRetentionLessThanDataStream
-                )
+                ),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetentionLessThanDataStream));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
         }
+
+        // Global retention does not apply to internal data streams
+        {
+            // Pick the values in such a way that global retention should have kicked in
+            boolean dataStreamWithRetention = randomBoolean();
+            TimeValue dataStreamRetention = dataStreamWithRetention ? TimeValue.timeValueDays(365) : null;
+            DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
+                TimeValue.timeValueDays(7),
+                TimeValue.timeValueDays(90)
+            );
+            DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(dataStreamRetention).build();
+
+            // Verify that global retention should have kicked in
+            var effectiveDataRetentionWithSource = lifecycle.getEffectiveDataRetentionWithSource(globalRetention, false);
+            if (dataStreamWithRetention) {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(globalRetention.maxRetention()));
+                assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
+            } else {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(globalRetention.defaultRetention()));
+                assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
+            }
+
+            // Now verify that internal data streams do not use global retention
+            // Verify that global retention should have kicked in
+            effectiveDataRetentionWithSource = lifecycle.getEffectiveDataRetentionWithSource(globalRetention, true);
+            if (dataStreamWithRetention) {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
+            } else {
+                assertThat(effectiveDataRetentionWithSource.v1(), nullValue());
+            }
+            assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
+        }
     }
 
     public void testEffectiveRetentionParams() {

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java

@@ -58,7 +58,7 @@ public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
         HeaderWarning.setThreadContext(threadContext);
 
         DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.newBuilder().downsampling(randomDownsampling()).build();
-        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null);
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null, randomBoolean());
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.isEmpty(), is(true));
 
@@ -71,7 +71,7 @@ public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
             TimeValue.timeValueDays(2),
             TimeValue.timeValueDays(dataStreamRetention.days() + randomIntBetween(1, 5))
         );
-        lifecycleWithRetention.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        lifecycleWithRetention.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.isEmpty(), is(true));
     }
@@ -85,7 +85,7 @@ public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
             randomTimeValue(2, 10, TimeUnit.DAYS),
             randomBoolean() ? null : TimeValue.timeValueDays(20)
         );
-        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.size(), is(1));
         assertThat(
@@ -107,7 +107,7 @@ public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
             .downsampling(randomDownsampling())
             .build();
         DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention);
-        lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.size(), is(1));
         String userRetentionPart = lifecycle.getDataStreamRetention() == null
@@ -188,6 +188,24 @@ public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
         );
     }
 
+    public void testValidateInternalDataStreamRetentionWithoutWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+        TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+        MetadataIndexTemplateService.validateLifecycle(
+            Metadata.builder().build(),
+            randomAlphaOfLength(10),
+            ComposableIndexTemplate.builder()
+                .template(new Template(null, null, null, DataStreamLifecycle.DEFAULT))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .indexPatterns(List.of("." + randomAlphaOfLength(10)))
+                .build(),
+            new DataStreamGlobalRetention(defaultRetention, null)
+        );
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(0));
+    }
+
     /**
      * Make sure we still take into account component templates during validation (and not just the index template).
      */

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java

@@ -1893,14 +1893,15 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
             dataStream.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, dataStream.isInternal())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             // We check that even if there was no retention provided by the user, the global retention applies
             assertThat(serialized, not(containsString("data_retention")));
-            if (dataStream.isSystem() == false && (globalRetention.defaultRetention() != null || globalRetention.maxRetention() != null)) {
+            if (dataStream.isInternal() == false
+                && (globalRetention.defaultRetention() != null || globalRetention.maxRetention() != null)) {
                 assertThat(serialized, containsString("effective_retention"));
             } else {
                 assertThat(serialized, not(containsString("effective_retention")));
@@ -2204,6 +2205,17 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
         assertThat(failureStoreDataStreamWithEmptyFailureIndices.isFailureStoreIndex(randomAlphaOfLength(10)), is(false));
     }
 
+    public void testInternalDataStream() {
+        var nonInternalDataStream = createTestInstance().copy().setSystem(false).setName(randomAlphaOfLength(10)).build();
+        assertThat(nonInternalDataStream.isInternal(), is(false));
+
+        var systemDataStream = nonInternalDataStream.copy().setSystem(true).setHidden(true).setName(randomAlphaOfLength(10)).build();
+        assertThat(systemDataStream.isInternal(), is(true));
+
+        var dotPrefixedDataStream = nonInternalDataStream.copy().setSystem(false).setName("." + randomAlphaOfLength(10)).build();
+        assertThat(dotPrefixedDataStream.isInternal(), is(true));
+    }
+
     private record DataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis, Long originationTimeInMillis) {
         public static DataStreamMetadata dataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis) {
             return new DataStreamMetadata(creationTimeInMillis, rolloverTimeInMillis, null);