[DSL] Remove REST APIs for global retention (#107565)

docs/changelog/105682.yaml

@@ -1,20 +0,0 @@
-pr: 105682
-summary: Introduce global retention in data stream lifecycle.
-area: Data streams
-type: feature
-issues:
-  - 106169
-highlight:
-  title: Add global retention in data stream lifecycle
-  body: |-
-    Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention 
-    allows us to configure two different retentions:
-    
-    - `default_retention` is applied to all data streams managed by the data stream lifecycle that do not have retention
-    defined on the data stream level.
-    - `max_retention` is applied to all data streams managed by the data stream lifecycle and it allows any data stream 
-    data to be deleted after the `max_retention` has passed.
-
-    Furthermore, we introduce the term `effective_retention` which is the retention applied at a certain moment to a data
-    stream considering all the available retention configurations.
-  notable: true

docs/reference/data-streams/data-stream-apis.asciidoc

@@ -27,12 +27,6 @@ preview:[]
 preview:[]
 * <<data-streams-get-lifecycle-stats,Get data stream lifecycle stats>>
 preview:[]
-* <<data-streams-put-global-retention,Update the global retention for data stream lifecycle managed data streams>>
-preview:[]
-* <<data-streams-get-global-retention,Get the global retention for data stream lifecycle managed data streams>>
-preview:[]
-* <<data-streams-delete-global-retention,Delete the global retention for data stream lifecycle managed data streams>>
-preview:[]
 
 The following API is available for <<tsds,time series data streams>>:
 
@@ -65,10 +59,4 @@ include::{es-ref-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[]
 
 include::{es-ref-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[]
 
-include::{es-ref-dir}/data-streams/lifecycle/apis/put-global-retention.asciidoc[]
-
-include::{es-ref-dir}/data-streams/lifecycle/apis/get-global-retention.asciidoc[]
-
-include::{es-ref-dir}/data-streams/lifecycle/apis/delete-global-retention.asciidoc[]
-
 include::{es-ref-dir}/indices/downsample-data-stream.asciidoc[]

docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc

@@ -1,121 +0,0 @@
-[[data-streams-delete-global-retention]]
-=== Delete the global retention of data streams
-++++
-<titleabbrev>Delete Data Stream Global Retention</titleabbrev>
-++++
-
-preview::[]
-
-Deletes the global retention configuration that applies on every data stream managed by <<data-stream-lifecycle,data stream lifecycle>>.
-
-[[delete-global-retention-api-prereqs]]
-==== {api-prereq-title}
-
-** If the {es} {security-features} are enabled, you must have the `manage_data_stream_global_retention` <<privileges-list-cluster,cluster privilege>> to use this API.
-
-[[data-streams-delete-global-retention-request]]
-==== {api-request-title}
-
-`DELETE _data_stream/_global_retention`
-
-[[data-streams-delete-global-retention-desc]]
-==== {api-description-title}
-
-Deletes the global retention configuration that is applied on data streams managed by data stream lifecycle.
-
-[role="child_attributes"]
-[[delete-global-retention-api-query-parms]]
-==== {api-query-parms-title}
-
-`dry_run`::
-(Boolean) Signals that the request should determine the effect of the removal of the existing without updating
-the global retention. The default value is `false`, which means the removal will happen.
-
-[[delete-global-retention-api-response-body]]
-==== {api-response-body-title}
-
-`acknowledged`::
-(boolean)
-True, if the global retention has been removed. False, if it fails or if it was a dry run.
-
-`dry_run`::
-(boolean)
-True, if this was a dry run, false otherwise.
-
-`affected_data_streams`::
-(array of objects)
-Contains information about the data streams affected by the change.
-+
-.Properties of objects in `affected_data_streams`
-[%collapsible%open]
-====
-`name`::
-(string)
-Name of the data stream.
-`previous_effective_retention`::
-(string)
-The retention that was effective before the change of this request. `infinite` if there was no retention applicable.
-`new_effective_retention`::
-(string)
-The retention that is or would be effective after this request. `infinite` if there is no retention applicable.
-====
-
-[[data-streams-delete-global-retention-example]]
-==== {api-examples-title}
-
-////
-
-[source,console]
---------------------------------------------------
-PUT _data_stream/_global_retention
-{
-  "default_retention": "7d",
-  "max_retention": "90d"
-}
-
-PUT /_index_template/template
-{
-  "index_patterns": ["my-data-stream*"],
-  "template": {
-    "lifecycle": {}
-  },
-  "data_stream": { }
-}
-
-PUT /_data_stream/my-data-stream
-----
-// TESTSETUP
-////
-
-////
-[source,console]
-----
-DELETE /_data_stream/my-data-stream*
-DELETE /_index_template/template
-DELETE /_data_stream/_global_retention
-----
-// TEARDOWN
-////
-
-Let's update the global retention:
-[source,console]
---------------------------------------------------
-DELETE _data_stream/_global_retention
---------------------------------------------------
-
-The response will look like the following:
-
-[source,console-result]
---------------------------------------------------
-{
-  "acknowledged": true,
-  "dry_run": false,
-  "affected_data_streams": [
-    {
-      "name": "my-data-stream",
-      "previous_effective_retention": "7d",
-      "new_effective_retention": "infinite"
-    }
-  ]
-}
---------------------------------------------------

docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc

@@ -1,90 +0,0 @@
-[[data-streams-get-global-retention]]
-=== Get the global retention of data streams
-++++
-<titleabbrev>Get Data Stream Global Retention</titleabbrev>
-++++
-
-preview::[]
-
-Gets the global retention that applies on every data stream managed by <<data-stream-lifecycle,data stream lifecycle>>.
-
-[[get-global-retention-api-prereqs]]
-==== {api-prereq-title}
-
-** If the {es} {security-features} are enabled, you must have the `monitor_data_stream_global_retention` or
-`manage_data_stream_global_retention` <<privileges-list-cluster,cluster privilege>> to use this API.
-
-[[data-streams-get-global-retention-request]]
-==== {api-request-title}
-
-`GET _data_stream/_global_retention`
-
-[[data-streams-get-global-retention-desc]]
-==== {api-description-title}
-
-Gets the global retention configuration that is applied on data streams managed by data stream lifecycle.
-
-[role="child_attributes"]
-[[get-global-retention-api-query-parms]]
-==== {api-query-parms-title}
-
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
-
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
-
-[[get-global-retention-api-response-body]]
-==== {api-response-body-title}
-
-`default_retention`::
-(Optional, string)
-The default retention that will apply to any data stream managed by data stream lifecycle that does not have a retention
-defined on the data stream level.
-
-`max_retention`::
-(Optional, string)
-The max retention that will apply to all data streams managed by data stream lifecycle. The max retention will override the
-retention of a data stream whose retention exceeds the max retention.
-
-
-[[data-streams-get-global-retention-example]]
-==== {api-examples-title}
-
-////
-
-[source,console]
---------------------------------------------------
-PUT _data_stream/_global_retention
-{
-  "default_retention": "7d",
-  "max_retention": "90d"
-}
---------------------------------------------------
-// TESTSETUP
-
-[source,console]
---------------------------------------------------
-DELETE _data_stream/_global_retention
---------------------------------------------------
-// TEARDOWN
-
-////
-
-Let's retrieve the global retention:
-
-[source,console]
---------------------------------------------------
-GET _data_stream/_global_retention
---------------------------------------------------
-
-The response will look like the following:
-
-[source,console-result]
---------------------------------------------------
-{
-  "default_retention": "7d",    <1>
-  "max_retention": "90d"        <2>
-}
---------------------------------------------------
-<1> 7 days retention will be applied to any data stream that does not have retention set in its lifecycle.
-<2> 90 days retention will be applied to all data streams that have retention that exceeds the 90 days, this
-applies to data streams that have infinite retention too.

docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc

@@ -130,18 +130,14 @@ The response will look like the following:
       "name": "my-data-stream-1",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d",
-        "effective_retention": "7d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "7d"
       }
     },
     {
       "name": "my-data-stream-2",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d",
-        "effective_retention": "7d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "7d"
       }
     }
   ]

docs/reference/data-streams/lifecycle/apis/put-global-retention.asciidoc

@@ -1,131 +0,0 @@
-[[data-streams-put-global-retention]]
-=== Update the global retention of data streams
-++++
-<titleabbrev>Update Data Stream Global Retention</titleabbrev>
-++++
-
-preview::[]
-
-Updates the global retention configuration that applies on every data stream managed by <<data-stream-lifecycle,data stream lifecycle>>.
-
-[[put-global-retention-api-prereqs]]
-==== {api-prereq-title}
-
-** If the {es} {security-features} are enabled, you must have the `manage_data_stream_global_retention` <<privileges-list-cluster,cluster privilege>> to use this API.
-
-[[data-streams-put-global-retention-request]]
-==== {api-request-title}
-
-`PUT _data_stream/_global_retention`
-
-[[data-streams-put-global-retention-desc]]
-==== {api-description-title}
-
-Updates the global retention configuration that is applied on data streams managed by data stream lifecycle.
-
-[role="child_attributes"]
-[[put-global-retention-api-query-parms]]
-==== {api-query-parms-title}
-
-`dry_run`::
-(Boolean) Signals that the request should determine the effect of the provided configuration without updating the
-global retention settings. The default value is `false`, which means the configuration provided will be applied.
-
-[[put-global-retention-api-request-body]]
-==== {api-request-body-title}
-
-`default_retention`::
-(Optional, string)
-The default retention that will apply to any data stream managed by data stream lifecycle that does not have a retention
-defined on the data stream level.
-
-`max_retention`::
-(Optional, string)
-The max retention that will apply to all data streams managed by data stream lifecycle. The max retention will override the
-retention of a data stream which retention exceeds the max retention.
-
-[[put-global-retention-api-response-body]]
-==== {api-response-body-title}
-
-`acknowledged`::
-(boolean)
-True, if the global retention has been updated to the provided values. False, if it fails or if it was a dry run.
-
-`dry_run`::
-(boolean)
-True, if this was a dry run, false otherwise.
-
-`affected_data_streams`::
-(array of objects)
-Contains information about the data streams affected by the change.
-+
-.Properties of objects in `affected_data_streams`
-[%collapsible%open]
-====
-`name`::
-(string)
-Name of the data stream.
-`previous_effective_retention`::
-(string)
-The retention that was effective before the change of this request. `infinite` if there was no retention applicable.
-`new_effective_retention`::
-(string)
-The retention that is or would be effective after this request. `infinite` if there is no retention applicable.
-====
-
-[[data-streams-put-global-retention-example]]
-==== {api-examples-title}
-
-////
-[source,console]
-----
-PUT /_index_template/template
-{
-  "index_patterns": ["my-data-stream*"],
-  "template": {
-    "lifecycle": {}
-  },
-  "data_stream": { }
-}
-
-PUT /_data_stream/my-data-stream
-----
-// TESTSETUP
-////
-
-////
-[source,console]
-----
-DELETE /_data_stream/my-data-stream*
-DELETE /_index_template/template
-DELETE /_data_stream/_global_retention
-----
-// TEARDOWN
-////
-
-Let's update the global retention:
-[source,console]
---------------------------------------------------
-PUT _data_stream/_global_retention
-{
-  "default_retention": "7d",
-  "max_retention": "90d"
-}
---------------------------------------------------
-
-The response will look like the following:
-
-[source,console-result]
---------------------------------------------------
-{
-  "acknowledged": true,
-  "dry_run": false,
-  "affected_data_streams": [
-    {
-      "name": "my-data-stream",
-      "previous_effective_retention": "infinite",
-      "new_effective_retention": "7d"
-    }
-  ]
-}
---------------------------------------------------

docs/reference/data-streams/lifecycle/index.asciidoc

@@ -16,8 +16,7 @@ To achieve that, it supports:
 * Automatic <<index-rollover,rollover>>, which chunks your incoming data in smaller pieces to facilitate better performance
 and backwards incompatible mapping changes.
 * Configurable retention, which allows you to configure the time period for which your data is guaranteed to be stored.
-{es} is allowed at a later time to delete data older than this time period. Retention can be configured on the data stream level
-or on a global level. Read more about the different options in this <<tutorial-manage-data-stream-retention,tutorial>>.
+{es} is allowed at a later time to delete data older than this time period.
 
 A data stream lifecycle also supports downsampling the data stream backing indices.
 See <<data-streams-put-lifecycle-downsampling-example, the downsampling example>> for 
@@ -43,10 +42,9 @@ data that is most likely to keep being queried.
 4. If <<data-streams-put-lifecycle-downsampling-example, downsampling>> is configured it will execute 
 all the configured downsampling rounds.
 5. Applies retention to the remaining backing indices. This means deleting the backing indices whose
-`generation_time` is longer than the effective retention period (read more about the
-<<effective-retention-calculation, effective retention calculation>>). The `generation_time` is only applicable to rolled
-over backing indices and it is either the time since the backing index got rolled over, or the time optionally configured
-in the <<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
+`generation_time` is longer than the configured retention period. The `generation_time` is only applicable to rolled over backing
+indices and it is either the time since the backing index got rolled over, or the time optionally configured in the
+<<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
 
 IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing
 index have passed the retention period. As a result, the retention period is not the exact time data gets deleted, but
@@ -79,6 +77,4 @@ include::tutorial-manage-new-data-stream.asciidoc[]
 
 include::tutorial-manage-existing-data-stream.asciidoc[]
 
-include::tutorial-manage-data-stream-retention.asciidoc[]
-
 include::tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc[]

docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc

@@ -1,183 +0,0 @@
-[role="xpack"]
-[[tutorial-manage-data-stream-retention]]
-=== Tutorial: Data stream retention
-
-preview::[]
-
-In this tutorial, we are going to go over the data stream lifecycle retention, define it, go over how it can be configured and how
-it can be applied. Keep in mind, the following options apply only to data streams that are managed by the data stream lifecycle.
-
-. <<what-is-retention>>
-. <<retention-configuration>>
-. <<effective-retention-calculation>>
-. <<effective-retention-application>>
-
-You can verify if a data steam is managed by the data stream lifecycle via the <<data-streams-get-lifecycle,get data stream lifecycle API>>:
-
-////
-[source,console]
-----
-PUT /_index_template/template
-{
-  "index_patterns": ["my-data-stream*"],
-  "template": {
-    "lifecycle": {}
-  },
-  "data_stream": { }
-}
-
-PUT /_data_stream/my-data-stream
-----
-// TESTSETUP
-////
-
-////
-[source,console]
-----
-DELETE /_data_stream/my-data-stream*
-DELETE /_index_template/template
-DELETE /_data_stream/_global_retention
-----
-// TEARDOWN
-////
-
-[source,console]
---------------------------------------------------
-GET _data_stream/my-data-stream/_lifecycle
---------------------------------------------------
-
-The result should look like this:
-
-[source,console-result]
---------------------------------------------------
-{
-  "data_streams": [
-    {
-      "name": "my-data-stream",                                   <1>
-      "lifecycle": {
-        "enabled": true                                           <2>
-      }
-    }
-  ]
-}
---------------------------------------------------
-// TESTRESPONSE[skip:the result is for illustrating purposes only]
-<1> The name of your data stream.
-<2> Ensure that the lifecycle is enabled, meaning this should be `true`.
-
-[discrete]
-[[what-is-retention]]
-==== What is data stream retention?
-
-We define retention as the least amount of time the data of a data stream are going to be kept in {es}. After this time period
-has passed, {es} is allowed to remove these data to free up space and/or manage costs.
-
-NOTE: Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
-
-We define 4 different types of retention:
-
-* The data stream retention, or `data_retention`, which is the retention configured on the data stream level. It can be
-set via an <<index-templates,index template>> for future data streams or via the <<data-streams-put-lifecycle, PUT data
-stream lifecycle API>> for an existing data stream. When the data stream retention is not set, it implies that the data
-need to be kept forever.
-* The global default retention, or `default_retention`, which is a retention configured on a cluster level and will be
-applied to all data streams managed by data stream lifecycle that do not have `data_retention` configured. Effectively,
-it ensures that there will be no data streams keeping their data forever. This can be set via the
-<<data-streams-put-global-retention, PUT global retention API>>.
-* The global max retention, or `max_retention`, which is a retention configured on a cluster level and will be applied to
-all data streams managed by data stream lifecycle. Effectively, it ensures that there will be no data streams whose retention
-will exceed this time period. This can be set via the <<data-streams-put-global-retention, PUT global retention API>>.
-* The effective retention, or `effective_retention`, which is the retention applied at a data stream on a given moment.
-Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is
-calculated as it is described <<effective-retention-calculation,here>>.
-
-[discrete]
-[[retention-configuration]]
-==== How to configure retention?
-
-- By setting the `data_retention` on the data stream level. This retention can be configured in two ways:
-+
--- For new data streams, it can be defined in the index template that would be applied during the data stream's creation.
-You can use the <<indices-put-template,create index template API>>, for example:
-+
-[source,console]
---------------------------------------------------
-PUT _index_template/template
-{
-  "index_patterns": ["my-data-stream*"],
-  "data_stream": { },
-  "priority": 500,
-  "template": {
-    "lifecycle": {
-      "data_retention": "7d"
-    }
-  },
-  "_meta": {
-    "description": "Template with data stream lifecycle"
-  }
-}
---------------------------------------------------
--- For an existing data stream, it can be set via the <<data-streams-put-lifecycle, PUT lifecycle API>>.
-+
-[source,console]
-----
-PUT _data_stream/my-data-stream/_lifecycle
-{
-  "data_retention": "30d" <1>
-}
-----
-// TEST[continued]
-<1> The retention period of this data stream is set to 30 days.
-
-- By setting the global retention via the `default_retention` and `max_retention` that are set on a cluster level. You
-can set them via the <<data-streams-put-global-retention, PUT global retention API>>. For example:
-+
-[source,console]
---------------------------------------------------
-PUT _data_stream/_global_retention
-{
-  "default_retention": "7d",
-  "max_retention": "90d"
-}
---------------------------------------------------
-// TEST[continued]
-
-[discrete]
-[[effective-retention-calculation]]
-==== How is the effective retention calculated?
-The effective is calculated in the following way:
-
-- The `effective_retention` is the `default_retention`, when `default_retention` is defined and the data stream does not
-have `data_retention`.
-- The `effective_retention` is the `data_retention`, when `data_retention` is defined and if `max_retention` is defined,
-it is less than the `max_retention`.
-- The `effective_retention` is the `max_retention`, when `max_retention` is defined, and the data stream has either no
-`data_retention` or its `data_retention` is greater than the `max_retention`.
-
-The above is demonstrated in the examples below:
-
-|===
-|`default_retention`    |`max_retention`    |`data_retention`   |`effective_retention`  |Retention determined by
-
-|Not set                |Not set            |Not set            |Infinite               |N/A
-|Not relevant           |12 months          |**30 days**        |30 days                |`data_retention`
-|Not relevant           |Not set            |**30 days**        |30 days                |`data_retention`
-|**30 days**            |12 months          |Not set            |30 days                |`default_retention`
-|**30 days**            |30 days            |Not set            |30 days                |`default_retention`
-|Not relevant           |**30 days**        |12 months          |30 days                |`max_retention`
-|Not set                |**30 days**        |Not set            |30 days                |`max_retention`
-|===
-
-[discrete]
-[[effective-retention-application]]
-==== How is the effective retention applied?
-
-Retention is applied to the remaining backing indices of a data stream as the last step of
-<<data-streams-lifecycle-how-it-works, a data stream lifecycle run>>. Data stream lifecycle will retrieve the backing indices
-whose `generation_time` is longer than the effective retention period and delete them. The `generation_time` is only
-applicable to rolled over backing indices and it is either the time since the backing index got rolled over, or the time
-optionally configured in the <<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
-
-IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing
-index have passed the retention period. As a result, the retention period is not the exact time data get deleted, but
-the minimum time data will be stored.

docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc

@@ -74,9 +74,7 @@ The response will look like:
       "generation_time": "6.84s",                       <9>
       "lifecycle": {
         "enabled": true,
-        "data_retention": "30d",
-        "effective_retention": "30d"                    <10>
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "30d"                         <10>
       }
     }
   }

docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc

@@ -93,12 +93,10 @@ The result will look like this:
 {
   "data_streams": [
     {
-      "name": "my-data-stream",                                   <1>
+      "name": "my-data-stream",<1>
       "lifecycle": {
-        "enabled": true,                                          <2>
-        "data_retention": "7d",                                   <3>
-        "effective_retention": "7d",                              <4>
-        "retention_determined_by": "data_stream_configuration"    <5>
+        "enabled": true,       <2>
+        "data_retention": "7d" <3>
       }
     }
   ]
@@ -106,11 +104,8 @@ The result will look like this:
 --------------------------------------------------
 <1> The name of your data stream.
 <2> Shows if the data stream lifecycle is enabled for this data stream.
-<3> The desired retention period of the data indexed in this data stream, this means that if there are no other limitations
-the data for this data stream will be preserved for at least 7 days.
-<4> The effective retention, this means that the data in this data stream will
+<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will
 be kept at least for 7 days. After that {es} can delete it at its own discretion.
-<5> The configuration that determined the effective retention.
 
 If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the
 <<data-streams-explain-lifecycle,explain data stream lifecycle API>>:
@@ -133,9 +128,7 @@ The result will look like this:
       "time_since_index_creation": "1.6m",                  <3>
       "lifecycle": {                                        <4>
         "enabled": true,
-        "data_retention": "7d",
-        "effective_retention": "7d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "7d"
       }
     }
   }

docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc

@@ -200,10 +200,10 @@ PUT _index_template/dsl-data-stream-template
   "template": {
     "settings": {
       "index.lifecycle.name": "pre-dsl-ilm-policy",
-      "index.lifecycle.prefer_ilm": false                       <1>
+      "index.lifecycle.prefer_ilm": false             <1>
     },
-    "lifecycle": {                                              <2>
-      "data_retention": "7d"                                    <3>
+    "lifecycle": {
+      "data_retention": "7d"                          <2>
     }
   }
 }
@@ -215,8 +215,6 @@ PUT _index_template/dsl-data-stream-template
 precedence over data stream lifecycle.
 <2> We're configuring the data stream lifecycle so _new_ data streams will be 
 managed by data stream lifecycle.
-<3> The desired retention, meaning that this data stream should keep the data for at least 7 days,
-if this retention is possible.
 
 We've now made sure that new data streams will be managed by data stream lifecycle.
 
@@ -270,9 +268,7 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d",
-        "effective_retention": "7d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "7d"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",         <3>
@@ -350,9 +346,7 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d",
-        "effective_retention": "7d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "7d"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java

@@ -1,147 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.datastreams.lifecycle;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DataStreamGlobalRetentionIT extends DisabledSecurityDataStreamTestCase {
-
-    @Before
-    public void setup() throws IOException {
-        updateClusterSettings(
-            Settings.builder()
-                .put("data_streams.lifecycle.poll_interval", "1s")
-                .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1")
-                .build()
-        );
-        // Create a template with the default lifecycle
-        Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
-        putComposableIndexTemplateRequest.setJsonEntity("""
-            {
-              "index_patterns": ["my-data-stream*"],
-              "data_stream": {},
-              "template": {
-                "lifecycle": {}
-              }
-            }
-            """);
-        assertOK(client().performRequest(putComposableIndexTemplateRequest));
-
-        // Create a data streams with one doc
-        Request createDocRequest = new Request("POST", "/my-data-stream/_doc?refresh=true");
-        createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}");
-        assertOK(client().performRequest(createDocRequest));
-    }
-
-    @After
-    public void cleanUp() throws IOException {
-        adminClient().performRequest(new Request("DELETE", "_data_stream/*"));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testDefaultRetention() throws Exception {
-        {
-            // Set global retention
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "10s"
-                }""");
-            assertAcknowledged(client().performRequest(request));
-        }
-
-        // Verify that the effective retention matches the default retention
-        {
-            Request request = new Request("GET", "/_data_stream/my-data-stream");
-            Response response = client().performRequest(request);
-            List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
-            assertThat(dataStreams.size(), is(1));
-            Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
-            assertThat(dataStream.get("name"), is("my-data-stream"));
-            Map<String, Object> lifecycle = (Map<String, Object>) dataStream.get("lifecycle");
-            assertThat(lifecycle.get("effective_retention"), is("10s"));
-            assertThat(lifecycle.get("retention_determined_by"), is("default_global_retention"));
-            assertThat(lifecycle.get("data_retention"), nullValue());
-        }
-
-        // Verify that the first generation index was removed
-        assertBusy(() -> {
-            Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream"));
-            Map<String, Object> dataStream = ((List<Map<String, Object>>) entityAsMap(response).get("data_streams")).get(0);
-            assertThat(dataStream.get("name"), is("my-data-stream"));
-            List<Object> backingIndices = (List<Object>) dataStream.get("indices");
-            assertThat(backingIndices.size(), is(1));
-            // 2 backing indices created + 1 for the deleted index
-            assertThat(dataStream.get("generation"), is(3));
-        }, 20, TimeUnit.SECONDS);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testMaxRetention() throws Exception {
-        {
-            // Set global retention
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "max_retention": "10s"
-                }""");
-            assertAcknowledged(client().performRequest(request));
-        }
-        boolean withDataStreamLevelRetention = randomBoolean();
-        if (withDataStreamLevelRetention) {
-            Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle");
-            request.setJsonEntity("""
-                {
-                  "data_retention": "30d"
-                }""");
-            assertAcknowledged(client().performRequest(request));
-        }
-
-        // Verify that the effective retention matches the max retention
-        {
-            Request request = new Request("GET", "/_data_stream/my-data-stream");
-            Response response = client().performRequest(request);
-            List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
-            assertThat(dataStreams.size(), is(1));
-            Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
-            assertThat(dataStream.get("name"), is("my-data-stream"));
-            Map<String, Object> lifecycle = (Map<String, Object>) dataStream.get("lifecycle");
-            assertThat(lifecycle.get("effective_retention"), is("10s"));
-            assertThat(lifecycle.get("retention_determined_by"), is("max_global_retention"));
-            if (withDataStreamLevelRetention) {
-                assertThat(lifecycle.get("data_retention"), is("30d"));
-            } else {
-                assertThat(lifecycle.get("data_retention"), nullValue());
-            }
-        }
-
-        // Verify that the first generation index was removed
-        assertBusy(() -> {
-            Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream"));
-            Map<String, Object> dataStream = ((List<Map<String, Object>>) entityAsMap(response).get("data_streams")).get(0);
-            assertThat(dataStream.get("name"), is("my-data-stream"));
-            List<Object> backingIndices = (List<Object>) dataStream.get("indices");
-            assertThat(backingIndices.size(), is(1));
-            // 2 backing indices created + 1 for the deleted index
-            assertThat(dataStream.get("generation"), is(3));
-        }, 20, TimeUnit.SECONDS);
-    }
-}

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionPermissionsRestIT.java

@@ -1,213 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle;
-
-import org.apache.http.HttpHost;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.FeatureFlag;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.cluster.util.resource.Resource;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.ClassRule;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-public class DataStreamGlobalRetentionPermissionsRestIT extends ESRestTestCase {
-
-    private static final String PASSWORD = "secret-test-password";
-
-    @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .feature(FeatureFlag.FAILURE_STORE_ENABLED)
-        .setting("xpack.watcher.enabled", "false")
-        .setting("xpack.ml.enabled", "false")
-        .setting("xpack.security.enabled", "true")
-        .setting("xpack.security.transport.ssl.enabled", "false")
-        .setting("xpack.security.http.ssl.enabled", "false")
-        .user("test_admin", PASSWORD, "superuser", false)
-        .user("test_manage_global_retention", PASSWORD, "manage_data_stream_global_retention", false)
-        .user("test_monitor_global_retention", PASSWORD, "monitor_data_stream_global_retention", false)
-        .user("test_monitor", PASSWORD, "manage_data_stream_lifecycle", false)
-        .user("test_no_privilege", PASSWORD, "no_privilege", false)
-        .rolesFile(Resource.fromClasspath("roles.yml"))
-        .build();
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        // If this test is running in a test framework that handles its own authorization, we don't want to overwrite it.
-        if (super.restClientSettings().keySet().contains(ThreadContext.PREFIX + ".Authorization")) {
-            return super.restClientSettings();
-        } else {
-            // Note: This user is assigned the role "manage_data_stream_lifecycle". That role is defined in roles.yml.
-            String token = basicAuthHeaderValue("test_data_stream_lifecycle", new SecureString(PASSWORD.toCharArray()));
-            return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-        }
-    }
-
-    @Override
-    protected Settings restAdminSettings() {
-        // If this test is running in a test framework that handles its own authorization, we don't want to overwrite it.
-        if (super.restClientSettings().keySet().contains(ThreadContext.PREFIX + ".Authorization")) {
-            return super.restClientSettings();
-        } else {
-            // Note: We use the admin user because the other one is too unprivileged, so it breaks the initialization of the test
-            String token = basicAuthHeaderValue("test_admin", new SecureString(PASSWORD.toCharArray()));
-            return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-        }
-    }
-
-    private Settings restManageGlobalRetentionClientSettings() {
-        String token = basicAuthHeaderValue("test_manage_global_retention", new SecureString(PASSWORD.toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    private Settings restMonitorGlobalRetentionClientSettings() {
-        String token = basicAuthHeaderValue("test_monitor_global_retention", new SecureString(PASSWORD.toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    private Settings restOnlyManageLifecycleClientSettings() {
-        String token = basicAuthHeaderValue("test_monitor", new SecureString(PASSWORD.toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    private Settings restNoPrivilegeClientSettings() {
-        String token = basicAuthHeaderValue("test_no_privilege", new SecureString(PASSWORD.toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    public void testManageGlobalRetentionPrivileges() throws Exception {
-        try (var client = buildClient(restManageGlobalRetentionClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) {
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "1d",
-                  "max_retention": "7d"
-                }""");
-            assertAcknowledged(client.performRequest(request));
-            Map<String, Object> response = entityAsMap(client.performRequest(new Request("GET", "/_data_stream/_global_retention")));
-            assertThat(response.get("default_retention"), equalTo("1d"));
-            assertThat(response.get("max_retention"), equalTo("7d"));
-            assertAcknowledged(client.performRequest(new Request("DELETE", "/_data_stream/_global_retention")));
-        }
-    }
-
-    public void testMonitorGlobalRetentionPrivileges() throws Exception {
-        {
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "1d",
-                  "max_retention": "7d"
-                }""");
-            assertAcknowledged(adminClient().performRequest(request));
-        }
-        try (var client = buildClient(restMonitorGlobalRetentionClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) {
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "1d",
-                  "max_retention": "7d"
-                }""");
-            ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request));
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString(
-                    "action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_monitor_global_retention]"
-                )
-            );
-            responseException = expectThrows(
-                ResponseException.class,
-                () -> client.performRequest(new Request("DELETE", "/_data_stream/_global_retention"))
-            );
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString(
-                    "action [cluster:admin/data_stream/global_retention/delete] is unauthorized for user [test_monitor_global_retention]"
-                )
-            );
-            Map<String, Object> response = entityAsMap(client.performRequest(new Request("GET", "/_data_stream/_global_retention")));
-            assertThat(response.get("default_retention"), equalTo("1d"));
-            assertThat(response.get("max_retention"), equalTo("7d"));
-        }
-    }
-
-    public void testManageLifecyclePrivileges() throws Exception {
-        try (var client = buildClient(restOnlyManageLifecycleClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) {
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "1d",
-                  "max_retention": "7d"
-                }""");
-            ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request));
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString("action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_monitor]")
-            );
-            // This use has the monitor privilege which includes the monitor_data_stream_global_retention
-            Response response = client.performRequest(new Request("GET", "/_data_stream/_global_retention"));
-            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        }
-    }
-
-    public void testNoPrivileges() throws Exception {
-        try (var client = buildClient(restNoPrivilegeClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) {
-            Request request = new Request("PUT", "_data_stream/_global_retention");
-            request.setJsonEntity("""
-                {
-                  "default_retention": "1d",
-                  "max_retention": "7d"
-                }""");
-            ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request));
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString("action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_no_privilege]")
-            );
-            responseException = expectThrows(
-                ResponseException.class,
-                () -> client.performRequest(new Request("DELETE", "/_data_stream/_global_retention"))
-            );
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString("action [cluster:admin/data_stream/global_retention/delete] is unauthorized for user [test_no_privilege]")
-            );
-            responseException = expectThrows(
-                ResponseException.class,
-                () -> client.performRequest(new Request("GET", "/_data_stream/_global_retention"))
-            );
-            assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403));
-            assertThat(
-                responseException.getMessage(),
-                containsString("action [cluster:monitor/data_stream/global_retention/get] is unauthorized for user [test_no_privilege]")
-            );
-        }
-    }
-}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java

@@ -58,12 +58,9 @@ import org.elasticsearch.datastreams.lifecycle.action.TransportPutDataStreamLife
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService;
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher;
 import org.elasticsearch.datastreams.lifecycle.rest.RestDataStreamLifecycleStatsAction;
-import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction;
-import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction;
-import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction;
@@ -290,9 +287,6 @@ public class DataStreamsPlugin extends Plugin implements ActionPlugin, HealthPlu
         handlers.add(new RestDeleteDataStreamLifecycleAction());
         handlers.add(new RestExplainDataStreamLifecycleAction());
         handlers.add(new RestDataStreamLifecycleStatsAction());
-        handlers.add(new RestPutDataStreamGlobalRetentionAction());
-        handlers.add(new RestGetDataStreamGlobalRetentionAction());
-        handlers.add(new RestDeleteDataStreamGlobalRetentionAction());
         return handlers;
     }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java

@@ -18,7 +18,6 @@ import org.elasticsearch.action.datastreams.lifecycle.ExplainIndexDataStreamLife
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -216,11 +215,7 @@ public class ExplainDataStreamLifecycleAction {
                 return builder;
             }), Iterators.map(indices.iterator(), explainIndexDataLifecycle -> (builder, params) -> {
                 builder.field(explainIndexDataLifecycle.getIndex());
-                ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams(
-                    DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS,
-                    params
-                );
-                explainIndexDataLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+                explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
                 return builder;
             }), Iterators.single((builder, params) -> {
                 builder.endObject();

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java

@@ -237,17 +237,22 @@ public class GetDataStreamLifecycleAction {
                 builder.startObject();
                 builder.startArray(DATA_STREAMS_FIELD.getPreferredName());
                 return builder;
-            }), Iterators.map(dataStreamLifecycles.iterator(), dataStreamLifecycle -> (builder, params) -> {
-                ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams(
-                    org.elasticsearch.cluster.metadata.DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS,
-                    params
-                );
-                return dataStreamLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
-            }), Iterators.single((builder, params) -> {
-                builder.endArray();
-                builder.endObject();
-                return builder;
-            }));
+            }),
+                Iterators.map(
+                    dataStreamLifecycles.iterator(),
+                    dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent(
+                        builder,
+                        params,
+                        rolloverConfiguration,
+                        globalRetention
+                    )
+                ),
+                Iterators.single((builder, params) -> {
+                    builder.endArray();
+                    builder.endObject();
+                    return builder;
+                })
+            );
         }
 
         @Override

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamGlobalRetentionAction.java

@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.datastreams.lifecycle.rest;
-
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.Scope;
-import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.RestChunkedToXContentListener;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.DELETE;
-
-/**
- * Removes the data stream global retention configuration
- */
-@ServerlessScope(Scope.PUBLIC)
-public class RestDeleteDataStreamGlobalRetentionAction extends BaseRestHandler {
-
-    @Override
-    public String getName() {
-        return "delete_data_stream_global_retention_action";
-    }
-
-    @Override
-    public List<Route> routes() {
-        return List.of(new Route(DELETE, "/_data_stream/_global_retention"));
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        DeleteDataStreamGlobalRetentionAction.Request request = new DeleteDataStreamGlobalRetentionAction.Request();
-        request.dryRun(restRequest.paramAsBoolean("dry_run", false));
-        return channel -> client.execute(
-            DeleteDataStreamGlobalRetentionAction.INSTANCE,
-            request,
-            new RestChunkedToXContentListener<>(channel)
-        );
-    }
-}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamGlobalRetentionAction.java

@@ -1,47 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.datastreams.lifecycle.rest;
-
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamGlobalRetentionAction;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.Scope;
-import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.RestToXContentListener;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-
-/**
- * Retrieves the data stream global retention configuration.
- */
-@ServerlessScope(Scope.PUBLIC)
-public class RestGetDataStreamGlobalRetentionAction extends BaseRestHandler {
-
-    @Override
-    public String getName() {
-        return "get_data_stream_global_retention_action";
-    }
-
-    @Override
-    public List<Route> routes() {
-        return List.of(new Route(GET, "/_data_stream/_global_retention"));
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        GetDataStreamGlobalRetentionAction.Request request = new GetDataStreamGlobalRetentionAction.Request();
-        request.local(restRequest.paramAsBoolean("local", request.local()));
-        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
-
-        return channel -> client.execute(GetDataStreamGlobalRetentionAction.INSTANCE, request, new RestToXContentListener<>(channel));
-    }
-}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamGlobalRetentionAction.java

@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.datastreams.lifecycle.rest;
-
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.Scope;
-import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.RestChunkedToXContentListener;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.PUT;
-
-/**
- * Updates the default_retention and the max_retention of the data stream global retention configuration. It
- * does not accept an empty payload.
- */
-@ServerlessScope(Scope.PUBLIC)
-public class RestPutDataStreamGlobalRetentionAction extends BaseRestHandler {
-
-    @Override
-    public String getName() {
-        return "put_data_stream_global_retention_action";
-    }
-
-    @Override
-    public List<Route> routes() {
-        return List.of(new Route(PUT, "/_data_stream/_global_retention"));
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        try (XContentParser parser = restRequest.contentParser()) {
-            PutDataStreamGlobalRetentionAction.Request request = PutDataStreamGlobalRetentionAction.Request.parseRequest(parser);
-            request.dryRun(restRequest.paramAsBoolean("dry_run", false));
-            return channel -> client.execute(
-                PutDataStreamGlobalRetentionAction.INSTANCE,
-                request,
-                new RestChunkedToXContentListener<>(channel)
-            );
-        }
-    }
-}

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java

@@ -37,7 +37,6 @@ import java.util.Map;
 
 import static org.elasticsearch.datastreams.lifecycle.action.ExplainDataStreamLifecycleAction.Response;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -196,23 +195,6 @@ public class ExplainDataStreamLifecycleResponseTests extends AbstractWireSeriali
                 Map<String, Object> lifecycleMap = (Map<String, Object>) explainIndexMap.get("lifecycle");
                 assertThat(lifecycleMap.get("data_retention"), nullValue());
 
-                if (response.getGlobalRetention() == null) {
-                    assertThat(lifecycleMap.get("effective_retention"), nullValue());
-                    assertThat(lifecycleMap.get("retention_determined_by"), nullValue());
-                } else if (response.getGlobalRetention().getDefaultRetention() != null) {
-                    assertThat(
-                        lifecycleMap.get("effective_retention"),
-                        equalTo(response.getGlobalRetention().getDefaultRetention().getStringRep())
-                    );
-                    assertThat(lifecycleMap.get("retention_determined_by"), equalTo("default_global_retention"));
-                } else {
-                    assertThat(
-                        lifecycleMap.get("effective_retention"),
-                        equalTo(response.getGlobalRetention().getMaxRetention().getStringRep())
-                    );
-                    assertThat(lifecycleMap.get("retention_determined_by"), equalTo("max_global_retention"));
-                }
-
                 Map<String, Object> lifecycleRollover = (Map<String, Object>) lifecycleMap.get("rollover");
                 assertThat(lifecycleRollover.get("min_primary_shard_docs"), is(4));
                 assertThat(lifecycleRollover.get("max_primary_shard_docs"), is(9));

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml

@@ -1,9 +1,9 @@
 ---
 "Explain backing index lifecycle":
-  - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
-      features: allowed_warnings
+  - requires:
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Data stream lifecycle was released as tech preview in 8.11"
+      test_runner_features: allowed_warnings
   - do:
       allowed_warnings:
         - "index template [template-with-lifecycle] has index patterns [managed-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [template-with-lifecycle] will take precedence during new index creation"
@@ -36,9 +36,7 @@
       indices.explain_data_lifecycle:
         index: $backing_index
   - match: { indices.$backing_index.managed_by_lifecycle: true }
-  - match: { indices.$backing_index.lifecycle.data_retention: "30d" }
-  - match: { indices.$backing_index.lifecycle.effective_retention: "30d"}
-  - match: { indices.$backing_index.lifecycle.retention_determined_by: "data_stream_configuration"}
+  - match: { indices.$backing_index.lifecycle.data_retention: '30d' }
   - match: { indices.$backing_index.lifecycle.enabled: true }
   - is_false: indices.$backing_index.lifecycle.rollover
 
@@ -48,9 +46,7 @@
         index: $backing_index
         include_defaults: true
   - match: { indices.$backing_index.managed_by_lifecycle: true }
-  - match: { indices.$backing_index.lifecycle.data_retention: "30d" }
-  - match: { indices.$backing_index.lifecycle.effective_retention: "30d"}
-  - match: { indices.$backing_index.lifecycle.retention_determined_by: "data_stream_configuration"}
+  - match: { indices.$backing_index.lifecycle.data_retention: '30d' }
   - is_true: indices.$backing_index.lifecycle.rollover
 
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml

@@ -1,8 +1,8 @@
 ---
 "Create data stream with lifecycle":
   - requires:
-      cluster_features: ["gte_v8.14.0"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Data stream lifecycle was GA in 8.11"
       test_runner_features: allowed_warnings
   - do:
       allowed_warnings:
@@ -35,7 +35,5 @@
   - match: { data_streams.0.template: 'template-with-lifecycle' }
   - match: { data_streams.0.hidden: false }
   - match: { data_streams.0.lifecycle.data_retention: '30d' }
-  - match: { data_streams.0.lifecycle.effective_retention: '30d'}
-  - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration'}
   - match: { data_streams.0.lifecycle.enabled: true }
   - is_true: data_streams.0.lifecycle.rollover

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml

@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: allowed_warnings
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycles with global retention are only supported in 8.14+"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Data stream lifecycles only supported in 8.11+"
   - do:
       allowed_warnings:
         - "index template [my-lifecycle] has index patterns [data-stream-with-lifecycle] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation"
@@ -47,8 +47,6 @@ setup:
   - length: { data_streams: 1}
   - match: { data_streams.0.name: data-stream-with-lifecycle }
   - match: { data_streams.0.lifecycle.data_retention: '10d' }
-  - match: { data_streams.0.lifecycle.effective_retention: '10d' }
-  - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration' }
   - match: { data_streams.0.lifecycle.enabled: true}
 
 ---
@@ -63,7 +61,6 @@ setup:
   - length: { data_streams: 1}
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.lifecycle.enabled: true}
-  - is_false: data_streams.0.lifecycle.effective_retention
 
 ---
 "Put data stream lifecycle":
@@ -95,7 +92,6 @@ setup:
   - length: { data_streams: 2 }
   - match: { data_streams.0.name: data-stream-with-lifecycle }
   - match: { data_streams.0.lifecycle.data_retention: "30d" }
-  - is_false: data_streams.0.lifecycle.effective_retention
   - match: { data_streams.0.lifecycle.enabled: false}
   - match: { data_streams.0.lifecycle.downsampling.0.after: '10d'}
   - match: { data_streams.0.lifecycle.downsampling.0.fixed_interval: '1h'}
@@ -103,7 +99,6 @@ setup:
   - match: { data_streams.0.lifecycle.downsampling.1.fixed_interval: '10h'}
   - match: { data_streams.1.name: simple-data-stream1 }
   - match: { data_streams.1.lifecycle.data_retention: "30d" }
-  - is_false: data_streams.0.lifecycle.effective_retention
   - match: { data_streams.1.lifecycle.enabled: false}
   - match: { data_streams.1.lifecycle.downsampling.0.after: '10d'}
   - match: { data_streams.1.lifecycle.downsampling.0.fixed_interval: '1h'}
@@ -129,8 +124,6 @@ setup:
   - match: { data_streams.0.lifecycle.enabled: true}
   - match: { data_streams.1.name: simple-data-stream1 }
   - match: { data_streams.1.lifecycle.data_retention: "30d" }
-  - match: { data_streams.1.lifecycle.effective_retention: "30d"}
-  - match: { data_streams.1.lifecycle.retention_determined_by: "data_stream_configuration"}
   - match: { data_streams.1.lifecycle.enabled: true}
 
 
@@ -144,8 +137,6 @@ setup:
   - length: { data_streams: 1}
   - match: { data_streams.0.name: data-stream-with-lifecycle }
   - match: { data_streams.0.lifecycle.data_retention: "10d" }
-  - match: { data_streams.0.lifecycle.effective_retention: "10d"}
-  - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"}
   - is_true: data_streams.0.lifecycle.rollover
 
 ---
@@ -163,8 +154,6 @@ setup:
   - length: { data_streams: 1 }
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.lifecycle.data_retention: "30d" }
-  - match: { data_streams.0.lifecycle.effective_retention: "30d"}
-  - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"}
   - match: { data_streams.0.lifecycle.enabled: true }
 
   - do:

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml

@@ -23,18 +23,13 @@ setup:
 
 ---
 "Get data stream lifecycle":
-  - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
 
   - do:
       indices.get_data_lifecycle:
         name: "*"
   - length: { data_streams: 1}
   - match: { data_streams.0.name: my-data-stream-1 }
-  - match: { data_streams.0.lifecycle.data_retention: "10d" }
-  - match: { data_streams.0.lifecycle.effective_retention: "10d"}
-  - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"}
+  - match: { data_streams.0.lifecycle.data_retention: '10d' }
   - match: { data_streams.0.lifecycle.enabled: true}
 
 ---
@@ -48,9 +43,7 @@ setup:
 
 ---
 "Put data stream lifecycle does not succeed when at lease one data stream does not exist":
-  - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+
   - do:
       catch:  missing
       indices.put_data_lifecycle:
@@ -64,16 +57,12 @@ setup:
         name: "*"
   - length: { data_streams: 1 }
   - match: { data_streams.0.name: my-data-stream-1 }
-  - match: { data_streams.0.lifecycle.data_retention: "10d" }
-  - match: { data_streams.0.lifecycle.effective_retention: "10d"}
-  - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"}
+  - match: { data_streams.0.lifecycle.data_retention: '10d' }
   - match: { data_streams.0.lifecycle.enabled: true }
 
 ---
 "Delete data stream lifecycle does not succeed when at lease one data stream does not exist":
-  - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+
   - do:
       catch:  missing
       indices.delete_data_lifecycle:
@@ -85,7 +74,5 @@ setup:
         name: "*"
   - length: { data_streams: 1 }
   - match: { data_streams.0.name: my-data-stream-1 }
-  - match: { data_streams.0.lifecycle.data_retention: "10d" }
-  - match: { data_streams.0.lifecycle.effective_retention: "10d"}
-  - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"}
+  - match: { data_streams.0.lifecycle.data_retention: '10d' }
   - match: { data_streams.0.lifecycle.enabled: true }

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_global_retention.yml

@@ -1,139 +0,0 @@
-setup:
-  - skip:
-      features: allowed_warnings
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Global retention was added in 8.14"
-  - do:
-      allowed_warnings:
-        - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation"
-      indices.put_index_template:
-        name: my-lifecycle
-        body:
-          index_patterns: [my-data-stream-*]
-          template:
-            settings:
-              index.number_of_replicas: 0
-            lifecycle: {}
-          data_stream: {}
-
----
-"CRUD global retention":
-  - do:
-      indices.create_data_stream:
-        name: my-data-stream-1
-  - do:
-      cluster.health:
-        index: my-data-stream-1
-        wait_for_status: green
-  - do:
-      data_streams.put_global_retention:
-        body:
-          default_retention: "7d"
-          max_retention: "90d"
-  - is_true: acknowledged
-  - is_false: dry_run
-  - match: {affected_data_streams.0.name: "my-data-stream-1"}
-  - match: {affected_data_streams.0.previous_effective_retention: "infinite"}
-  - match: {affected_data_streams.0.new_effective_retention: "7d"}
-
-  - do:
-      data_streams.get_global_retention: { }
-  - match: { default_retention: "7d" }
-  - match: { max_retention: "90d" }
-
-  - do:
-      data_streams.delete_global_retention: { }
-  - is_true: acknowledged
-  - is_false: dry_run
-  - match: { affected_data_streams.0.name: "my-data-stream-1" }
-  - match: { affected_data_streams.0.previous_effective_retention: "7d" }
-  - match: { affected_data_streams.0.new_effective_retention: "infinite" }
-
-  - do:
-      data_streams.get_global_retention: { }
-  - is_false: default_retention
-  - is_false: max_retention
-
-  - do:
-      indices.delete_data_stream:
-        name: my-data-stream-1
----
-"Dry run global retention":
-  - do:
-      indices.create_data_stream:
-        name: my-data-stream-2
-  - do:
-      indices.put_data_lifecycle:
-        name: "my-data-stream-2"
-        body: >
-          {
-            "data_retention": "90d"
-          }
-  - is_true: acknowledged
-
-  - do:
-      data_streams.put_global_retention:
-        dry_run: true
-        body:
-          default_retention: "7d"
-          max_retention: "30d"
-  - is_false: acknowledged
-  - is_true: dry_run
-  - match: {affected_data_streams.0.name: "my-data-stream-2"}
-  - match: {affected_data_streams.0.previous_effective_retention: "90d"}
-  - match: {affected_data_streams.0.new_effective_retention: "30d"}
-
-  - do:
-      indices.get_data_stream:
-        name: "my-data-stream-2"
-        include_defaults: true
-  - match: { data_streams.0.name: my-data-stream-2 }
-  - match: { data_streams.0.lifecycle.effective_retention: '90d' }
-  - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration' }
-  - do:
-      indices.delete_data_stream:
-        name: my-data-stream-2
----
-"Default global retention is retrieved by data stream and index templates":
-  - do:
-      indices.create_data_stream:
-        name: my-data-stream-3
-
-  - do:
-      data_streams.put_global_retention:
-        body:
-          default_retention: "7d"
-          max_retention: "90d"
-  - is_true: acknowledged
-  - is_false: dry_run
-  - match: {affected_data_streams.0.name: "my-data-stream-3"}
-  - match: {affected_data_streams.0.previous_effective_retention: "infinite"}
-  - match: {affected_data_streams.0.new_effective_retention: "7d"}
-
-  - do:
-      data_streams.get_global_retention: { }
-  - match: { default_retention: "7d" }
-  - match: { max_retention: "90d" }
-
-  - do:
-      indices.get_data_stream:
-        name: "my-data-stream-3"
-  - match: { data_streams.0.name: my-data-stream-3 }
-  - match: { data_streams.0.lifecycle.effective_retention: '7d' }
-  - match: { data_streams.0.lifecycle.retention_determined_by: 'default_global_retention' }
-  - match: { data_streams.0.lifecycle.enabled: true }
-
-  - do:
-      indices.get_index_template:
-        name: my-lifecycle
-
-  - match: { index_templates.0.name: my-lifecycle }
-  - match: { index_templates.0.index_template.template.lifecycle.enabled: true }
-  - match: { index_templates.0.index_template.template.lifecycle.effective_retention: "7d" }
-  - match: { index_templates.0.index_template.template.lifecycle.retention_determined_by: "default_global_retention" }
-
-  - do:
-      data_streams.delete_global_retention: { }
-  - do:
-      indices.delete_data_stream:
-        name: my-data-stream-3

rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.delete_global_retention.json

@@ -1,35 +0,0 @@
-{
-  "data_streams.delete_global_retention":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-delete-global-retention.html",
-      "description":"Deletes the global retention configuration that applies to all data streams managed by the data stream lifecycle."
-    },
-    "stability":"experimental",
-    "visibility":"public",
-    "headers":{
-      "accept": [ "application/json"],
-      "content_type": ["application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/_data_stream/_global_retention",
-          "methods":[
-            "DELETE"
-          ]
-        }
-      ]
-    },
-    "params":{
-      "dry_run":{
-        "type":"boolean",
-        "description":"Determines whether the global retention provided should be applied or only the impact should be determined.",
-        "default":false
-      },
-      "master_timeout":{
-        "type":"time",
-        "description":"Specify timeout for connection to master."
-      }
-    }
-  }
-}

rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.get_global_retention.json

@@ -1,29 +0,0 @@
-{
-  "data_streams.get_global_retention":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-global-retention.html",
-      "description":"Returns global retention configuration that applies to all data streams managed by the data stream lifecycle."
-    },
-    "stability":"experimental",
-    "visibility":"public",
-    "headers":{
-      "accept": [ "application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/_data_stream/_global_retention",
-          "methods":[
-            "GET"
-          ]
-        }
-      ]
-    },
-    "params":{
-      "local":{
-        "type":"boolean",
-        "description":"Return the global retention retrieved from the node that received the request."
-      }
-    }
-  }
-}

rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.put_global_retention.json

@@ -1,39 +0,0 @@
-{
-  "data_streams.put_global_retention":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-put-global-retention.html",
-      "description":"Updates the global retention configuration that applies to all data streams managed by the data stream lifecycle."
-    },
-    "stability":"experimental",
-    "visibility":"public",
-    "headers":{
-      "accept": [ "application/json"],
-      "content_type": ["application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/_data_stream/_global_retention",
-          "methods":[
-            "PUT"
-          ]
-        }
-      ]
-    },
-    "params":{
-      "dry_run":{
-        "type":"boolean",
-        "description":"Determines whether the global retention provided should be applied or only the impact should be determined.",
-        "default":false
-      },
-      "master_timeout":{
-        "type":"time",
-        "description":"Specify timeout for connection to master"
-      }
-    },
-    "body":{
-      "description":"The global retention configuration including optional values for default and max retention.",
-      "required":true
-    }
-  }
-}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml

@@ -117,8 +117,8 @@
 ---
 "Add data stream lifecycle":
   - requires:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with global retention was available from 8.14"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Data stream lifecycle was available from 8.11"
 
   - do:
       cluster.put_component_template:
@@ -141,14 +141,12 @@
   - match: {component_templates.0.component_template.version: 1}
   - match: {component_templates.0.component_template.template.lifecycle.enabled: true}
   - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"}
-  - match: {component_templates.0.component_template.template.lifecycle.effective_retention: "10d"}
-  - match: {component_templates.0.component_template.template.lifecycle.retention_determined_by: "data_stream_configuration"}
 
 ---
 "Get data stream lifecycle with default rollover":
   - requires:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was available from 8.14"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Data stream lifecycle was available from 8.11"
 
   - do:
       cluster.put_component_template:
@@ -172,6 +170,4 @@
   - match: {component_templates.0.component_template.version: 1}
   - match: {component_templates.0.component_template.template.lifecycle.enabled: true}
   - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"}
-  - match: {component_templates.0.component_template.template.lifecycle.effective_retention: "10d"}
-  - match: {component_templates.0.component_template.template.lifecycle.retention_determined_by: "data_stream_configuration"}
   - is_true: component_templates.0.component_template.template.lifecycle.rollover

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml

@@ -93,8 +93,8 @@ setup:
 ---
 "Add data stream lifecycle":
   - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Data stream lifecycle in index templates was updated after 8.10"
       features: allowed_warnings
 
   - do:
@@ -124,14 +124,12 @@ setup:
   - match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}}
   - match: {index_templates.0.index_template.template.lifecycle.enabled: true}
   - match: {index_templates.0.index_template.template.lifecycle.data_retention: "30d"}
-  - match: {index_templates.0.index_template.template.lifecycle.effective_retention: "30d"}
-  - match: {index_templates.0.index_template.template.lifecycle.retention_determined_by: "data_stream_configuration"}
 
 ---
 "Get data stream lifecycle with default rollover":
   - skip:
-      cluster_features: ["data_stream.lifecycle.global_retention"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Data stream lifecycle in index templates was updated after 8.10"
       features: allowed_warnings
 
   - do:
@@ -154,13 +152,11 @@ setup:
   - match: {index_templates.0.index_template.index_patterns: ["data-stream-with-lifecycle-*"]}
   - match: {index_templates.0.index_template.template.lifecycle.enabled: true}
   - match: {index_templates.0.index_template.template.lifecycle.data_retention: "30d"}
-  - match: {index_templates.0.index_template.template.lifecycle.effective_retention: "30d"}
-  - match: {index_templates.0.index_template.template.lifecycle.retention_determined_by: "data_stream_configuration"}
   - is_true: index_templates.0.index_template.template.lifecycle.rollover
 
 ---
 "Reject data stream lifecycle without data stream configuration":
-  - skip:
+  - requires:
       cluster_features: ["gte_v8.11.0"]
       reason: "Data stream lifecycle in index templates was updated after 8.10"
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml

@@ -227,8 +227,8 @@
 ---
 "Simulate index template with lifecycle and include defaults":
   - requires:
-      cluster_features: ["gte_v8.14.0"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+      cluster_features: ["gte_v8.11.0"]
+      reason: "Lifecycle is only available in 8.11+"
       test_runner_features: ["default_shards"]
 
   - do:
@@ -248,7 +248,5 @@
 
   - match: {template.lifecycle.enabled: true}
   - match: {template.lifecycle.data_retention: "7d"}
-  - match: {template.lifecycle.effective_retention: "7d"}
-  - match: {template.lifecycle.retention_determined_by: "data_stream_configuration"}
   - is_true: template.lifecycle.rollover
   - match: {overlapping: []}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml

@@ -1,9 +1,9 @@
 ---
 "Simulate template without a template in the body":
-  - requires:
-      cluster_features: ["gte_v7.9.0"]
+  - skip:
+      version: " - 7.8.99"
       reason: "only available in 7.9+"
-      test_runner_features: ["default_shards"]
+      features: ["default_shards"]
 
   - do:
       indices.put_index_template:
@@ -30,10 +30,10 @@
 
 ---
 "Simulate index template specifying a new template":
-  - requires:
-      cluster_features: ["gte_v7.9.0"]
+  - skip:
+      version: " - 7.8.99"
       reason: "only available in 7.9+"
-      test_runner_features: ["default_shards"]
+      features: ["default_shards"]
 
   - do:
       indices.put_index_template:
@@ -84,10 +84,10 @@
 
 ---
 "Simulate template matches overlapping legacy and composable templates":
-  - requires:
-      cluster_features: ["gte_v7.9.0"]
+  - skip:
+      version: " - 7.8.99"
       reason: "only available in 7.9+"
-      test_runner_features: ["allowed_warnings", "default_shards"]
+      features: ["allowed_warnings", "default_shards"]
 
   - do:
       indices.put_template:
@@ -147,10 +147,10 @@
 
 ---
 "Simulate replacing a template with a newer version":
-  - requires:
-      cluster_features: ["gte_v8.0.0"]
+  - skip:
+      version: " - 7.99.99"
       reason: "not yet backported"
-      test_runner_features: ["allowed_warnings", "default_shards"]
+      features: ["allowed_warnings", "default_shards"]
 
   - do:
       indices.put_index_template:
@@ -202,8 +202,8 @@
 ---
 "Simulate template with lifecycle and include defaults":
   - requires:
-      cluster_features: ["gte_v8.14.0"]
-      reason: "Data stream lifecycle with effective retention was released in 8.14"
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Lifecycle is only available in 8.11+"
       test_runner_features: ["default_shards"]
 
   - do:
@@ -223,6 +223,4 @@
 
   - match: {template.lifecycle.enabled: true}
   - match: {template.lifecycle.data_retention: "7d"}
-  - match: {template.lifecycle.effective_retention: "7d"}
-  - match: {template.lifecycle.retention_determined_by: "data_stream_configuration"}
   - is_true: template.lifecycle.rollover

server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java

@@ -16,7 +16,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -191,14 +190,13 @@ public class GetComponentTemplateAction extends ActionType<GetComponentTemplateA
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params);
             builder.startObject();
             builder.startArray(COMPONENT_TEMPLATES.getPreferredName());
             for (Map.Entry<String, ComponentTemplate> componentTemplate : this.componentTemplates.entrySet()) {
                 builder.startObject();
                 builder.field(NAME.getPreferredName(), componentTemplate.getKey());
                 builder.field(COMPONENT_TEMPLATE.getPreferredName());
-                componentTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+                componentTemplate.getValue().toXContent(builder, params, rolloverConfiguration, globalRetention);
                 builder.endObject();
             }
             builder.endArray();

server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java

@@ -16,7 +16,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -190,14 +189,13 @@ public class GetComposableIndexTemplateAction extends ActionType<GetComposableIn
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params);
             builder.startObject();
             builder.startArray(INDEX_TEMPLATES.getPreferredName());
             for (Map.Entry<String, ComposableIndexTemplate> indexTemplate : this.indexTemplates.entrySet()) {
                 builder.startObject();
                 builder.field(NAME.getPreferredName(), indexTemplate.getKey());
                 builder.field(INDEX_TEMPLATE.getPreferredName());
-                indexTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+                indexTemplate.getValue().toXContent(builder, params, rolloverConfiguration, globalRetention);
                 builder.endObject();
             }
             builder.endArray();

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java

@@ -12,7 +12,6 @@ import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -114,11 +113,10 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params);
         builder.startObject();
         if (this.resolvedTemplate != null) {
             builder.field(TEMPLATE.getPreferredName());
-            this.resolvedTemplate.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+            this.resolvedTemplate.toXContent(builder, params, rolloverConfiguration, globalRetention);
         }
         if (this.overlappingTemplates != null) {
             builder.startArray(OVERLAPPING.getPreferredName());

server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java

@@ -20,7 +20,6 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamAutoShardingEvent;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -544,11 +543,10 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params);
             builder.startObject();
             builder.startArray(DATA_STREAMS_FIELD.getPreferredName());
             for (DataStreamInfo dataStream : dataStreams) {
-                dataStream.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+                dataStream.toXContent(builder, params, rolloverConfiguration, globalRetention);
             }
             builder.endArray();
             builder.endObject();

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java

@@ -132,11 +132,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
             }
             if (this.lifecycle != null) {
                 builder.field(LIFECYCLE_FIELD.getPreferredName());
-                Params withEffectiveRetentionParams = new DelegatingMapParams(
-                    DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS,
-                    params
-                );
-                lifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention);
+                lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
             }
             if (this.error != null) {
                 if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) {

server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java

@@ -34,7 +34,6 @@ import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomMa
 import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.not;
 
 public class GetComponentTemplateResponseTests extends AbstractWireSerializingTestCase<GetComponentTemplateAction.Response> {
     @Override
@@ -103,9 +102,6 @@ public class GetComponentTemplateResponseTests extends AbstractWireSerializingTe
                 .keySet()) {
                 assertThat(serialized, containsString(label));
             }
-            // We check that even if there was no retention provided by the user, the global retention applies
-            assertThat(serialized, not(containsString("data_retention")));
-            assertThat(serialized, containsString("effective_retention"));
         }
     }