[8.x] Allow stored source in logsdb and tsdb (#114454) (#114648)

* Allow stored source in logsdb and tsdb (#114454)

(cherry picked from commit a62228a74422f24a45e5ab93d572a2a4d2383c32)

# Conflicts:
#	modules/aggregations/build.gradle
#	modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
#	rest-api-spec/build.gradle

* Fix tests

* Fix tests

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml

@@ -291,23 +291,6 @@ setup:
                     sum:
                       sum:
                         field: val
----
-"Configure with no synthetic source":
-  - requires:
-      cluster_features: ["gte_v8.15.0"]
-      reason: "Error message changed in 8.15.0"
-
-  - do:
-      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
-      indices.create:
-        index: tsdb_error
-        body:
-          settings:
-            mode: time_series
-            routing_path: [key]
-          mappings:
-            _source:
-              enabled: false
 
 ---
 "Number for keyword routing field":

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java

@@ -114,18 +114,62 @@ public class LogsIndexModeCustomSettingsIT extends LogsIndexModeRestTestIT {
               }
             }""";
 
+        assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertOK(createDataStream(client, "logs-custom-dev"));
+
+        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
+        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
+        assertThat(sourceMode, equalTo("stored"));
+    }
+
+    public void testConfigureDisabledSourceBeforeIndexCreation() {
+        var storedSourceMapping = """
+            {
+              "template": {
+                "settings": {
+                  "index": {
+                    "mode": "logsdb"
+                  }
+                },
+                "mappings": {
+                  "_source": {
+                    "enabled": false
+                  }
+                }
+              }
+            }""";
+
         Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping));
         assertThat(
             e.getMessage(),
-            containsString("Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source")
+            containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode")
         );
         assertThat(e.getMessage(), containsString("mapper_parsing_exception"));
+    }
 
-        assertOK(createDataStream(client, "logs-custom-dev"));
+    public void testConfigureDisabledSourceModeBeforeIndexCreation() {
+        var storedSourceMapping = """
+            {
+              "template": {
+                "settings": {
+                  "index": {
+                    "mode": "logsdb"
+                  }
+                },
+                "mappings": {
+                  "_source": {
+                    "mode": "disabled"
+                  }
+                }
+              }
+            }""";
 
-        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
-        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
-        assertThat(sourceMode, equalTo("synthetic"));
+        Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertThat(
+            e.getMessage(),
+            containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode")
+        );
+        assertThat(e.getMessage(), containsString("mapper_parsing_exception"));
     }
 
     public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException {
@@ -141,8 +185,45 @@ public class LogsIndexModeCustomSettingsIT extends LogsIndexModeRestTestIT {
             }""";
 
         assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertOK(createDataStream(client, "logs-custom-dev"));
+
+        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
+        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
+        assertThat(sourceMode, equalTo("stored"));
+    }
+
+    public void testConfigureDisabledSourceWhenIndexIsCreated() throws IOException {
+        var disabledModeMapping = """
+            {
+              "template": {
+                "mappings": {
+                  "_source": {
+                    "enabled": false
+                  }
+                }
+              }
+            }""";
+
+        assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping));
+        ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev"));
+        assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode"));
+    }
+
+    public void testConfigureDisabledSourceModeWhenIndexIsCreated() throws IOException {
+        var disabledModeMapping = """
+            {
+              "template": {
+                "mappings": {
+                  "_source": {
+                    "mode": "disabled"
+                  }
+                }
+              }
+            }""";
+
+        assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping));
         ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev"));
-        assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source"));
+        assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode"));
     }
 
     public void testOverrideIndexCodec() throws IOException {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml

@@ -1,21 +1,10 @@
 ---
-stored _source mode is not supported:
+stored _source mode is supported:
   - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
 
   - do:
-      catch: bad_request
       indices.create:
         index: test-stored-source
         body:
@@ -25,31 +14,17 @@ stored _source mode is not supported:
           mappings:
             _source:
               mode: stored
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
+  - do:
+      indices.get:
+        index: test-stored-source
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { test-stored-source.mappings._source.mode: "stored" }
 
 ---
 disabled _source is not supported:
   - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new error message
 
   - do:
       catch: bad_request
@@ -62,20 +37,15 @@ disabled _source is not supported:
           mappings:
             _source:
               enabled: false
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
 
   - match: { error.type: "mapper_parsing_exception" }
   - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
 
   - do:
       catch: bad_request
       indices.create:
-        index: test-disabled-source
+        index: test-disabled-mode-source
         body:
           settings:
             index:
@@ -83,12 +53,7 @@ disabled _source is not supported:
           mappings:
             _source:
               mode: disabled
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
 
   - match: { error.type: "mapper_parsing_exception" }
   - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -456,13 +456,12 @@ nested fields:
   - match: {tsdb-synthetic.mappings._source.mode: synthetic}
 
 ---
-regular source:
+stored source is supported:
   - requires:
-      cluster_features: ["gte_v8.7.0"]
-      reason: synthetic source
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
 
   - do:
-      catch: '/time series indices only support synthetic source/'
       indices.create:
         index: tsdb_index
         body:
@@ -486,14 +485,21 @@ regular source:
                       uid:
                         type: keyword
                         time_series_dimension: true
+
+  - do:
+      indices.get:
+        index: tsdb_index
+
+  - match: { tsdb_index.mappings._source.mode: "stored" }
+
 ---
-disabled source:
+disabled source is not supported:
   - requires:
-      cluster_features: ["gte_v8.7.0"]
-      reason: synthetic source
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new error message
 
   - do:
-      catch: '/time series indices only support synthetic source/'
+      catch: bad_request
       indices.create:
         index: tsdb_index
         body:
@@ -518,6 +524,10 @@ disabled source:
                         type: keyword
                         time_series_dimension: true
 
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
 ---
 source include/exclude:
   - requires:

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -217,8 +217,8 @@ public enum IndexMode {
 
         @Override
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
-            if (sourceFieldMapper.isSynthetic() == false) {
-                throw new IllegalArgumentException("time series indices only support synthetic source");
+            if (sourceFieldMapper.enabled() == false) {
+                throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.TIME_SERIES + "] index mode");
             }
         }
 
@@ -292,8 +292,8 @@ public enum IndexMode {
 
         @Override
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
-            if (sourceFieldMapper.isSynthetic() == false) {
-                throw new IllegalArgumentException("Indices with with index mode [" + IndexMode.LOGSDB + "] only support synthetic source");
+            if (sourceFieldMapper.enabled() == false) {
+                throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.LOGSDB + "] index mode");
             }
         }
 

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -58,6 +58,10 @@ public class MapperFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(RangeFieldMapper.DATE_RANGE_INDEXING_FIX, IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE);
+        return Set.of(
+            RangeFieldMapper.DATE_RANGE_INDEXING_FIX,
+            IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE,
+            SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION
+        );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java

@@ -12,7 +12,6 @@ package org.elasticsearch.index.mapper;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -147,10 +146,6 @@ public final class MappingParser {
                 assert fieldNodeMap.isEmpty();
 
                 if (metadataFieldMapper instanceof SourceFieldMapper sfm) {
-                    // Validation in other places should have failed first
-                    assert sfm.isSynthetic()
-                        || (sfm.isSynthetic() == false && mappingParserContext.getIndexSettings().getMode() != IndexMode.TIME_SERIES)
-                        : "synthetic source can't be disabled in a time series index";
                     isSourceSynthetic = sfm.isSynthetic();
                 }
 

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -53,6 +53,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
     public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature(
         "mapper.source.synthetic_source_copy_to_inside_objects_fix"
     );
+    public static final NodeFeature REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION = new NodeFeature(
+        "mapper.source.remove_synthetic_source_only_validation"
+    );
 
     public static final String NAME = "_source";
     public static final String RECOVERY_SOURCE_NAME = "_recovery_source";
@@ -288,9 +291,6 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
-                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
-                }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml

@@ -70,12 +70,11 @@ create an index with disabled source mode and logsdb index mode without setting:
               mode: disabled
 
   - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
 
 ---
 create an index with stored source mode and logsdb index mode without setting:
   - do:
-      catch: bad_request
       indices.create:
         index: test_stored_logsdb
         body:
@@ -86,8 +85,11 @@ create an index with stored source mode and logsdb index mode without setting:
             _source:
               mode: stored
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - do:
+      indices.get_mapping:
+        index: test_stored_logsdb
+
+  - match: { test_stored_logsdb.mappings._source.mode: stored }
 
 ---
 create an index with synthetic source mode and logsdb index mode without setting:
@@ -131,12 +133,11 @@ create an index with disabled source mode and time series index mode without set
                 time_series_dimension: true
 
   - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: time series indices only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
 
 ---
 create an index with stored source mode and time series index mode without setting:
   - do:
-      catch: bad_request
       indices.create:
         index: test_stored_time_series
         body:
@@ -155,9 +156,11 @@ create an index with stored source mode and time series index mode without setti
                 type: keyword
                 time_series_dimension: true
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: time series indices only support synthetic source" }
+  - do:
+      indices.get_mapping:
+        index: test_stored_time_series
 
+  - match: { test_stored_time_series.mappings._source.mode: stored }
 
 ---
 create an index with synthetic source mode and time series index mode without setting:
@@ -489,7 +492,6 @@ create an index with logsdb index mode and synthetic source:
 ---
 create an index with time_series index mode and stored source:
   - do:
-      catch: bad_request
       indices.create:
         index: test_time_series_index_mode_undefined
         body:
@@ -507,13 +509,20 @@ create an index with time_series index mode and stored source:
                 type: keyword
                 time_series_dimension: true
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: time series indices only support synthetic source" }
+  - do:
+      indices.get_settings:
+        index: "test_time_series_index_mode_undefined"
+  - match: { test_time_series_index_mode_undefined.settings.index.mapping.source.mode: stored }
+
+  - do:
+      indices.get_mapping:
+        index: test_time_series_index_mode_undefined
+
+  - match: { test_time_series_index_mode_undefined.mappings._source.mode: stored }
 
 ---
 create an index with logsdb index mode and stored source:
   - do:
-      catch: bad_request
       indices.create:
         index: test_logsdb_index_mode_undefined
         body:
@@ -522,8 +531,11 @@ create an index with logsdb index mode and stored source:
               mode: logsdb
               mapping.source.mode: stored
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - do:
+      indices.get_mapping:
+        index: test_logsdb_index_mode_undefined
+
+  - match: { test_logsdb_index_mode_undefined.mappings._source.mode: stored }
 
 ---
 create an index with time_series index mode and disabled source:
@@ -738,7 +750,7 @@ modify logsdb index source mode to disabled after index creation:
           _source:
             mode: disabled
   - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
 
 ---
 modify logsdb index source mode to stored after index creation:
@@ -757,8 +769,8 @@ modify logsdb index source mode to stored after index creation:
         body:
           _source:
             mode: stored
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" }
 
 ---
 modify time_series index source mode to disabled after index creation:
@@ -787,7 +799,7 @@ modify time_series index source mode to disabled after index creation:
           _source:
             mode: disabled
   - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: time series indices only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
 
 ---
 modify time_series index source mode to stored after index creation:
@@ -815,5 +827,5 @@ modify time_series index source mode to stored after index creation:
         body:
           _source:
             mode: stored
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: time series indices only support synthetic source" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" }