Parcourir la source

TSDB: fix resize time_series index when _source set enabled to false (#81488)

when _source enabled set false in a time_series index, resize will
failed. reproduce: in tsdb/80_index_resize.yml, add _source enabled to
false, like:

```
......
          mappings:
            _source:
              enabled: false
            properties:
              "@timestamp":
......

```

the tests will failed due to _source conflicted.

this case is similar to the `DataStreamTimestampFieldMapper`, I fixed up it just as `DataStreamTimestampFieldMapper`

the reason why resize failed is that, resize will create an index without current index's mapping. But when creating a time_series index, it will add default mapping. when shard recovery, default mapping will merge with the origin index's mapping, it cause the _source conflict
weizijun il y a 3 ans
Parent
commit
dc358f024d

+ 91 - 0
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml

@@ -148,3 +148,94 @@ clone:
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
+
+---
+clone no source index:
+  - skip:
+      version: " - 8.00.99"
+      reason: introduced in 8.1.0
+
+  - do:
+      indices.create:
+        index: test_no_source
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ metricset, k8s.pod.uid ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            _source:
+              enabled: false
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                      ip:
+                        type: ip
+                      network:
+                        properties:
+                          tx:
+                            type: long
+                          rx:
+                            type: long
+
+  - do:
+      bulk:
+        refresh: true
+        index: test_no_source
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
+
+  - do:
+      indices.put_settings:
+        index: test_no_source
+        body:
+          index.blocks.write: true
+
+  - do:
+      indices.clone:
+        index: test_no_source
+        target: test_no_source_clone
+
+  - do:
+      search:
+        index: test_no_source_clone
+        body:
+          docvalue_fields:
+            - field: _tsid
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}

+ 3 - 1
server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -65,7 +65,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     public static class Builder extends MetadataFieldMapper.Builder {
 
-        private final Parameter<Boolean> enabled = Parameter.boolParam("enabled", false, m -> toType(m).enabled, Defaults.ENABLED);
+        private final Parameter<Boolean> enabled = Parameter.boolParam("enabled", false, m -> toType(m).enabled, Defaults.ENABLED)
+            // this field mapper may be enabled but once enabled, may not be disabled
+            .setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous && current == false));
         private final Parameter<List<String>> includes = Parameter.stringArrayParam(
             "includes",
             false,

+ 10 - 1
server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java

@@ -32,7 +32,16 @@ public class SourceFieldMapperTests extends MetadataMapperTestCase {
 
     @Override
     protected void registerParameters(ParameterChecker checker) throws IOException {
-        checker.registerConflictCheck("enabled", b -> b.field("enabled", false));
+        checker.registerConflictCheck(
+            "enabled",
+            topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", false).endObject()),
+            topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject())
+        );
+        checker.registerUpdateCheck(
+            topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()),
+            topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", false).endObject()),
+            dm -> assertFalse(dm.metadataMapper(SourceFieldMapper.class).enabled())
+        );
         checker.registerConflictCheck("includes", b -> b.array("includes", "foo*"));
         checker.registerConflictCheck("excludes", b -> b.array("excludes", "foo*"));
     }