Merge pull request ESQL-1517 from elastic/main

🤖 ESQL: Merge upstream

benchmarks/build.gradle

@@ -51,6 +51,9 @@ dependencies {
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 tasks.named("compileJava").configure {
   options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
+  // org.elasticsearch.plugins.internal is used in signatures classes used in benchmarks but we don't want to expose it publicly
+  // adding an export to allow compilation with gradle. This does not solve a problem in intellij as it does not use compileJava task
+  options.compilerArgs.addAll(["--add-exports", "org.elasticsearch.server/org.elasticsearch.plugins.internal=ALL-UNNAMED"])
 }
 
 tasks.register('copyExpression', Copy) {

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -71,7 +72,8 @@ public class MapperServiceFactory {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         try {

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -41,6 +41,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -188,7 +189,8 @@ public class QueryParserHelperBenchmark {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         try {

docs/changelog/97416.yaml

@@ -0,0 +1,5 @@
+pr: 97416
+summary: Enable parallel collection in Dfs phase
+area: Search
+type: enhancement
+issues: []

docs/changelog/97822.yaml

@@ -0,0 +1,5 @@
+pr: 97822
+summary: Allow Lucene directory implementations to estimate their size
+area: Store
+type: enhancement
+issues: []

docs/changelog/97890.yaml

@@ -0,0 +1,5 @@
+pr: 97890
+summary: "[Profiling] Consider static settings in status"
+area: Application
+type: bug
+issues: []

docs/changelog/97961.yaml

@@ -0,0 +1,5 @@
+pr: 97961
+summary: Infrastructure to report upon document parsing
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/98051.yaml

@@ -0,0 +1,5 @@
+pr: 98051
+summary: Mark customer settings for serverless
+area: Infra/Settings
+type: enhancement
+issues: []

docs/changelog/98085.yaml

@@ -0,0 +1,5 @@
+pr: 98085
+summary: Allow `ByteSizeDirectory` to expose their data set sizes
+area: Store
+type: enhancement
+issues: []

docs/changelog/98091.yaml

@@ -0,0 +1,6 @@
+pr: 98091
+summary: '`GlobalAggregator` should call rewrite() before `createWeight()`'
+area: Aggregations
+type: bug
+issues:
+ - 98076

docs/reference/aggregations/bucket/range-aggregation.asciidoc

@@ -338,7 +338,25 @@ with latency metrics (in milliseconds) for different networks:
 
 [source,console]
 ----
-PUT metrics_index/_doc/1
+PUT metrics_index
+{
+  "mappings": {
+    "properties": {
+      "network": {
+        "properties": {
+          "name": {
+            "type": "keyword"
+          }
+        }
+      },
+      "latency_histo": {
+         "type": "histogram"
+      }
+    }
+  }
+}
+
+PUT metrics_index/_doc/1?refresh
 {
   "network.name" : "net-1",
   "latency_histo" : {
@@ -347,7 +365,7 @@ PUT metrics_index/_doc/1
    }
 }
 
-PUT metrics_index/_doc/2
+PUT metrics_index/_doc/2?refresh
 {
   "network.name" : "net-2",
   "latency_histo" : {
@@ -385,24 +403,24 @@ return the following output:
       "buckets": [
         {
           "key": "*-2.0",
-          "to": 2,
+          "to": 2.0,
           "doc_count": 11
         },
         {
           "key": "2.0-3.0",
-          "from": 2,
-          "to": 3,
+          "from": 2.0,
+          "to": 3.0,
           "doc_count": 0
         },
         {
           "key": "3.0-10.0",
-          "from": 3,
-          "to": 10,
+          "from": 3.0,
+          "to": 10.0,
           "doc_count": 55
         },
         {
           "key": "10.0-*",
-          "from": 10,
+          "from": 10.0,
           "doc_count": 31
         }
       ]
@@ -410,7 +428,7 @@ return the following output:
   }
 }
 ----
-// TESTRESPONSE[skip:test not setup]
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 [IMPORTANT]
 ========

docs/reference/ccr/apis/follow-request-body.asciidoc

@@ -2,6 +2,7 @@
   (object) Settings to override from the leader index. Note that certain
   settings can not be overrode (e.g., `index.number_of_shards`).
 
+// tag::ccr-resume-follow-request-body[]
 `max_read_request_operation_count`::
   (integer) The maximum number of operations to pull per read from the remote
   cluster.
@@ -101,3 +102,4 @@ values for the above described index follow request parameters:
 }
 
 --------------------------------------------------
+// end::ccr-resume-follow-request-body[]

docs/reference/ccr/apis/follow/post-resume-follow.asciidoc

@@ -68,7 +68,7 @@ returns, the follower index will resume fetching operations from the leader inde
 
 [[ccr-post-resume-follow-request-body]]
 ==== {api-request-body-title}
-include::../follow-request-body.asciidoc[]
+include::../follow-request-body.asciidoc[tag=ccr-resume-follow-request-body]
 
 [[ccr-post-resume-follow-examples]]
 ==== {api-examples-title}

docs/reference/redirects.asciidoc

@@ -1908,12 +1908,12 @@ Refer to <<data-streams-delete-lifecycle,Update data stream lifecycle API>>.
 coming::[8.10.0]
 
 [role="exclude",id="get-synonyms"]
-=== Get synonym set API
+=== Get synonyms set API
 
 coming::[8.10.0]
 
 [role="exclude",id="list-synonyms"]
-=== List synonym sets API
+=== List synonyms sets API
 
 coming::[8.10.0]
 
@@ -1923,11 +1923,11 @@ coming::[8.10.0]
 coming::[8.10.0]
 
 [role="exclude",id="delete-synonyms"]
-=== Delete synonym sets API
+=== Delete synonyms sets API
 
 coming::[8.10.0]
 
 [role="exclude",id="put-synonyms"]
-=== Create or update synonym sets API
+=== Create or update synonyms sets API
 
-coming::[8.10.0]
+coming::[8.10.0]

docs/reference/rest-api/index.asciidoc

@@ -50,6 +50,7 @@ not be included yet.
 * <<snapshot-restore-apis,Snapshot and restore APIs>>
 * <<snapshot-lifecycle-management-api,Snapshot lifecycle management APIs>>
 * <<sql-apis,SQL APIs>>
+* <<synonyms-apis,Synonyms APIs>>
 * <<transform-apis,{transform-cap} APIs>>
 * <<usage-api,Usage API>>
 * <<watcher-api,Watcher APIs>>
@@ -95,6 +96,7 @@ include::{xes-repo-dir}/rest-api/security.asciidoc[]
 include::{es-repo-dir}/snapshot-restore/apis/snapshot-restore-apis.asciidoc[]
 include::{es-repo-dir}/slm/apis/slm-api.asciidoc[]
 include::{es-repo-dir}/sql/apis/sql-apis.asciidoc[]
+include::{es-repo-dir}/synonyms/apis/synonyms-apis.asciidoc[]
 include::{es-repo-dir}/transform/apis/index.asciidoc[]
 include::usage.asciidoc[]
 include::{xes-repo-dir}/rest-api/watcher.asciidoc[]

docs/reference/synonyms/apis/delete-synonyms-set.asciidoc

@@ -0,0 +1,189 @@
+[[delete-synonyms-set]]
+=== Delete synonyms set
+
+beta::[]
+
+++++
+<titleabbrev>Delete synonyms set</titleabbrev>
+++++
+
+Deletes a synonyms set.
+
+[[delete-synonyms-set-request]]
+==== {api-request-title}
+
+`DELETE _synonyms/<synonyms_set>`
+
+[[delete-synonyms-set-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `manage_search_synonyms` cluster privilege.
+* You can only delete a synonyms set that is not in use by any index analyzer. See <<delete-synonym-set-analyzer-requirements>> for more information.
+
+[[delete-synonyms-set-path-params]]
+==== {api-path-parms-title}
+
+`<synonyms_set>`::
+(Required, string)
+Synonyms set identifier to delete.
+
+
+[[delete-synonyms-set-response-codes]]
+==== {api-response-codes-title}
+
+`400`::
+The `synonyms_set` identifier was not provided, or the synonyms set can't be deleted because it does not meet the <<delete-synonym-set-analyzer-requirements,specified requirements>>.
+
+`404` (Missing resources)::
+No synonyms set with the identifier `synonyms_set` was found.
+
+[[delete-synonyms-set-example]]
+==== {api-examples-title}
+
+The following example deletes a synonyms set called `my-synonyms-set`:
+
+
+////
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+  "synonyms_set": [
+    {
+      "id": "test-1",
+      "synonyms": "hello, hi"
+    }
+  ]
+}
+----
+// TESTSETUP
+////
+
+[source,console]
+----
+DELETE _synonyms/my-synonyms-set
+----
+
+[discrete]
+[[delete-synonym-set-analyzer-requirements]]
+==== Delete synonyms set analyzer requirements
+
+synonyms sets can be used in  <<analysis-synonym-graph-tokenfilter,synonym graph token filters>> and <<analysis-synonym-tokenfilter,synonym token filters>>.
+These synonym filters can be used as part of <<search-analyzer, search analyzers>>.
+
+Analyzers need to be loaded when an index is restored (such as when a node starts, or the index becomes open).
+Even if the analyzer is not used on any field mapping, it still needs to be loaded on the index recovery phase.
+
+If any analyzers cannot be loaded, the index becomes unavailable and the cluster status becomes <<red-yellow-cluster-status,red or yellow>> as index shards are not available.
+
+To prevent that, synonyms sets that are used in analyzers can't be deleted.
+A delete request in this case will return a `400` response code with the following error message:
+
+////
+[source,console]
+----
+PUT /index-1
+{
+  "settings": {
+    "analysis": {
+      "filter": {
+        "synonyms_filter": {
+          "type": "synonym_graph",
+          "synonyms_set": "my-synonyms-set",
+          "updateable": true
+        }
+      },
+      "analyzer": {
+        "my_index_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase"]
+        },
+        "my_search_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "synonyms_filter"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "analyzer": "my_index_analyzer",
+        "search_analyzer": "my_search_analyzer"
+      }
+    }
+  }
+}
+
+PUT /index-2
+{
+  "settings": {
+    "analysis": {
+      "filter": {
+        "synonyms_filter": {
+          "type": "synonym_graph",
+          "synonyms_set": "my-synonyms-set",
+          "updateable": true
+        }
+      },
+      "analyzer": {
+        "my_index_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase"]
+        },
+        "my_search_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "synonyms_filter"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "analyzer": "my_index_analyzer",
+        "search_analyzer": "my_search_analyzer"
+      }
+    }
+  }
+}
+
+DELETE _synonyms/my-synonyms-set
+----
+// TEST[catch:bad_request]
+////
+
+[source,console-result]
+----
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "illegal_argument_exception",
+        "reason": "Synonyms set [my-synonyms-set] cannot be deleted as it is used in the following indices: index-1, index-2",
+        "stack_trace": ...
+      }
+    ],
+    "type": "illegal_argument_exception",
+    "reason": "Synonyms set [my-synonyms-set] cannot be deleted as it is used in the following indices: index-1, index-2",
+    "stack_trace": ...
+  },
+  "status": 400
+}
+----
+// TESTRESPONSE[s/"stack_trace": \.\.\./"stack_trace": $body.$_path/]
+
+To remove a synonyms set, you must first remove all indices that contain analyzers using it.
+You can migrate an index by creating a new index that does not contain the token filter with the synonyms set, and use the <<docs-reindex>> in order to copy over the index data.
+Once finished, you can delete the index.
+
+When the synonyms set is not used in analyzers, you will be able to delete it.
+
+
+

docs/reference/synonyms/apis/get-synonyms-set.asciidoc

@@ -0,0 +1,104 @@
+[[get-synonyms-set]]
+=== Get synonyms set
+
+beta::[]
+
+++++
+<titleabbrev>Get synonyms set</titleabbrev>
+++++
+
+Retrieves a synonyms set.
+
+[[get-synonyms-set-request]]
+==== {api-request-title}
+
+`GET _synonyms/<synonyms_set>`
+
+[[get-synonyms-set-prereqs]]
+==== {api-prereq-title}
+
+Requires the `manage_search_synonyms` cluster privilege.
+
+[[get-synonyms-set-path-params]]
+==== {api-path-parms-title}
+
+`<synonyms_set>`::
+(Required, string)
+Synonyms set identifier to retrieve.
+
+[[get-synonyms-set-query-params]]
+==== {api-query-parms-title}
+
+`from`::
+(Optional, integer) Starting offset for synonyms rules to retrieve. Defaults to `0`.
+
+`size`::
+(Optional, integer) Specifies the maximum number of synonyms rules to retrieve. Defaults to `10`.
+
+[[get-synonyms-set-response-codes]]
+==== {api-response-codes-title}
+
+`400`::
+The `synonyms_set` identifier was not provided.
+
+`404` (Missing resources)::
+No synonyms set with the identifier `synonyms_set` was found.
+
+[[get-synonyms-set-example]]
+==== {api-examples-title}
+
+The following example retrieves a synonyms set called `my-synonyms-set`:
+
+////
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+  "synonyms_set": [
+    {
+      "id": "test-1",
+      "synonyms": "hello, hi"
+    },
+    {
+      "id": "test-2",
+      "synonyms": "bye, goodbye"
+    },
+    {
+      "id": "test-3",
+      "synonyms": "test => check"
+    }
+  ]
+}
+----
+// TESTSETUP
+////
+
+[source,console]
+----
+GET _synonyms/my-synonyms-set
+----
+
+The synonyms set information returned will include the total number of synonyms rules that the synonyms set contains, and the synonyms rules according to the `from` and `size` parameters.
+
+A sample response:
+
+[source,console-result]
+----
+{
+  "count": 3,
+  "synonyms_set": [
+    {
+      "id": "test-1",
+      "synonyms": "hello, hi"
+    },
+    {
+      "id": "test-2",
+      "synonyms": "bye, goodbye"
+    },
+    {
+      "id": "test-3",
+      "synonyms": "test => check"
+    }
+  ]
+}
+----

docs/reference/synonyms/apis/list-synonyms-sets.asciidoc

@@ -0,0 +1,107 @@
+[[list-synonyms-sets]]
+=== List synonyms sets
+
+beta::[]
+
+++++
+<titleabbrev>List synonyms sets</titleabbrev>
+++++
+
+Retrieves a summary of all defined synonyms sets.
+
+This API allows to retrieve the total number of synonyms sets defined.
+For each synonyms set, its identifier and the total number of defined synonym rules is returned.
+
+[[list-synonyms-sets-request]]
+==== {api-request-title}
+
+`GET _synonyms`
+
+[[list-synonyms-sets-prereqs]]
+==== {api-prereq-title}
+
+Requires the `manage_search_synonyms` cluster privilege.
+
+[[list-synonyms-sets-query-params]]
+==== {api-query-parms-title}
+
+`from`::
+(Optional, integer) Starting offset for synonyms sets to retrieve. Defaults to `0`.
+
+`size`::
+(Optional, integer) Specifies the maximum number of synonyms sets to retrieve. Defaults to `10`.
+
+[[list-synonyms-sets-example]]
+==== {api-examples-title}
+
+The following example retrieves all defined synonyms sets:
+
+////
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+  "synonyms_set": [
+    {
+      "synonyms": "hello, hi"
+    },
+    {
+      "synonyms": "other, another"
+    },
+    {
+      "synonyms": "bye, goodbye"
+    }
+  ]
+}
+
+PUT _synonyms/ecommerce-synonyms
+{
+  "synonyms_set": [
+    {
+      "synonyms": "pants, trousers"
+    },
+    {
+      "synonyms": "dress, frock"
+    }
+  ]
+}
+
+PUT _synonyms/new-ecommerce-synonyms
+{
+  "synonyms_set": [
+    {
+      "synonyms": "tie, bowtie"
+    }
+  ]
+}
+----
+// TESTSETUP
+////
+
+[source,console]
+----
+GET _synonyms
+----
+
+A sample response:
+
+[source,console-result]
+----
+{
+  "count": 3,
+  "results": [
+    {
+      "synonyms_set": "ecommerce-synonyms",
+      "count": 2
+    },
+    {
+      "synonyms_set": "my-synonyms-set",
+      "count": 3
+    },
+    {
+      "synonyms_set": "new-ecommerce-synonyms",
+      "count": 1
+    }
+  ]
+}
+----

docs/reference/synonyms/apis/put-synonyms-set.asciidoc

@@ -0,0 +1,210 @@
+[[put-synonyms-set]]
+=== Create or update synonyms set
+
+beta::[]
+
+++++
+<titleabbrev>Create or update synonyms set</titleabbrev>
+++++
+
+Creates or updates a synonyms set.
+
+[[put-synonyms-set-request]]
+==== {api-request-title}
+
+`PUT _synonyms/<synonyms_set>`
+
+[[put-synonyms-set-prereqs]]
+==== {api-prereq-title}
+
+Requires the `manage_search_synonyms` cluster privilege.
+
+[[put-synonyms-set-path-params]]
+==== {api-path-parms-title}
+
+`<synonyms_set>`::
+(Required, string)
+Synonyms set identifier to create.
+This identifier will be used by other <<synonyms-apis>> to manage the synonyms set.
+
+[[put-synonyms-set-api-request-body]]
+==== {api-request-body-title}
+
+`synonyms_set`::
+(Required, array of synonym rules objects)
+The synonym rules definitions for the synonyms set.
+
+.Properties of `synonyms_set` objects
+[%collapsible%open]
+=====
+
+`id`::
+(Optional, string)
+// TODO link to synonym rules APIs
+The identifier associated to the synonym rule, that can be used to manage individual synonym rules via Synonym Rules APIs.
+In case a synonym rule id is not specified, an identifier will be created automatically by {es}.
+
+`synonyms`::
+(Requried, string)
+The synonym rule. This needs to be in <<_solr_synonyms>> format. Some examples are:
+* "i-pod, i pod => ipod",
+* "universe, cosmos"
+
+=====
+
+[[put-synonyms-set-example]]
+==== {api-examples-title}
+
+The following example creates a new synonyms set called `my-synonyms-set`:
+
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+  "synonyms_set": [
+    {
+      "id": "test-1",
+      "synonyms": "hello, hi"
+    },
+    {
+      "synonyms": "bye, goodbye"
+    },
+    {
+      "id": "test-2",
+      "synonyms": "test => check"
+    }
+  ]
+}
+----
+
+If any of the synonym rules included is not valid, the API will return an error.
+
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+  "synonyms_set": [
+    {
+      "synonyms": "hello => hi => howdy"
+    }
+  ]
+}
+----
+// TEST[catch:bad_request]
+
+
+[source,console-result]
+----
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "action_request_validation_exception",
+        "reason": "Validation Failed: 1: More than one explicit mapping specified in the same synonyms rule: [hello => hi => howdy];",
+        "stack_trace": ...
+      }
+    ],
+    "type": "action_request_validation_exception",
+    "reason": "Validation Failed: 1: More than one explicit mapping specified in the same synonyms rule: [hello => hi => howdy];",
+    "stack_trace": ...
+  },
+  "status": 400
+}
+----
+// TESTRESPONSE[s/"stack_trace": \.\.\./"stack_trace": $body.$_path/]
+
+
+[discrete]
+==== Analyzer reloading
+When an existing synonyms set is updated, the <<search-analyzer, search analyzers>> that use the synonyms set are reloaded automatically for all indices.
+This would be equivalent to invoking <<indices-reload-analyzers>> for all indices that use the synonyms set.
+
+For example, creating an index with a synonyms set and updating it:
+
+[source,console]
+----
+PUT _synonyms/my-synonyms-set
+{
+    "synonyms_set": [
+        {
+            "id": "test-1",
+            "synonyms": "hello, hi"
+        }
+    ]
+}
+
+PUT /test-index
+{
+  "settings": {
+    "analysis": {
+      "filter": {
+        "synonyms_filter": {
+          "type": "synonym_graph",
+          "synonyms_set": "my-synonyms-set",
+          "updateable": true
+        }
+      },
+      "analyzer": {
+        "my_index_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase"]
+        },
+        "my_search_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": ["lowercase", "synonyms_filter"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "analyzer": "my_index_analyzer",
+        "search_analyzer": "my_search_analyzer"
+      }
+    }
+  }
+}
+
+PUT _synonyms/my-synonyms-set
+{
+    "synonyms_set": [
+        {
+            "id": "test-1",
+            "synonyms": "hello, hi, howdy"
+        }
+    ]
+}
+----
+
+
+The reloading result is included as part of the response:
+
+[source,console-result]
+----
+{
+  "result": "updated",
+  "reload_analyzers_details": {
+    "_shards": {
+      "total": 2,
+      "successful": 1,
+      "failed": 0
+    },
+    "reload_details": [
+      {
+        "index": "test-index",
+        "reloaded_analyzers": [
+          "my_search_analyzer"
+        ],
+        "reloaded_node_ids": [
+          "1wYFZzq8Sxeu_Jvt9mlbkg"
+        ]
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/1wYFZzq8Sxeu_Jvt9mlbkg/$body.reload_analyzers_details.reload_details.0.reloaded_node_ids.0/]

docs/reference/synonyms/apis/synonyms-apis.asciidoc

@@ -0,0 +1,36 @@
+[[synonyms-apis]]
+== Synonyms APIs
+
+beta::[]
+
+++++
+<titleabbrev>Synonyms APIs</titleabbrev>
+++++
+
+---
+
+The synonyms management API provides a convenient way to define and manage synonyms in an internal system index. Related synonyms can be grouped in a "synonyms set".
+Create as many synonym sets as you need.
+
+This provides an alternative to:
+- Defining inline synonyms in an analyzer definition, which impacts mapping size and can lead to performance issues.
+- Using synonyms files, which implies uploading and managing file consistency on all cluster nodes.
+
+Synonyms sets can be used to configure <<analysis-synonym-graph-tokenfilter,synonym graph token filters>> and <<analysis-synonym-tokenfilter,synonym token filters>>.
+These filters are applied as part of the <<search-analyzer,search analyzer>> analysis process.
+
+You can use these APIs to dynamically update synonyms sets used at search time.
+Your search results will immediately reflect the synonyms set changes.
+
+Use the following APIs to manage synonyms sets:
+
+* <<put-synonyms-set>>
+* <<get-synonyms-set>>
+* <<list-synonyms-sets>>
+* <<delete-synonyms-set>>
+
+include::put-synonyms-set.asciidoc[]
+include::get-synonyms-set.asciidoc[]
+include::list-synonyms-sets.asciidoc[]
+include::delete-synonyms-set.asciidoc[]
+

modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/global_with_aliases.yml

@@ -0,0 +1,30 @@
+"global agg with a terms filtered alias":
+  - skip:
+      version: "- 8.9.99"
+      reason: Fixed in 8.10
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"name": "one"}'
+          - '{"index": {}}'
+          - '{"name": "two"}'
+          - '{"index": {}}'
+          - '{"name": "two"}'
+
+  - do:
+      indices.put_alias:
+        index: test
+        name: test-filtered
+        body: {"filter": {"terms": {"name": [ "one" ] }}}
+
+  - do:
+      search:
+        index: test-filtered
+        body:
+          aggs:
+            all_docs:
+              global: {}

modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverWithPipelinesIT.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.FilterXContentParserWrapper;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class DocumentParsingObserverWithPipelinesIT extends ESIntegTestCase {
+
+    private static String TEST_INDEX_NAME = "test-index-name";
+    // the assertions are done in plugin which is static and will be created by ES server.
+    // hence a static flag to make sure it is indeed used
+    public static boolean hasWrappedParser;
+
+    public void testDocumentIsReportedWithPipelines() throws IOException {
+        hasWrappedParser = false;
+        // pipeline adding fields, changing destination is not affecting reporting
+        final BytesReference pipelineBody = new BytesArray("""
+            {
+              "processors": [
+                {
+                   "set": {
+                     "field": "my-text-field",
+                     "value": "xxxx"
+                   }
+                 },
+                 {
+                   "set": {
+                     "field": "my-boolean-field",
+                     "value": true
+                   }
+                 }
+              ]
+            }
+            """);
+        clusterAdmin().putPipeline(new PutPipelineRequest("pipeline", pipelineBody, XContentType.JSON)).actionGet();
+
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).setPipeline("pipeline")
+                .id("1")
+                .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertTrue(hasWrappedParser);
+        // there are more assertions in a TestDocumentParsingObserver
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestDocumentParsingObserverPlugin.class, IngestCommonPlugin.class);
+    }
+
+    public static class TestDocumentParsingObserverPlugin extends Plugin implements DocumentParsingObserverPlugin, IngestPlugin {
+
+        private static final TestDocumentParsingObserver DOCUMENT_PARSING_OBSERVER = new TestDocumentParsingObserver();
+
+        public TestDocumentParsingObserverPlugin() {}
+
+        @Override
+        public Supplier<DocumentParsingObserver> getDocumentParsingObserverSupplier() {
+            // returns a static instance, because we want to assert that the wrapping is called only once
+            return () -> DOCUMENT_PARSING_OBSERVER;
+        }
+
+    }
+
+    public static class TestDocumentParsingObserver implements DocumentParsingObserver {
+        long mapCounter = 0;
+        long wrapperCounter = 0;
+        String indexName;
+
+        @Override
+        public XContentParser wrapParser(XContentParser xContentParser) {
+            wrapperCounter++;
+            hasWrappedParser = true;
+            return new FilterXContentParserWrapper(xContentParser) {
+
+                @Override
+                public Map<String, Object> map() throws IOException {
+                    mapCounter++;
+                    return super.map();
+                }
+            };
+        }
+
+        @Override
+        public void setIndexName(String indexName) {
+            this.indexName = indexName;
+        }
+
+        @Override
+        public void close() {
+            assertThat(indexName, equalTo(TEST_INDEX_NAME));
+            assertThat(mapCounter, equalTo(1L));
+
+            assertThat(
+                "we only want to use a wrapped counter once, once document is reported it no longer needs to wrap",
+                wrapperCounter,
+                equalTo(1L)
+            );
+        }
+
+    }
+
+}

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -480,40 +480,31 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 indexName,
                 Settings.builder()
                     .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
                     .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
-                    .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "120s")
+                    .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "24h")
                     .put("index.routing.allocation.include._name", CLUSTER_NAME + "-0")
                     .build()
             );
+            ensureGreen(indexName);
             indexDocs(indexName, 0, randomInt(10));
-            // allocate replica to node-2
             updateIndexSettings(
                 indexName,
                 Settings.builder()
-                    .put("index.routing.allocation.include._name", CLUSTER_NAME + "-0," + CLUSTER_NAME + "-2," + CLUSTER_NAME + "-*")
+                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    .putNull("index.routing.allocation.include._name")
             );
             ensureGreen(indexName);
             closeIndex(indexName);
         }
 
-        final Version indexVersionCreated = indexVersionCreated(indexName);
-        if (indexVersionCreated.onOrAfter(Version.V_7_2_0)) {
-            // index was created on a version that supports the replication of closed indices,
-            // so we expect the index to be closed and replicated
+        if (indexVersionCreated(indexName).onOrAfter(Version.V_7_2_0)) {
+            // index was created on a version that supports the replication of closed indices, so we expect it to be closed and replicated
+            assertTrue(minimumNodeVersion().onOrAfter(Version.V_7_2_0));
             ensureGreen(indexName);
             assertClosedIndex(indexName, true);
-            if (minimumNodeVersion().onOrAfter(Version.V_7_2_0)) {
-                switch (CLUSTER_TYPE) {
-                    case OLD:
-                        break;
-                    case MIXED:
-                        assertNoopRecoveries(indexName, s -> s.startsWith(CLUSTER_NAME + "-0"));
-                        break;
-                    case UPGRADED:
-                        assertNoopRecoveries(indexName, s -> s.startsWith(CLUSTER_NAME));
-                        break;
-                }
+            if (CLUSTER_TYPE != ClusterType.OLD) {
+                assertNoopRecoveries(indexName, s -> CLUSTER_TYPE == ClusterType.UPGRADED || s.startsWith(CLUSTER_NAME + "-0"));
             }
         } else {
             assertClosedIndex(indexName, false);

rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete.json

@@ -1,7 +1,7 @@
 {
   "synonyms.delete": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonyms.html",
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonyms-set.html",
       "description": "Deletes a synonym set"
     },
     "stability": "experimental",

rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get.json

@@ -1,7 +1,7 @@
 {
   "synonyms.get": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonyms.html",
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonyms-set.html",
       "description": "Retrieves a synonym set"
     },
     "stability": "experimental",

rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put.json

@@ -1,7 +1,7 @@
 {
   "synonyms.put": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonyms.html",
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonyms-set.html",
       "description": "Creates or updates a synonyms set"
     },
     "stability": "experimental",

rest-api-spec/src/main/resources/rest-api-spec/api/synonyms_sets.get.json

@@ -1,7 +1,7 @@
 {
   "synonyms_sets.get": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/list-synonyms.html",
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/list-synonyms-sets.html",
       "description": "Retrieves a summary of all defined synonym sets"
     },
     "stability": "experimental",

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml

@@ -90,7 +90,7 @@ setup:
                 search_analyzer: my_analyzer1
 
   - do:
-      catch: /Synonym set \[test-get-synonyms\] cannot be deleted as it is used in the following indices:\ my_index1/
+      catch: /Synonyms set \[test-get-synonyms\] cannot be deleted as it is used in the following indices:\ my_index1/
       synonyms.delete:
         synonyms_set: test-get-synonyms
 

server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentParsingObserverIT.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.FilterXContentParserWrapper;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xcontent.XContentFactory.cborBuilder;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class DocumentParsingObserverIT extends ESIntegTestCase {
+
+    private static String TEST_INDEX_NAME = "test-index-name";
+
+    // the assertions are done in plugin which is static and will be created by ES server.
+    // hence a static flag to make sure it is indeed used
+    public static boolean hasWrappedParser;
+
+    public void testDocumentIsReportedUponBulk() throws IOException {
+        hasWrappedParser = false;
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertTrue(hasWrappedParser);
+        // there are more assertions in a TestDocumentParsingObserver
+
+        hasWrappedParser = false;
+        // the format of the request does not matter
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).id("2").source(cborBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertTrue(hasWrappedParser);
+        // there are more assertions in a TestDocumentParsingObserver
+
+        hasWrappedParser = false;
+        // white spaces does not matter
+        client().index(new IndexRequest(TEST_INDEX_NAME).id("3").source("""
+            {
+            "test":
+
+            "I am sam i am"
+            }
+            """, XContentType.JSON)).actionGet();
+        assertTrue(hasWrappedParser);
+        // there are more assertions in a TestDocumentParsingObserver
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestDocumentParsingObserverPlugin.class);
+    }
+
+    public static class TestDocumentParsingObserverPlugin extends Plugin implements DocumentParsingObserverPlugin, IngestPlugin {
+
+        public TestDocumentParsingObserverPlugin() {}
+
+        @Override
+        public Supplier<DocumentParsingObserver> getDocumentParsingObserverSupplier() {
+            return () -> new TestDocumentParsingObserver();
+        }
+    }
+
+    public static class TestDocumentParsingObserver implements DocumentParsingObserver {
+        long counter = 0;
+        String indexName;
+
+        @Override
+        public XContentParser wrapParser(XContentParser xContentParser) {
+            hasWrappedParser = true;
+            return new FilterXContentParserWrapper(xContentParser) {
+                @Override
+                public Token nextToken() throws IOException {
+                    counter++;
+                    return super.nextToken();
+                }
+            };
+        }
+
+        @Override
+        public void setIndexName(String indexName) {
+            this.indexName = indexName;
+        }
+
+        @Override
+        public void close() {
+            assertThat(indexName, equalTo(TEST_INDEX_NAME));
+            assertThat(counter, equalTo(5L));
+        }
+    }
+}

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@@ -915,7 +915,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
     }
 
     private int numberOfFiles(Path dir) throws Exception {
-        awaitMasterFinishRepoOperations(); // wait for potential background blob deletes to complete on master
         final AtomicInteger count = new AtomicInteger();
         forEachFileRecursively(dir, ((path, basicFileAttributes) -> count.incrementAndGet()));
         return count.get();

server/src/main/java/module-info.java

@@ -297,7 +297,6 @@ module org.elasticsearch.server {
     exports org.elasticsearch.plugins;
     exports org.elasticsearch.plugins.interceptor to org.elasticsearch.security;
     exports org.elasticsearch.plugins.spi;
-    exports org.elasticsearch.plugins.internal to org.elasticsearch.settings.secure;
     exports org.elasticsearch.repositories;
     exports org.elasticsearch.repositories.blobstore;
     exports org.elasticsearch.repositories.fs;
@@ -372,6 +371,7 @@ module org.elasticsearch.server {
 
     exports org.elasticsearch.action.datastreams.lifecycle;
     exports org.elasticsearch.action.downsample;
+    exports org.elasticsearch.plugins.internal to org.elasticsearch.metering, org.elasticsearch.settings.secure;
 
     provides java.util.spi.CalendarDataProvider with org.elasticsearch.common.time.IsoCalendarDataProvider;
     provides org.elasticsearch.xcontent.ErrorOnUnknown with org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown;

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -171,9 +171,11 @@ public record TransportVersion(int id) implements VersionId<TransportVersion> {
     public static final TransportVersion V_8_500_046 = registerTransportVersion(8_500_046, "61666d4c-a4f0-40db-8a3d-4806718247c5");
     public static final TransportVersion V_8_500_047 = registerTransportVersion(8_500_047, "4b1682fe-c37e-4184-80f6-7d57fcba9b3d");
     public static final TransportVersion V_8_500_048 = registerTransportVersion(8_500_048, "f9658aa5-f066-4edb-bcb9-40bf256c9294");
+    public static final TransportVersion V_8_500_049 = registerTransportVersion(8_500_049, "828bb6ce-2fbb-11ee-be56-0242ac120002");
+    public static final TransportVersion V_8_500_050 = registerTransportVersion(8_500_050, "69722fa2-7c0a-4227-86fb-6d6a9a0a0321");
 
     private static class CurrentHolder {
-        private static final TransportVersion CURRENT = findCurrent(V_8_500_048);
+        private static final TransportVersion CURRENT = findCurrent(V_8_500_050);
 
         // finds the pluggable current version, or uses the given fallback
         private static TransportVersion findCurrent(TransportVersion fallback) {

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java

@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion;
+import org.elasticsearch.cluster.coordination.Reconfigurator;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -57,6 +58,7 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNod
     );
 
     private volatile int maxVotingConfigExclusions;
+    private final Reconfigurator reconfigurator;
 
     @Inject
     public TransportAddVotingConfigExclusionsAction(
@@ -66,7 +68,8 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNod
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Reconfigurator reconfigurator
     ) {
         super(
             AddVotingConfigExclusionsAction.NAME,
@@ -83,6 +86,7 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNod
 
         maxVotingConfigExclusions = MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING, this::setMaxVotingConfigExclusions);
+        this.reconfigurator = reconfigurator;
     }
 
     private void setMaxVotingConfigExclusions(int maxVotingConfigExclusions) {
@@ -96,6 +100,7 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNod
         ClusterState state,
         ActionListener<ActionResponse.Empty> listener
     ) throws Exception {
+        reconfigurator.ensureVotingConfigCanBeModified();
 
         resolveVotingConfigExclusionsAndCheckMaximum(request, state, maxVotingConfigExclusions);
         // throws IAE if no nodes matched or maximum exceeded

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java

@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion;
+import org.elasticsearch.cluster.coordination.Reconfigurator;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -41,6 +42,7 @@ public class TransportClearVotingConfigExclusionsAction extends TransportMasterN
     ActionResponse.Empty> {
 
     private static final Logger logger = LogManager.getLogger(TransportClearVotingConfigExclusionsAction.class);
+    private final Reconfigurator reconfigurator;
 
     @Inject
     public TransportClearVotingConfigExclusionsAction(
@@ -48,7 +50,8 @@ public class TransportClearVotingConfigExclusionsAction extends TransportMasterN
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Reconfigurator reconfigurator
     ) {
         super(
             ClearVotingConfigExclusionsAction.NAME,
@@ -62,6 +65,7 @@ public class TransportClearVotingConfigExclusionsAction extends TransportMasterN
             in -> ActionResponse.Empty.INSTANCE,
             ThreadPool.Names.SAME
         );
+        this.reconfigurator = reconfigurator;
     }
 
     @Override
@@ -71,6 +75,7 @@ public class TransportClearVotingConfigExclusionsAction extends TransportMasterN
         ClusterState initialState,
         ActionListener<ActionResponse.Empty> listener
     ) throws Exception {
+        reconfigurator.ensureVotingConfigCanBeModified();
 
         final long startTimeMillis = threadPool.relativeTimeInMillis();
 

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -349,7 +349,8 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                 request.source(),
                 request.getContentType(),
                 request.routing(),
-                request.getDynamicTemplates()
+                request.getDynamicTemplates(),
+                request.pipelinesHaveRun() == false
             );
             result = primary.applyIndexOperationOnPrimary(
                 version,
@@ -617,7 +618,8 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                     indexRequest.source(),
                     indexRequest.getContentType(),
                     indexRequest.routing(),
-                    Map.of()
+                    Map.of(),
+                    false
                 );
                 result = replica.applyIndexOperationOnReplica(
                     primaryResponse.getSeqNo(),

server/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -34,6 +34,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -66,6 +67,7 @@ import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocWriteRequest<IndexRequest>, CompositeIndicesRequest {
 
     private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IndexRequest.class);
+    private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersion.V_8_500_049;
 
     /**
      * Max length of the source document to include into string()
@@ -119,6 +121,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
      * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised.
      */
     private Object rawTimestamp;
+    private boolean pipelinesHaveRun = false;
 
     public IndexRequest(StreamInput in) throws IOException {
         this(null, in);
@@ -160,6 +163,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             dynamicTemplates = in.readMap(StreamInput::readString);
         }
+        if (in.getTransportVersion().onOrAfter(PIPELINES_HAVE_RUN_FIELD_ADDED)) {
+            pipelinesHaveRun = in.readBoolean();
+        }
     }
 
     public IndexRequest() {
@@ -357,6 +363,10 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         return XContentHelper.convertToMap(source, false, contentType).v2();
     }
 
+    public Map<String, Object> sourceAsMap(DocumentParsingObserver documentParsingObserver) {
+        return XContentHelper.convertToMap(source, false, contentType, documentParsingObserver).v2();
+    }
+
     /**
      * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format
      *
@@ -713,6 +723,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
                 throw new IllegalArgumentException("[dynamic_templates] parameter requires all nodes on " + Version.V_7_13_0 + " or later");
             }
         }
+        if (out.getTransportVersion().onOrAfter(PIPELINES_HAVE_RUN_FIELD_ADDED)) {
+            out.writeBoolean(pipelinesHaveRun);
+        }
     }
 
     @Override
@@ -808,4 +821,12 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         assert this.rawTimestamp == null : "rawTimestamp only set in ingest phase, it can't be set twice";
         this.rawTimestamp = rawTimestamp;
     }
+
+    public void setPipelinesHaveRun() {
+        pipelinesHaveRun = true;
+    }
+
+    public boolean pipelinesHaveRun() {
+        return pipelinesHaveRun;
+    }
 }

server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java

@@ -148,6 +148,11 @@ public class Reconfigurator {
         return clusterState;
     }
 
+    public void ensureVotingConfigCanBeModified() {
+        // Temporary workaround until #98055 is tackled
+        // no-op
+    }
+
     record VotingConfigNode(String id, boolean live, boolean currentMaster, boolean inCurrentConfig)
         implements
             Comparable<VotingConfigNode> {

server/src/main/java/org/elasticsearch/cluster/coordination/stateless/SingleNodeReconfigurator.java

@@ -49,4 +49,10 @@ public class SingleNodeReconfigurator extends Reconfigurator {
             )
             .build();
     }
+
+    @Override
+    public void ensureVotingConfigCanBeModified() {
+        assert false;
+        throw new IllegalStateException("Voting configuration cannot be modified using atomic-register based coordination");
+    }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -264,10 +264,10 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
     public static final Setting<AutoExpandReplicas> INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING;
 
     public enum APIBlock implements Writeable {
-        READ_ONLY("read_only", INDEX_READ_ONLY_BLOCK),
-        READ("read", INDEX_READ_BLOCK),
-        WRITE("write", INDEX_WRITE_BLOCK),
-        METADATA("metadata", INDEX_METADATA_BLOCK),
+        READ_ONLY("read_only", INDEX_READ_ONLY_BLOCK, Property.ServerlessPublic),
+        READ("read", INDEX_READ_BLOCK, Property.ServerlessPublic),
+        WRITE("write", INDEX_WRITE_BLOCK, Property.ServerlessPublic),
+        METADATA("metadata", INDEX_METADATA_BLOCK, Property.ServerlessPublic),
         READ_ONLY_ALLOW_DELETE("read_only_allow_delete", INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
 
         final String name;
@@ -282,6 +282,13 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
             this.block = block;
         }
 
+        APIBlock(String name, ClusterBlock block, Property serverlessProperty) {
+            this.name = name;
+            this.settingName = "index.blocks." + name;
+            this.setting = Setting.boolSetting(settingName, false, Property.Dynamic, Property.IndexScope, serverlessProperty);
+            this.block = block;
+        }
+
         public String settingName() {
             return settingName;
         }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -25,6 +25,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -181,7 +182,8 @@ public class IndexMetadataVerifier {
                     mapperRegistry,
                     () -> null,
                     indexSettings.getMode().idFieldMapperWithoutFieldData(),
-                    scriptService
+                    scriptService,
+                    () -> DocumentParsingObserver.EMPTY_INSTANCE
                 )
             ) {
                 mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -502,6 +502,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
         ResourceWatcherService.RELOAD_INTERVAL_LOW,
         SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
         SearchModule.INDICES_MAX_NESTED_DEPTH_SETTING,
+        SearchModule.SEARCH_CONCURRENCY_ENABLED,
         ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
         ThreadPool.LATE_TIME_INTERVAL_WARN_THRESHOLD_SETTING,
         ThreadPool.SLOW_SCHEDULER_TASK_WARN_THRESHOLD_SETTING,

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -181,8 +181,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                     );
                 }
             }
-        }, Property.IndexScope), // this allows similarity settings to be passed
-        Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed
+        }, Property.IndexScope, Property.ServerlessPublic), // this allows similarity settings to be passed
+        Setting.groupSetting("index.analysis.", Property.IndexScope, Property.ServerlessPublic), // this allows analysis settings to be
+                                                                                                 // passed
 
         // TSDB index settings
         IndexSettings.MODE,

server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java

@@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -138,6 +139,21 @@ public class XContentHelper {
      * Exactly the same as {@link XContentHelper#convertToMap(BytesReference, boolean, XContentType, Set, Set)} but
      * none of the fields are filtered
      */
+    public static Tuple<XContentType, Map<String, Object>> convertToMap(
+        BytesReference bytes,
+        boolean ordered,
+        XContentType xContentType,
+        DocumentParsingObserver documentParsingObserver
+    ) {
+        return parseToType(
+            ordered ? XContentParser::mapOrdered : XContentParser::map,
+            bytes,
+            xContentType,
+            XContentParserConfiguration.EMPTY,
+            documentParsingObserver
+        );
+    }
+
     public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) {
         return parseToType(
             ordered ? XContentParser::mapOrdered : XContentParser::map,
@@ -181,10 +197,25 @@ public class XContentHelper {
         BytesReference bytes,
         @Nullable XContentType xContentType,
         @Nullable XContentParserConfiguration config
+    ) throws ElasticsearchParseException {
+        return parseToType(extractor, bytes, xContentType, config, DocumentParsingObserver.EMPTY_INSTANCE);
+    }
+
+    public static <T> Tuple<XContentType, T> parseToType(
+        CheckedFunction<XContentParser, T, IOException> extractor,
+        BytesReference bytes,
+        @Nullable XContentType xContentType,
+        @Nullable XContentParserConfiguration config,
+        DocumentParsingObserver documentParsingObserver
     ) throws ElasticsearchParseException {
         config = config != null ? config : XContentParserConfiguration.EMPTY;
-        try (XContentParser parser = xContentType != null ? createParser(config, bytes, xContentType) : createParser(config, bytes)) {
-            return new Tuple<>(parser.contentType(), extractor.apply(parser));
+        try (
+            XContentParser parser = documentParsingObserver.wrapParser(
+                xContentType != null ? createParser(config, bytes, xContentType) : createParser(config, bytes)
+            )
+        ) {
+            Tuple<XContentType, T> xContentTypeTTuple = new Tuple<>(parser.contentType(), extractor.apply(parser));
+            return xContentTypeTTuple;
         } catch (IOException e) {
             throw new ElasticsearchParseException("Failed to parse content to type", e);
         }

server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java

@@ -91,6 +91,7 @@ public class DiscoveryModule {
     );
 
     private final Coordinator coordinator;
+    private final Reconfigurator reconfigurator;
 
     public DiscoveryModule(
         Settings settings,
@@ -182,7 +183,7 @@ public class DiscoveryModule {
                 );
         }
 
-        var reconfigurator = getReconfigurator(settings, clusterSettings, clusterCoordinationPlugins);
+        this.reconfigurator = getReconfigurator(settings, clusterSettings, clusterCoordinationPlugins);
         var preVoteCollectorFactory = getPreVoteCollectorFactory(clusterCoordinationPlugins);
         var leaderHeartbeatService = getLeaderHeartbeatService(settings, clusterCoordinationPlugins);
 
@@ -282,4 +283,8 @@ public class DiscoveryModule {
     public Coordinator getCoordinator() {
         return coordinator;
     }
+
+    public Reconfigurator getReconfigurator() {
+        return reconfigurator;
+    }
 }

server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java

@@ -653,5 +653,10 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
         public InetSocketAddress getRemoteAddress() {
             return inner.getRemoteAddress();
         }
+
+        @Override
+        public String toString() {
+            return inner.toString();
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -57,6 +57,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -76,6 +77,7 @@ import java.util.function.BiFunction;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 /**
  * IndexModule represents the central extension point for index level custom implementations like:
@@ -163,6 +165,7 @@ public final class IndexModule {
     private final IndexSettings indexSettings;
     private final AnalysisRegistry analysisRegistry;
     private final EngineFactory engineFactory;
+    private final Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
     private final SetOnce<DirectoryWrapper> indexDirectoryWrapper = new SetOnce<>();
     private final SetOnce<Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>>> indexReaderWrapper =
         new SetOnce<>();
@@ -182,10 +185,11 @@ public final class IndexModule {
      * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
      * via {@link org.elasticsearch.plugins.Plugin#onIndexModule(IndexModule)}.
      *
-     * @param indexSettings       the index settings
-     * @param analysisRegistry    the analysis registry
-     * @param engineFactory       the engine factory
+     * @param indexSettings      the index settings
+     * @param analysisRegistry   the analysis registry
+     * @param engineFactory      the engine factory
      * @param directoryFactories the available store types
+     * @param documentParsingObserverSupplier the document reporter factory
      */
     public IndexModule(
         final IndexSettings indexSettings,
@@ -194,11 +198,13 @@ public final class IndexModule {
         final Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories,
         final BooleanSupplier allowExpensiveQueries,
         final IndexNameExpressionResolver expressionResolver,
-        final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories
+        final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
+        final Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         this.indexSettings = indexSettings;
         this.analysisRegistry = analysisRegistry;
         this.engineFactory = Objects.requireNonNull(engineFactory);
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
         this.searchOperationListeners.add(new SearchSlowLog(indexSettings));
         this.indexOperationListeners.add(new IndexingSlowLog(indexSettings));
         this.directoryFactories = Collections.unmodifiableMap(directoryFactories);
@@ -535,7 +541,8 @@ public final class IndexModule {
                 recoveryStateFactory,
                 indexFoldersDeletionListener,
                 snapshotCommitSupplier,
-                indexCommitListener.get()
+                indexCommitListener.get(),
+                documentParsingObserverSupplier
             );
             success = true;
             return indexService;
@@ -645,7 +652,8 @@ public final class IndexModule {
                 throw new UnsupportedOperationException("no index query shard context available");
             },
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
-            scriptService
+            scriptService,
+            documentParsingObserverSupplier
         );
     }
 

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -80,6 +80,7 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -155,6 +156,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final IndexNameExpressionResolver expressionResolver;
     private final Supplier<Sort> indexSortSupplier;
     private final ValuesSourceRegistry valuesSourceRegistry;
+    private Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
 
     public IndexService(
         IndexSettings indexSettings,
@@ -187,9 +189,11 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         IndexStorePlugin.RecoveryStateFactory recoveryStateFactory,
         IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener,
         IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
-        Engine.IndexCommitListener indexCommitListener
+        Engine.IndexCommitListener indexCommitListener,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         super(indexSettings);
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
         assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS
             : "IndexCreationContext.RELOAD_ANALYZERS should only be used when reloading analysers";
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -214,7 +218,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                 // we parse all percolator queries as they would be parsed on shard 0
                 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()),
                 idFieldMapper,
-                scriptService
+                scriptService,
+                documentParsingObserverSupplier
             );
             this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService);
             if (indexSettings.getIndexSortConfig().hasIndexSort()) {

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -57,12 +57,14 @@ public final class IndexSettings {
         "index.query.default_field",
         Collections.singletonList("*"),
         Property.IndexScope,
-        Property.Dynamic
+        Property.Dynamic,
+        Property.ServerlessPublic
     );
     public static final Setting<Boolean> QUERY_STRING_LENIENT_SETTING = Setting.boolSetting(
         "index.query_string.lenient",
         false,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final Setting<Boolean> QUERY_STRING_ANALYZE_WILDCARD = Setting.boolSetting(
         "indices.query.query_string.analyze_wildcard",
@@ -276,7 +278,7 @@ public final class IndexSettings {
             return STATELESS_DEFAULT_REFRESH_INTERVAL;
         }
         return DEFAULT_REFRESH_INTERVAL;
-    }, new RefreshIntervalValidator(), Property.Dynamic, Property.IndexScope);
+    }, new RefreshIntervalValidator(), Property.Dynamic, Property.IndexScope, Property.ServerlessPublic);
 
     static class RefreshIntervalValidator implements Setting.Validator<TimeValue> {
         @Override
@@ -496,7 +498,8 @@ public final class IndexSettings {
         IngestService.NOOP_PIPELINE_NAME,
         Function.identity(),
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
 
     public static final Setting<String> FINAL_PIPELINE = new Setting<>(
@@ -504,7 +507,8 @@ public final class IndexSettings {
         IngestService.NOOP_PIPELINE_NAME,
         Function.identity(),
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
 
     /**
@@ -552,7 +556,8 @@ public final class IndexSettings {
         -1,
         -1,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
     public static final Setting<Boolean> LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING = Setting.boolSetting(

server/src/main/java/org/elasticsearch/index/IndexSortConfig.java

@@ -61,7 +61,8 @@ public final class IndexSortConfig {
     public static final Setting<List<String>> INDEX_SORT_FIELD_SETTING = Setting.stringListSetting(
         "index.sort.field",
         Setting.Property.IndexScope,
-        Setting.Property.Final
+        Setting.Property.Final,
+        Setting.Property.ServerlessPublic
     );
 
     /**
@@ -72,7 +73,8 @@ public final class IndexSortConfig {
         Collections.emptyList(),
         IndexSortConfig::parseOrderMode,
         Setting.Property.IndexScope,
-        Setting.Property.Final
+        Setting.Property.Final,
+        Setting.Property.ServerlessPublic
     );
 
     /**
@@ -83,7 +85,8 @@ public final class IndexSortConfig {
         Collections.emptyList(),
         IndexSortConfig::parseMultiValueMode,
         Setting.Property.IndexScope,
-        Setting.Property.Final
+        Setting.Property.Final,
+        Setting.Property.ServerlessPublic
     );
 
     /**
@@ -94,7 +97,8 @@ public final class IndexSortConfig {
         Collections.emptyList(),
         IndexSortConfig::validateMissingValue,
         Setting.Property.IndexScope,
-        Setting.Property.Final
+        Setting.Property.Final,
+        Setting.Property.ServerlessPublic
     );
 
     public static final FieldSortSpec[] TIME_SERIES_SORT;

server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java

@@ -197,20 +197,23 @@ public final class MergePolicyConfig {
         DEFAULT_EXPUNGE_DELETES_ALLOWED,
         0.0d,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING = Setting.byteSizeSetting(
         "index.merge.policy.floor_segment",
         DEFAULT_FLOOR_SEGMENT,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING = Setting.intSetting(
         "index.merge.policy.max_merge_at_once",
         DEFAULT_MAX_MERGE_AT_ONCE,
         2,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING = Setting.intSetting(
         "index.merge.policy.max_merge_at_once_explicit",
@@ -247,7 +250,8 @@ public final class MergePolicyConfig {
         5.0d,
         50.0d,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     // don't convert to Setting<> and register... we only set this in tests and register via a plugin
     public static final String INDEX_MERGE_ENABLED = "index.merge.enabled";

server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java

@@ -106,7 +106,7 @@ public final class EngineConfig {
                 }
                 return s;
         }
-    }, Property.IndexScope, Property.NodeScope);
+    }, Property.IndexScope, Property.NodeScope, Property.ServerlessPublic);
 
     /**
      * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.

server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java

@@ -256,7 +256,8 @@ final class TranslogDirectoryReader extends DirectoryReader {
                     operation.source(),
                     XContentHelper.xContentType(operation.source()),
                     operation.routing(),
-                    Map.of()
+                    Map.of(),
+                    false
                 ),
                 mappingLookup
             );

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -19,6 +19,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -37,6 +38,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 /**
  * A parser for documents
@@ -44,11 +46,17 @@ import java.util.function.Consumer;
 public final class DocumentParser {
 
     private final XContentParserConfiguration parserConfiguration;
+    private final Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
     private final MappingParserContext mappingParserContext;
 
-    DocumentParser(XContentParserConfiguration parserConfiguration, MappingParserContext mappingParserContext) {
+    DocumentParser(
+        XContentParserConfiguration parserConfiguration,
+        MappingParserContext mappingParserContext,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
+    ) {
         this.mappingParserContext = mappingParserContext;
         this.parserConfiguration = parserConfiguration;
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
     }
 
     /**
@@ -65,7 +73,16 @@ public final class DocumentParser {
         }
         final RootDocumentParserContext context;
         final XContentType xContentType = source.getXContentType();
-        try (XContentParser parser = XContentHelper.createParser(parserConfiguration, source.source(), xContentType)) {
+
+        // only observe a document if it was not already reported (done in IngestService)
+        DocumentParsingObserver documentParsingObserver = source.toBeReported()
+            ? documentParsingObserverSupplier.get()
+            : DocumentParsingObserver.EMPTY_INSTANCE;
+        try (
+            XContentParser parser = documentParsingObserver.wrapParser(
+                XContentHelper.createParser(parserConfiguration, source.source(), xContentType)
+            )
+        ) {
             context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser);
             validateStart(context.parser());
             MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers();
@@ -79,6 +96,13 @@ public final class DocumentParser {
         }
         assert context.path.pathAsText("").isEmpty() : "found leftover path elements: " + context.path.pathAsText("");
 
+        Mapping dynamicUpdate = createDynamicUpdate(context);
+
+        // if a mappingUpdate is required, the parsing will be triggered again
+        if (dynamicUpdate == null) {
+            documentParsingObserver.setIndexName(mappingParserContext.getIndexSettings().getIndex().getName());
+            documentParsingObserver.close();
+        }
         return new ParsedDocument(
             context.version(),
             context.seqID(),
@@ -87,7 +111,7 @@ public final class DocumentParser {
             context.reorderParentAndGetDocs(),
             context.sourceToParse().source(),
             context.sourceToParse().getXContentType(),
-            createDynamicUpdate(context)
+            dynamicUpdate
         ) {
             @Override
             public String documentDescription() {
@@ -284,7 +308,6 @@ public final class DocumentParser {
         XContentParser.Token token = parser.currentToken();
         String currentFieldName = null;
         assert token == XContentParser.Token.FIELD_NAME || token == XContentParser.Token.END_OBJECT;
-
         while (token != XContentParser.Token.END_OBJECT) {
             if (token == null) {
                 throwEOF(context.parent(), context);

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -60,9 +60,15 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
     public static final Setting<Boolean> IGNORE_MALFORMED_SETTING = Setting.boolSetting(
         "index.mapping.ignore_malformed",
         false,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
+    );
+    public static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting(
+        "index.mapping.coerce",
+        false,
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
-    public static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", false, Property.IndexScope);
 
     protected static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldMapper.class);
     @SuppressWarnings("rawtypes")

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -27,6 +27,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -93,7 +94,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         1000L,
         0,
         Property.Dynamic,
-        Property.IndexScope
+        Property.IndexScope,
+        Property.ServerlessPublic
     );
     public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting(
         "index.mapping.depth.limit",
@@ -123,6 +125,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
     private final IndexVersion indexVersionCreated;
     private final MapperRegistry mapperRegistry;
     private final Supplier<MappingParserContext> mappingParserContextSupplier;
+    private final Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
+
     private volatile DocumentMapper mapper;
 
     public MapperService(
@@ -134,7 +138,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         MapperRegistry mapperRegistry,
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
-        ScriptCompiler scriptCompiler
+        ScriptCompiler scriptCompiler,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         this(
             () -> clusterService.state().getMinTransportVersion(),
@@ -145,7 +150,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             mapperRegistry,
             searchExecutionContextSupplier,
             idFieldMapper,
-            scriptCompiler
+            scriptCompiler,
+            documentParsingObserverSupplier
         );
     }
 
@@ -158,7 +164,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         MapperRegistry mapperRegistry,
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
-        ScriptCompiler scriptCompiler
+        ScriptCompiler scriptCompiler,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         super(indexSettings);
         this.indexVersionCreated = indexSettings.getIndexVersionCreated();
@@ -176,7 +183,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             indexSettings,
             idFieldMapper
         );
-        this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get());
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
+        this.documentParser = new DocumentParser(
+            parserConfiguration,
+            this.mappingParserContextSupplier.get(),
+            documentParsingObserverSupplier
+        );
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(
             indexSettings.getIndexVersionCreated()
         );

server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java

@@ -27,13 +27,15 @@ public class SourceToParse {
     private final XContentType xContentType;
 
     private final Map<String, String> dynamicTemplates;
+    private boolean toBeReported;
 
     public SourceToParse(
         @Nullable String id,
         BytesReference source,
         XContentType xContentType,
         @Nullable String routing,
-        Map<String, String> dynamicTemplates
+        Map<String, String> dynamicTemplates,
+        boolean toBeReported
     ) {
         this.id = id;
         // we always convert back to byte array, since we store it and Field only supports bytes..
@@ -42,10 +44,15 @@ public class SourceToParse {
         this.xContentType = Objects.requireNonNull(xContentType);
         this.routing = routing;
         this.dynamicTemplates = Objects.requireNonNull(dynamicTemplates);
+        this.toBeReported = toBeReported;
     }
 
     public SourceToParse(String id, BytesReference source, XContentType xContentType) {
-        this(id, source, xContentType, null, Map.of());
+        this(id, source, xContentType, null, Map.of(), false);
+    }
+
+    public boolean toBeReported() {
+        return toBeReported;
     }
 
     public BytesReference source() {

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -1858,7 +1858,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     index.getAutoGeneratedIdTimestamp(),
                     true,
                     origin,
-                    new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing(), Map.of())
+                    new SourceToParse(
+                        index.id(),
+                        index.source(),
+                        XContentHelper.xContentType(index.source()),
+                        index.routing(),
+                        Map.of(),
+                        false
+                    )
                 );
             }
             case DELETE -> {

server/src/main/java/org/elasticsearch/index/store/ByteSizeCachingDirectory.java

@@ -9,20 +9,16 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.elasticsearch.common.lucene.store.FilterIndexOutput;
 import org.elasticsearch.common.util.SingleObjectCache;
 import org.elasticsearch.core.TimeValue;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.nio.file.AccessDeniedException;
-import java.nio.file.NoSuchFileException;
 
-final class ByteSizeCachingDirectory extends FilterDirectory {
+final class ByteSizeCachingDirectory extends ByteSizeDirectory {
 
     private static class SizeAndModCount {
         final long size;
@@ -36,20 +32,6 @@ final class ByteSizeCachingDirectory extends FilterDirectory {
         }
     }
 
-    private static long estimateSizeInBytes(Directory directory) throws IOException {
-        long estimatedSize = 0;
-        String[] files = directory.listAll();
-        for (String file : files) {
-            try {
-                estimatedSize += directory.fileLength(file);
-            } catch (NoSuchFileException | FileNotFoundException | AccessDeniedException e) {
-                // ignore, the file is not there no more; on Windows, if one thread concurrently deletes a file while
-                // calling Files.size, you can also sometimes hit AccessDeniedException
-            }
-        }
-        return estimatedSize;
-    }
-
     private final SingleObjectCache<SizeAndModCount> size;
     // Both these variables need to be accessed under `this` lock.
     private long modCount = 0;
@@ -57,7 +39,7 @@ final class ByteSizeCachingDirectory extends FilterDirectory {
 
     ByteSizeCachingDirectory(Directory in, TimeValue refreshInterval) {
         super(in);
-        size = new SingleObjectCache<SizeAndModCount>(refreshInterval, new SizeAndModCount(0L, -1L, true)) {
+        size = new SingleObjectCache<>(refreshInterval, new SizeAndModCount(0L, -1L, true)) {
             @Override
             protected SizeAndModCount refresh() {
                 // It is ok for the size of the directory to be more recent than
@@ -103,8 +85,8 @@ final class ByteSizeCachingDirectory extends FilterDirectory {
         };
     }
 
-    /** Return the cumulative size of all files in this directory. */
-    long estimateSizeInBytes() throws IOException {
+    @Override
+    public long estimateSizeInBytes() throws IOException {
         try {
             return size.getOrRefresh().size;
         } catch (UncheckedIOException e) {
@@ -113,6 +95,11 @@ final class ByteSizeCachingDirectory extends FilterDirectory {
         }
     }
 
+    @Override
+    public long estimateDataSetSizeInBytes() throws IOException {
+        return estimateSizeInBytes(); // data set size is equal to directory size for most implementations
+    }
+
     @Override
     public IndexOutput createOutput(String name, IOContext context) throws IOException {
         return wrapIndexOutput(super.createOutput(name, context));

server/src/main/java/org/elasticsearch/index/store/ByteSizeDirectory.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.store;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.NoSuchFileException;
+
+public abstract class ByteSizeDirectory extends FilterDirectory {
+
+    protected static long estimateSizeInBytes(Directory directory) throws IOException {
+        long estimatedSize = 0;
+        String[] files = directory.listAll();
+        for (String file : files) {
+            try {
+                estimatedSize += directory.fileLength(file);
+            } catch (NoSuchFileException | FileNotFoundException | AccessDeniedException e) {
+                // ignore, the file is not there no more; on Windows, if one thread concurrently deletes a file while
+                // calling Files.size, you can also sometimes hit AccessDeniedException
+            }
+        }
+        return estimatedSize;
+    }
+
+    protected ByteSizeDirectory(Directory in) {
+        super(in);
+    }
+
+    /**
+     * @return the size of the directory
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    public abstract long estimateSizeInBytes() throws IOException;
+
+    /**
+     * @return the size of the total data set of the directory (which can differ from {{@link #estimateSizeInBytes()}})
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    public abstract long estimateDataSetSizeInBytes() throws IOException;
+
+}

server/src/main/java/org/elasticsearch/index/store/Store.java

@@ -27,7 +27,6 @@ import org.apache.lucene.store.BufferedChecksum;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -162,10 +161,10 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
 
     public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, ShardLock shardLock, OnClose onClose) {
         super(shardId, indexSettings);
-        final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING);
-        logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval);
-        ByteSizeCachingDirectory sizeCachingDir = new ByteSizeCachingDirectory(directory, refreshInterval);
-        this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId));
+        this.directory = new StoreDirectory(
+            byteSizeDirectory(directory, indexSettings, logger),
+            Loggers.getLogger("index.store.deletes", shardId)
+        );
         this.shardLock = shardLock;
         this.onClose = onClose;
 
@@ -355,8 +354,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
      */
     public StoreStats stats(long reservedBytes, LongUnaryOperator localSizeFunction) throws IOException {
         ensureOpen();
-        long sizeInBytes = directory.estimateSize();
-        return new StoreStats(localSizeFunction.applyAsLong(sizeInBytes), sizeInBytes, reservedBytes);
+        long sizeInBytes = directory.estimateSizeInBytes();
+        long dataSetSizeInBytes = directory.estimateDataSetSizeInBytes();
+        return new StoreStats(localSizeFunction.applyAsLong(sizeInBytes), dataSetSizeInBytes, reservedBytes);
     }
 
     /**
@@ -443,6 +443,16 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         }
     }
 
+    private static ByteSizeDirectory byteSizeDirectory(Directory directory, IndexSettings indexSettings, Logger logger) {
+        if (directory instanceof ByteSizeDirectory byteSizeDirectory) {
+            return byteSizeDirectory;
+        } else {
+            final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING);
+            logger.debug("store stats are refreshed with {} [{}]", INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval);
+            return new ByteSizeCachingDirectory(directory, refreshInterval);
+        }
+    }
+
     /**
      * Reads a MetadataSnapshot from the given index locations or returns an empty snapshot if it can't be read.
      *
@@ -720,18 +730,23 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         shardLock.setDetails("closing shard");
     }
 
-    static final class StoreDirectory extends FilterDirectory {
+    static final class StoreDirectory extends ByteSizeDirectory {
 
         private final Logger deletesLogger;
 
-        StoreDirectory(ByteSizeCachingDirectory delegateDirectory, Logger deletesLogger) {
+        StoreDirectory(ByteSizeDirectory delegateDirectory, Logger deletesLogger) {
             super(delegateDirectory);
             this.deletesLogger = deletesLogger;
         }
 
-        /** Estimate the cumulative size of all files in this directory in bytes. */
-        long estimateSize() throws IOException {
-            return ((ByteSizeCachingDirectory) getDelegate()).estimateSizeInBytes();
+        @Override
+        public long estimateSizeInBytes() throws IOException {
+            return ((ByteSizeDirectory) getDelegate()).estimateSizeInBytes();
+        }
+
+        @Override
+        public long estimateDataSetSizeInBytes() throws IOException {
+            return ((ByteSizeDirectory) getDelegate()).estimateDataSetSizeInBytes();
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java

@@ -304,7 +304,14 @@ public class TermVectorsService {
     }
 
     private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
-        SourceToParse source = new SourceToParse("_id_for_tv_api", request.doc(), request.xContentType(), request.routing(), Map.of());
+        SourceToParse source = new SourceToParse(
+            "_id_for_tv_api",
+            request.doc(),
+            request.xContentType(),
+            request.routing(),
+            Map.of(),
+            false
+        );
         DocumentParser documentParser = indexShard.mapperService().documentParser();
         MappingLookup mappingLookup = indexShard.mapperService().mappingLookup();
         ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup);

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -127,6 +127,7 @@ import org.elasticsearch.indices.store.CompositeIndexFoldersDeletionListener;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -169,6 +170,7 @@ import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
@@ -230,6 +232,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final OldShardsStats oldShardsStats = new OldShardsStats();
     private final MapperRegistry mapperRegistry;
     private final NamedWriteableRegistry namedWriteableRegistry;
+    private final Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
     private final Map<String, IndexStorePlugin.SnapshotCommitSupplier> snapshotCommitSuppliers;
     private final IndexingMemoryController indexingMemoryController;
     private final TimeValue cleanInterval;
@@ -287,7 +290,8 @@ public class IndicesService extends AbstractLifecycleComponent
         Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
         List<IndexStorePlugin.IndexFoldersDeletionListener> indexFoldersDeletionListeners,
         Map<String, IndexStorePlugin.SnapshotCommitSupplier> snapshotCommitSuppliers,
-        CheckedBiConsumer<ShardSearchRequest, StreamOutput, IOException> requestCacheKeyDifferentiator
+        CheckedBiConsumer<ShardSearchRequest, StreamOutput, IOException> requestCacheKeyDifferentiator,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -303,6 +307,7 @@ public class IndicesService extends AbstractLifecycleComponent
         this.indicesQueryCache = new IndicesQueryCache(settings);
         this.mapperRegistry = mapperRegistry;
         this.namedWriteableRegistry = namedWriteableRegistry;
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
         indexingMemoryController = new IndexingMemoryController(
             settings,
             threadPool,
@@ -756,7 +761,8 @@ public class IndicesService extends AbstractLifecycleComponent
             directoryFactories,
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
-            recoveryStateFactories
+            recoveryStateFactories,
+            documentParsingObserverSupplier
         );
         for (IndexingOperationListener operationListener : indexingOperationListeners) {
             indexModule.addIndexOperationListener(operationListener);
@@ -832,7 +838,8 @@ public class IndicesService extends AbstractLifecycleComponent
             directoryFactories,
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
-            recoveryStateFactories
+            recoveryStateFactories,
+            documentParsingObserverSupplier
         );
         pluginsService.forEach(p -> p.onIndexModule(indexModule));
         return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService);

server/src/main/java/org/elasticsearch/ingest/IngestService.java

@@ -59,6 +59,7 @@ import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.node.ReportingService;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -85,6 +86,7 @@ import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
@@ -103,6 +105,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
     private final MasterServiceTaskQueue<PipelineClusterStateUpdateTask> taskQueue;
     private final ClusterService clusterService;
     private final ScriptService scriptService;
+    private final Supplier<DocumentParsingObserver> documentParsingObserverSupplier;
     private final Map<String, Processor.Factory> processorFactories;
     // Ideally this should be in IngestMetadata class, but we don't have the processor factories around there.
     // We know of all the processor factories when a node with all its plugin have been initialized. Also some
@@ -177,10 +180,12 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
         AnalysisRegistry analysisRegistry,
         List<IngestPlugin> ingestPlugins,
         Client client,
-        MatcherWatchdog matcherWatchdog
+        MatcherWatchdog matcherWatchdog,
+        Supplier<DocumentParsingObserver> documentParsingObserverSupplier
     ) {
         this.clusterService = clusterService;
         this.scriptService = scriptService;
+        this.documentParsingObserverSupplier = documentParsingObserverSupplier;
         this.processorFactories = processorFactories(
             ingestPlugins,
             new Processor.Parameters(
@@ -709,9 +714,16 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
                             totalMetrics.postIngest(ingestTimeInNanos);
                             ref.close();
                         });
+                        DocumentParsingObserver documentParsingObserver = documentParsingObserverSupplier.get();
+
+                        IngestDocument ingestDocument = newIngestDocument(indexRequest, documentParsingObserver);
 
-                        IngestDocument ingestDocument = newIngestDocument(indexRequest);
                         executePipelines(pipelines, indexRequest, ingestDocument, documentListener);
+                        indexRequest.setPipelinesHaveRun();
+
+                        documentParsingObserver.setIndexName(indexRequest.index());
+                        documentParsingObserver.close();
+
                         i++;
                     }
                 }
@@ -999,14 +1011,14 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
     /**
      * Builds a new ingest document from the passed-in index request.
      */
-    private static IngestDocument newIngestDocument(final IndexRequest request) {
+    private static IngestDocument newIngestDocument(final IndexRequest request, DocumentParsingObserver documentParsingObserver) {
         return new IngestDocument(
             request.index(),
             request.id(),
             request.version(),
             request.routing(),
             request.versionType(),
-            request.sourceAsMap()
+            request.sourceAsMap(documentParsingObserver)
         );
     }
 

server/src/main/java/org/elasticsearch/node/Node.java

@@ -44,6 +44,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService;
 import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.coordination.MasterHistoryService;
+import org.elasticsearch.cluster.coordination.Reconfigurator;
 import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService;
 import org.elasticsearch.cluster.desirednodes.DesiredNodesSettingsValidator;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
@@ -178,6 +179,8 @@ import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.plugins.ShutdownAwarePlugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.plugins.TracerPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
+import org.elasticsearch.plugins.internal.DocumentParsingObserverPlugin;
 import org.elasticsearch.plugins.internal.ReloadAwarePlugin;
 import org.elasticsearch.readiness.ReadinessService;
 import org.elasticsearch.repositories.RepositoriesModule;
@@ -245,6 +248,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -518,6 +522,9 @@ public class Node implements Closeable {
                     new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher()
                 );
             }
+
+            Supplier<DocumentParsingObserver> documentParsingObserverSupplier = getDocumentParsingObserverSupplier();
+
             final IngestService ingestService = new IngestService(
                 clusterService,
                 threadPool,
@@ -526,7 +533,8 @@ public class Node implements Closeable {
                 analysisModule.getAnalysisRegistry(),
                 pluginsService.filterPlugins(IngestPlugin.class),
                 client,
-                IngestService.createGrokThreadWatchdog(this.environment, threadPool)
+                IngestService.createGrokThreadWatchdog(this.environment, threadPool),
+                documentParsingObserverSupplier
             );
             final SetOnce<RepositoriesService> repositoriesServiceReference = new SetOnce<>();
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
@@ -685,7 +693,8 @@ public class Node implements Closeable {
                 recoveryStateFactories,
                 indexFoldersDeletionListeners,
                 snapshotCommitSuppliers,
-                searchModule.getRequestCacheKeyDifferentiator()
+                searchModule.getRequestCacheKeyDifferentiator(),
+                documentParsingObserverSupplier
             );
 
             final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation);
@@ -1081,6 +1090,7 @@ public class Node implements Closeable {
                 b.bind(SnapshotsInfoService.class).toInstance(snapshotsInfoService);
                 b.bind(GatewayMetaState.class).toInstance(gatewayMetaState);
                 b.bind(Coordinator.class).toInstance(discoveryModule.getCoordinator());
+                b.bind(Reconfigurator.class).toInstance(discoveryModule.getReconfigurator());
                 {
                     processRecoverySettings(settingsModule.getClusterSettings(), recoverySettings);
                     final SnapshotFilesProvider snapshotFilesProvider = new SnapshotFilesProvider(repositoryService);
@@ -1197,6 +1207,16 @@ public class Node implements Closeable {
         }
     }
 
+    private Supplier<DocumentParsingObserver> getDocumentParsingObserverSupplier() {
+        List<DocumentParsingObserverPlugin> plugins = pluginsService.filterPlugins(DocumentParsingObserverPlugin.class);
+        if (plugins.size() == 1) {
+            return plugins.get(0).getDocumentParsingObserverSupplier();
+        } else if (plugins.size() == 0) {
+            return () -> DocumentParsingObserver.EMPTY_INSTANCE;
+        }
+        throw new IllegalStateException("too many DocumentParsingObserverPlugin instances");
+    }
+
     /**
      * If the JVM was started with the Elastic APM agent and a config file argument was specified, then
      * delete the config file. The agent only reads it once, when supplied in this fashion, and it

server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingObserver.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.Closeable;
+
+/**
+ * An interface to allow wrapping an XContentParser and observe the events emitted while parsing
+ * A default implementation returns a noop DocumentParsingObserver - does not wrap a XContentParser and
+ * does not do anything upon finishing parsing.
+ */
+public interface DocumentParsingObserver extends Closeable {
+    /**
+     * a default noop implementation
+     */
+    DocumentParsingObserver EMPTY_INSTANCE = new DocumentParsingObserver() {
+        @Override
+        public XContentParser wrapParser(XContentParser xContentParser) {
+            return xContentParser;
+        }
+
+        @Override
+        public void setIndexName(String indexName) {}
+
+        @Override
+        public void close() {}
+    };
+
+    /**
+     * Decorates a provided xContentParser with additional logic (gather some state).
+     * The Decorator parser should use a state from DocumentParsingObserver
+     * in order to perform an action upon finished parsing which will be aware of the state
+     * gathered during parsing
+     *
+     * @param xContentParser to be decorated
+     * @return a decorator xContentParser
+     */
+    XContentParser wrapParser(XContentParser xContentParser);
+
+    /**
+     * Sets an indexName associated with parsed document.
+     * @param indexName an index name that is associated with the parsed document
+     */
+    void setIndexName(String indexName);
+
+    /**
+     * An action to be performed upon finished parsing.
+     */
+    void close();
+}

server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingObserverPlugin.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import java.util.function.Supplier;
+
+/**
+ * An internal plugin that will return a supplier of DocumentParsingObserver.
+ */
+public interface DocumentParsingObserverPlugin {
+
+    /**
+     * @return a supplier of DocumentParsingObserver to allow observing parsing events
+     */
+    Supplier<DocumentParsingObserver> getDocumentParsingObserverSupplier();
+}

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -149,7 +149,6 @@ import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.core.Strings.format;
@@ -1385,11 +1384,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                     finalizeSnapshotContext::updatedClusterState,
                     ActionListener.wrap(newRepoData -> {
                         finalizeSnapshotContext.onResponse(newRepoData);
-                        if (writeShardGens) {
-                            cleanupOldShardGens(existingRepositoryData, newRepoData, finalizeSnapshotContext, snapshotInfo);
-                        } else {
-                            finalizeSnapshotContext.onDone(snapshotInfo);
-                        }
+                        cleanupOldMetadata(existingRepositoryData, newRepoData, finalizeSnapshotContext, snapshotInfo, writeShardGens);
                     }, onUpdateFailure)
                 );
             }, onUpdateFailure))) {
@@ -1442,30 +1437,43 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }, onUpdateFailure));
     }
 
-    // Delete all old shard gen blobs that aren't referenced any longer as a result from moving to updated repository data
-    private void cleanupOldShardGens(
+    // Delete all old shard gen and root level index blobs that aren't referenced any longer as a result from moving to updated
+    // repository data
+    private void cleanupOldMetadata(
         RepositoryData existingRepositoryData,
         RepositoryData updatedRepositoryData,
         FinalizeSnapshotContext finalizeSnapshotContext,
-        SnapshotInfo snapshotInfo
+        SnapshotInfo snapshotInfo,
+        boolean writeShardGenerations
     ) {
         final Set<String> toDelete = new HashSet<>();
-        final int prefixPathLen = basePath().buildAsString().length();
-        updatedRepositoryData.shardGenerations()
-            .obsoleteShardGenerations(existingRepositoryData.shardGenerations())
-            .forEach(
-                (indexId, gens) -> gens.forEach(
-                    (shardId, oldGen) -> toDelete.add(
-                        shardPath(indexId, shardId).buildAsString().substring(prefixPathLen) + INDEX_FILE_PREFIX + oldGen
+        // Delete all now outdated index files up to 1000 blobs back from the new generation.
+        // If there are more than 1000 dangling index-N cleanup functionality on repo delete will take care of them.
+        long newRepoGeneration = updatedRepositoryData.getGenId();
+        for (long gen = Math.max(
+            Math.max(existingRepositoryData.getGenId() - 1, 0),
+            newRepoGeneration - 1000
+        ); gen < newRepoGeneration; gen++) {
+            toDelete.add(INDEX_FILE_PREFIX + gen);
+        }
+        if (writeShardGenerations) {
+            final int prefixPathLen = basePath().buildAsString().length();
+            updatedRepositoryData.shardGenerations()
+                .obsoleteShardGenerations(existingRepositoryData.shardGenerations())
+                .forEach(
+                    (indexId, gens) -> gens.forEach(
+                        (shardId, oldGen) -> toDelete.add(
+                            shardPath(indexId, shardId).buildAsString().substring(prefixPathLen) + INDEX_FILE_PREFIX + oldGen
+                        )
                     )
-                )
-            );
-        for (Map.Entry<RepositoryShardId, Set<ShardGeneration>> obsoleteEntry : finalizeSnapshotContext.obsoleteShardGenerations()
-            .entrySet()) {
-            final String containerPath = shardPath(obsoleteEntry.getKey().index(), obsoleteEntry.getKey().shardId()).buildAsString()
-                .substring(prefixPathLen) + INDEX_FILE_PREFIX;
-            for (ShardGeneration shardGeneration : obsoleteEntry.getValue()) {
-                toDelete.add(containerPath + shardGeneration);
+                );
+            for (Map.Entry<RepositoryShardId, Set<ShardGeneration>> obsoleteEntry : finalizeSnapshotContext.obsoleteShardGenerations()
+                .entrySet()) {
+                final String containerPath = shardPath(obsoleteEntry.getKey().index(), obsoleteEntry.getKey().shardId()).buildAsString()
+                    .substring(prefixPathLen) + INDEX_FILE_PREFIX;
+                for (ShardGeneration shardGeneration : obsoleteEntry.getValue()) {
+                    toDelete.add(containerPath + shardGeneration);
+                }
             }
         }
         if (toDelete.isEmpty() == false) {
@@ -1473,7 +1481,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 try {
                     deleteFromContainer(blobContainer(), toDelete.iterator());
                 } catch (Exception e) {
-                    logger.warn("Failed to clean up old shard generation blobs", e);
+                    logger.warn("Failed to clean up old metadata blobs", e);
                 } finally {
                     finalizeSnapshotContext.onDone(snapshotInfo);
                 }
@@ -2377,24 +2385,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
                     logger.trace("[{}] successfully set safe repository generation to [{}]", metadata.name(), newGen);
                     cacheRepositoryData(newRepositoryData, version);
-                    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() {
-                        @Override
-                        public void onFailure(Exception e) {
-                            logger.warn(() -> "Failed to clean up old index blobs from before [" + newGen + "]", e);
-                        }
-
-                        @Override
-                        protected void doRun() throws Exception {
-                            // Delete all now outdated index files up to 1000 blobs back from the new generation.
-                            // If there are more than 1000 dangling index-N cleanup functionality on repo delete will take care of them.
-                            deleteFromContainer(
-                                blobContainer(),
-                                LongStream.range(Math.max(Math.max(expectedGen - 1, 0), newGen - 1000), newGen)
-                                    .mapToObj(gen -> INDEX_FILE_PREFIX + gen)
-                                    .iterator()
-                            );
-                        }
-                    });
                     delegate.onResponse(newRepositoryData);
                 }
             });

server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

@@ -138,7 +138,8 @@ final class DefaultSearchContext extends SearchContext {
         TimeValue timeout,
         int minimumDocsPerSlice,
         FetchPhase fetchPhase,
-        boolean lowLevelCancellation
+        boolean lowLevelCancellation,
+        boolean parallelize
     ) throws IOException {
         this.readerContext = readerContext;
         this.request = request;
@@ -156,6 +157,10 @@ final class DefaultSearchContext extends SearchContext {
             engineSearcher.getQueryCachingPolicy(),
             minimumDocsPerSlice,
             lowLevelCancellation,
+            // TODO not set the for now, this needs a special thread pool and can be enabled after its introduction
+            // parallelize
+            // ? (EsThreadPoolExecutor) this.indexService.getThreadPool().executor(ThreadPool.Names.CONCURRENT_COLLECTION_TBD)
+            // : null,
             null
         );
         releasables.addAll(List.of(engineSearcher, searcher));

server/src/main/java/org/elasticsearch/search/SearchModule.java

@@ -281,6 +281,13 @@ public class SearchModule {
         Setting.Property.NodeScope
     );
 
+    public static final Setting<Boolean> SEARCH_CONCURRENCY_ENABLED = Setting.boolSetting(
+        "search.concurrency_enabled",
+        true,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     private final Map<String, Highlighter> highlighters;
 
     private final List<FetchSubPhase> fetchSubPhases = new ArrayList<>();

server/src/main/java/org/elasticsearch/search/SearchService.java

@@ -141,6 +141,7 @@ import static org.elasticsearch.core.TimeValue.timeValueHours;
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.core.TimeValue.timeValueMinutes;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+import static org.elasticsearch.search.SearchModule.SEARCH_CONCURRENCY_ENABLED;
 
 public class SearchService extends AbstractLifecycleComponent implements IndexEventListener {
     private static final Logger logger = LogManager.getLogger(SearchService.class);
@@ -252,6 +253,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
     private final DfsPhase dfsPhase = new DfsPhase();
 
     private final FetchPhase fetchPhase;
+    private volatile boolean enableConcurrentCollection;
 
     private volatile long defaultKeepAlive;
 
@@ -341,6 +343,17 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         enableRewriteAggsToFilterByFilter = ENABLE_REWRITE_AGGS_TO_FILTER_BY_FILTER.get(settings);
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(ENABLE_REWRITE_AGGS_TO_FILTER_BY_FILTER, this::setEnableRewriteAggsToFilterByFilter);
+
+        enableConcurrentCollection = SEARCH_CONCURRENCY_ENABLED.get(settings);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_CONCURRENCY_ENABLED, this::setEnableConcurrentCollection);
+    }
+
+    private void setEnableConcurrentCollection(boolean concurrentCollection) {
+        this.enableConcurrentCollection = concurrentCollection;
+    }
+
+    boolean isConcurrentCollectionEnabled() {
+        return this.enableConcurrentCollection;
     }
 
     private static void validateKeepAlives(TimeValue defaultKeepAlive, TimeValue maxKeepAlive) {
@@ -456,8 +469,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private DfsSearchResult executeDfsPhase(ShardSearchRequest request, SearchShardTask task) throws IOException {
         ReaderContext readerContext = createOrGetReaderContext(request);
-        try (
-            Releasable scope = tracer.withScope(task);
+        try (@SuppressWarnings("unused") // withScope call is necessary to instrument search execution
+        Releasable scope = tracer.withScope(task);
             Releasable ignored = readerContext.markAsUsed(getKeepAlive(request));
             SearchContext context = createContext(readerContext, request, task, ResultsType.DFS, false)
         ) {
@@ -994,7 +1007,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         boolean includeAggregations
     ) throws IOException {
         checkCancelled(task);
-        final DefaultSearchContext context = createSearchContext(readerContext, request, defaultSearchTimeout);
+        final DefaultSearchContext context = createSearchContext(readerContext, request, defaultSearchTimeout, resultsType);
         resultsType.addResultsObject(context);
         try {
             if (request.scroll() != null) {
@@ -1026,15 +1039,19 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         final Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
         final ShardSearchContextId id = new ShardSearchContextId(sessionId, idGenerator.incrementAndGet());
         try (ReaderContext readerContext = new ReaderContext(id, indexService, indexShard, reader, -1L, true)) {
-            DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout);
+            DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout, null);
             searchContext.addReleasable(readerContext.markAsUsed(0L));
             return searchContext;
         }
     }
 
     @SuppressWarnings("unchecked")
-    private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSearchRequest request, TimeValue timeout)
-        throws IOException {
+    private DefaultSearchContext createSearchContext(
+        ReaderContext reader,
+        ShardSearchRequest request,
+        TimeValue timeout,
+        ResultsType resultsType
+    ) throws IOException {
         boolean success = false;
         DefaultSearchContext searchContext = null;
         try {
@@ -1051,7 +1068,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 timeout,
                 minimumDocsPerSlice,
                 fetchPhase,
-                lowLevelCancellation
+                lowLevelCancellation,
+                this.enableConcurrentCollection && concurrentSearchEnabled(resultsType, request.source())
             );
             // we clone the query shard context here just for rewriting otherwise we
             // might end up with incorrect state since we are using now() or script services
@@ -1071,6 +1089,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         return searchContext;
     }
 
+    static boolean concurrentSearchEnabled(ResultsType resultsType, SearchSourceBuilder source) {
+        if (resultsType == ResultsType.DFS) {
+            return true; // only enable concurrent collection for DFS phase for now
+        }
+        return false;
+    }
+
     private void freeAllContextForIndex(Index index) {
         assert index != null;
         for (ReaderContext ctx : activeReaders.values()) {

server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java

@@ -333,6 +333,18 @@ public class AggregatorFactories {
             return false;
         }
 
+        /**
+         * Return false if this aggregation or any of the child aggregations does not support concurrent search
+         */
+        public boolean supportsConcurrentExecution() {
+            for (AggregationBuilder builder : aggregationBuilders) {
+                if (builder.supportsConcurrentExecution() == false) {
+                    return false;
+                }
+            }
+            return isInSortOrderExecutionRequired() == false;
+        }
+
         public Builder addAggregator(AggregationBuilder factory) {
             if (names.add(factory.name) == false) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");

server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java

@@ -31,7 +31,9 @@ public class GlobalAggregator extends BucketsAggregator implements SingleBucketA
         throws IOException {
 
         super(name, subFactories, context, null, CardinalityUpperBound.ONE, metadata);
-        weight = context.filterQuery(new MatchAllDocsQuery()).createWeight(context.searcher(), scoreMode(), 1.0f);
+        weight = context.searcher()
+            .rewrite(context.filterQuery(new MatchAllDocsQuery()))
+            .createWeight(context.searcher(), scoreMode(), 1.0f);
     }
 
     @Override

server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.search.internal;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -65,6 +67,9 @@ import java.util.concurrent.ThreadPoolExecutor;
  * Context-aware extension of {@link IndexSearcher}.
  */
 public class ContextIndexSearcher extends IndexSearcher implements Releasable {
+
+    private static final Logger logger = LogManager.getLogger(ContextIndexSearcher.class);
+
     /**
      * The interval at which we check for search cancellation when we cannot use
      * a {@link CancellableBulkScorer}. See {@link #intersectScorerAndBitSet}.
@@ -138,15 +143,24 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
         boolean wrapWithExitableDirectoryReader,
         ThreadPoolExecutor executor
     ) throws IOException {
-        // concurrency is handle in this class so don't pass the executor to the parent class
-        super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader);
+        // we need to pass the executor up so it can potentially be used as a sliceExecutor by knn search
+        super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader, executor);
         setSimilarity(similarity);
         setQueryCache(queryCache);
         setQueryCachingPolicy(queryCachingPolicy);
         this.cancellable = cancellable;
         this.queueSizeBasedExecutor = executor != null ? new QueueSizeBasedExecutor(executor) : null;
         this.minimumDocsPerSlice = minimumDocsPerSlice;
-        this.leafSlices = executor == null ? null : slices(leafContexts);
+        if (executor != null) {
+            this.leafSlices = computeSlices(
+                getLeafContexts(),
+                queueSizeBasedExecutor.threadPoolExecutor.getMaximumPoolSize(),
+                minimumDocsPerSlice
+            );
+            assert (this.leafSlices.length <= executor.getMaximumPoolSize());
+        } else {
+            this.leafSlices = null;
+        }
     }
 
     // package private for testing
@@ -228,9 +242,15 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
         }
     }
 
+    /**
+     * Overwrite superclass to force one slice per segment for knn search.
+     * This is only needed temporarily by knn query rewrite, for the main
+     * search collection we forked the search method and inject our own slicing logic
+     * until this is available in Lucene itself
+     */
     @Override
     protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
-        return computeSlices(leaves, queueSizeBasedExecutor.threadPoolExecutor.getMaximumPoolSize(), minimumDocsPerSlice);
+        return IndexSearcher.slices(leaves, Math.max(1, leaves.size()), 1);
     }
 
     /**
@@ -342,6 +362,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
 
                 listTasks.add(task);
             }
+            logger.trace("Collecting using " + listTasks.size() + " tasks.");
 
             queueSizeBasedExecutor.invokeAll(listTasks);
             RuntimeException exception = null;

server/src/main/java/org/elasticsearch/search/query/PartialHitCountCollector.java

@@ -29,23 +29,20 @@ class PartialHitCountCollector extends TotalHitCountCollector {
     private final HitsThresholdChecker hitsThresholdChecker;
     private boolean earlyTerminated;
 
-    PartialHitCountCollector(int totalHitsThreshold) {
-        this(new HitsThresholdChecker(totalHitsThreshold));
-        assert totalHitsThreshold != Integer.MAX_VALUE : "use TotalHitCountCollector for exact total hits tracking";
-    }
-
     PartialHitCountCollector(HitsThresholdChecker hitsThresholdChecker) {
         this.hitsThresholdChecker = hitsThresholdChecker;
     }
 
     @Override
     public ScoreMode scoreMode() {
-        // Does not need scores like TotalHitCountCollector (COMPLETE_NO_SCORES), but not exhaustive as it early terminates.
-        return ScoreMode.TOP_DOCS;
+        return hitsThresholdChecker.totalHitsThreshold == Integer.MAX_VALUE ? super.scoreMode() : ScoreMode.TOP_DOCS;
     }
 
     @Override
     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+        if (hitsThresholdChecker.totalHitsThreshold == Integer.MAX_VALUE) {
+            return super.getLeafCollector(context);
+        }
         earlyTerminateIfNeeded();
         return new FilterLeafCollector(super.getLeafCollector(context)) {
             @Override
@@ -68,12 +65,11 @@ class PartialHitCountCollector extends TotalHitCountCollector {
         return earlyTerminated;
     }
 
-    private static class HitsThresholdChecker {
+    static class HitsThresholdChecker {
         private final int totalHitsThreshold;
         private final AtomicInteger numCollected = new AtomicInteger();
 
         HitsThresholdChecker(int totalHitsThreshold) {
-            assert totalHitsThreshold != Integer.MAX_VALUE : "use TotalHitCountCollector for exact total hits tracking";
             this.totalHitsThreshold = totalHitsThreshold;
         }
 
@@ -95,6 +91,10 @@ class PartialHitCountCollector extends TotalHitCountCollector {
             this.hitsThresholdChecker = new HitsThresholdChecker(totalHitsThreshold);
         }
 
+        CollectorManager(HitsThresholdChecker hitsThresholdChecker) {
+            this.hitsThresholdChecker = hitsThresholdChecker;
+        }
+
         @Override
         public PartialHitCountCollector newCollector() {
             return new PartialHitCountCollector(hitsThresholdChecker);

server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorManagerFactory.java

@@ -38,8 +38,6 @@ import org.apache.lucene.search.TopDocsCollector;
 import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.search.TotalHitCountCollectorManager;
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.action.search.MaxScoreCollector;
 import org.elasticsearch.common.lucene.Lucene;
@@ -59,7 +57,6 @@ import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -116,33 +113,12 @@ abstract class TopDocsCollectorManagerFactory {
                 // for bwc hit count is set to 0, it will be converted to -1 by the coordinating node
                 this.hitCountSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);
             } else {
-                if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE) {
-                    final int[] totalHits = new int[1];
-                    TotalHitCountCollectorManager totalHitCountCollectorManager = new TotalHitCountCollectorManager();
-                    // wrapping total hit count collector manager is needed to bridge the type mismatch between <Void> and <Integer>.
-                    this.collectorManager = new CollectorManager<>() {
-                        @Override
-                        public Collector newCollector() throws IOException {
-                            return totalHitCountCollectorManager.newCollector();
-                        }
-
-                        @Override
-                        public Void reduce(Collection<Collector> collectors) throws IOException {
-                            totalHits[0] = totalHitCountCollectorManager.reduce(
-                                collectors.stream().map(c -> (TotalHitCountCollector) c).toList()
-                            );
-                            return null;
-                        }
-                    };
-                    this.hitCountSupplier = () -> new TotalHits(totalHits[0], TotalHits.Relation.EQUAL_TO);
-                } else {
-                    PartialHitCountCollector.CollectorManager cm = new PartialHitCountCollector.CollectorManager(trackTotalHitsUpTo);
-                    this.collectorManager = cm;
-                    this.hitCountSupplier = () -> new TotalHits(
-                        cm.getTotalHits(),
-                        cm.hasEarlyTerminated() ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO
-                    );
-                }
+                PartialHitCountCollector.CollectorManager cm = new PartialHitCountCollector.CollectorManager(trackTotalHitsUpTo);
+                this.collectorManager = cm;
+                this.hitCountSupplier = () -> new TotalHits(
+                    cm.getTotalHits(),
+                    cm.hasEarlyTerminated() ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO
+                );
             }
         }
 

server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java

@@ -334,7 +334,7 @@ public class SynonymsManagementAPIService {
                         deleteListener.delegateFailure(
                             (checkListener, obj) -> checkListener.onFailure(
                                 new ResourceNotFoundException(
-                                    "synonym rule [" + synonymRuleId + "] not found on synonym set [" + synonymSetId + "]"
+                                    "synonym rule [" + synonymRuleId + "] not found on synonyms set [" + synonymSetId + "]"
                                 )
                             )
                         )
@@ -381,7 +381,7 @@ public class SynonymsManagementAPIService {
         client.prepareGet(SYNONYMS_ALIAS_NAME, synonymsSetId)
             .execute(new DelegatingIndexNotFoundActionListener<>(synonymsSetId, listener, (l, getResponse) -> {
                 if (getResponse.isExists() == false) {
-                    l.onFailure(new ResourceNotFoundException("Synonym set [" + synonymsSetId + "] not found"));
+                    l.onFailure(new ResourceNotFoundException("Synonyms set [" + synonymsSetId + "] not found"));
                     return;
                 }
                 l.onResponse(null);
@@ -409,7 +409,7 @@ public class SynonymsManagementAPIService {
                     .collect(Collectors.toSet());
                 reloadListener.onFailure(
                     new IllegalArgumentException(
-                        "Synonym set ["
+                        "Synonyms set ["
                             + synonymSetId
                             + "] cannot be deleted as it is used in the following indices: "
                             + String.join(", ", indices)
@@ -421,14 +421,14 @@ public class SynonymsManagementAPIService {
             deleteSynonymsSetObjects(synonymSetId, listener.delegateFailure((deleteObjectsListener, bulkByScrollResponse) -> {
                 if (bulkByScrollResponse.getDeleted() == 0) {
                     // If nothing was deleted, synonym set did not exist
-                    deleteObjectsListener.onFailure(new ResourceNotFoundException("Synonym set [" + synonymSetId + "] not found"));
+                    deleteObjectsListener.onFailure(new ResourceNotFoundException("Synonyms set [" + synonymSetId + "] not found"));
                     return;
                 }
                 final List<BulkItemResponse.Failure> bulkFailures = bulkByScrollResponse.getBulkFailures();
                 if (bulkFailures.isEmpty() == false) {
                     deleteObjectsListener.onFailure(
                         new InvalidParameterException(
-                            "Error deleting synonym set: "
+                            "Error deleting synonyms set: "
                                 + bulkFailures.stream().map(BulkItemResponse.Failure::getMessage).collect(Collectors.joining("\n"))
                         )
                     );
@@ -499,7 +499,7 @@ public class SynonymsManagementAPIService {
         public void onFailure(Exception e) {
             Throwable cause = ExceptionsHelper.unwrapCause(e);
             if (cause instanceof IndexNotFoundException) {
-                delegate.onFailure(new ResourceNotFoundException("synonym set [" + synonymSetId + "] not found"));
+                delegate.onFailure(new ResourceNotFoundException("synonyms set [" + synonymSetId + "] not found"));
                 return;
             }
             delegate.onFailure(e);

server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java

@@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.Reconfigurator;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -46,6 +47,7 @@ import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 import static java.util.Collections.emptySet;
@@ -71,6 +73,7 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
     private ClusterStateObserver clusterStateObserver;
     private ClusterSettings clusterSettings;
     private int staticMaximum;
+    private FakeReconfigurator reconfigurator;
 
     @BeforeClass
     public static void createThreadPoolAndClusterService() {
@@ -116,6 +119,7 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
         }
         final Settings nodeSettings = nodeSettingsBuilder.build();
         clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        reconfigurator = new FakeReconfigurator();
 
         new TransportAddVotingConfigExclusionsAction(
             nodeSettings,
@@ -124,7 +128,8 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
             clusterService,
             threadPool,
             new ActionFilters(emptySet()),
-            TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext())
+            TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext()),
+            reconfigurator
         ); // registers action
 
         transportService.start();
@@ -476,6 +481,28 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
         assertThat(rootCause.getMessage(), startsWith("timed out waiting for voting config exclusions [{other1}"));
     }
 
+    public void testCannotAddVotingConfigExclusionsWhenItIsDisabled() {
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final SetOnce<TransportException> exceptionHolder = new SetOnce<>();
+
+        reconfigurator.disableUserVotingConfigModifications();
+
+        transportService.sendRequest(
+            localNode,
+            AddVotingConfigExclusionsAction.NAME,
+            new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[] { "other1" }, TimeValue.timeValueMillis(100)),
+            expectError(e -> {
+                exceptionHolder.set(e);
+                countDownLatch.countDown();
+            })
+        );
+
+        safeAwait(countDownLatch);
+        final Throwable rootCause = exceptionHolder.get().getRootCause();
+        assertThat(rootCause, instanceOf(IllegalStateException.class));
+        assertThat(rootCause.getMessage(), startsWith("Unable to modify the voting configuration"));
+    }
+
     private TransportResponseHandler<ActionResponse.Empty> expectSuccess(Consumer<ActionResponse.Empty> onResponse) {
         return responseHandler(onResponse, e -> { throw new AssertionError("unexpected", e); });
     }
@@ -558,4 +585,23 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
         }
     }
 
+    static class FakeReconfigurator extends Reconfigurator {
+        private final AtomicBoolean canModifyVotingConfiguration = new AtomicBoolean(true);
+
+        FakeReconfigurator() {
+            super(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        }
+
+        @Override
+        public void ensureVotingConfigCanBeModified() {
+            if (canModifyVotingConfiguration.get() == false) {
+                throw new IllegalStateException("Unable to modify the voting configuration");
+            }
+        }
+
+        void disableUserVotingConfigModifications() {
+            canModifyVotingConfiguration.set(false);
+        }
+    }
+
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java

@@ -57,6 +57,7 @@ public class TransportClearVotingConfigExclusionsActionTests extends ESTestCase
     private static VotingConfigExclusion otherNode1Exclusion, otherNode2Exclusion;
 
     private TransportService transportService;
+    private TransportAddVotingConfigExclusionsActionTests.FakeReconfigurator reconfigurator;
 
     @BeforeClass
     public static void createThreadPoolAndClusterService() {
@@ -86,13 +87,15 @@ public class TransportClearVotingConfigExclusionsActionTests extends ESTestCase
             null,
             emptySet()
         );
+        reconfigurator = new TransportAddVotingConfigExclusionsActionTests.FakeReconfigurator();
 
         new TransportClearVotingConfigExclusionsAction(
             transportService,
             clusterService,
             threadPool,
             new ActionFilters(emptySet()),
-            TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext())
+            TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext()),
+            reconfigurator
         ); // registers action
 
         transportService.start();
@@ -185,6 +188,28 @@ public class TransportClearVotingConfigExclusionsActionTests extends ESTestCase
         assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), empty());
     }
 
+    public void testCannotClearVotingConfigurationWhenItIsDisabled() {
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final SetOnce<TransportException> exceptionHolder = new SetOnce<>();
+
+        reconfigurator.disableUserVotingConfigModifications();
+
+        transportService.sendRequest(
+            localNode,
+            ClearVotingConfigExclusionsAction.NAME,
+            new ClearVotingConfigExclusionsRequest(),
+            expectError(e -> {
+                exceptionHolder.set(e);
+                countDownLatch.countDown();
+            })
+        );
+
+        safeAwait(countDownLatch);
+        final Throwable rootCause = exceptionHolder.get().getRootCause();
+        assertThat(rootCause, instanceOf(IllegalStateException.class));
+        assertThat(rootCause.getMessage(), startsWith("Unable to modify the voting configuration"));
+    }
+
     private TransportResponseHandler<ActionResponse.Empty> expectSuccess(Consumer<ActionResponse.Empty> onResponse) {
         return responseHandler(onResponse, e -> { throw new AssertionError("unexpected", e); });
     }

server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.ProcessorInfo;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.reservedstate.service.ReservedClusterStateService;
@@ -88,7 +89,8 @@ public class ReservedPipelineActionTests extends ESTestCase {
             null,
             Collections.singletonList(DUMMY_PLUGIN),
             client,
-            null
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         Map<String, Processor.Factory> factories = ingestService.getProcessorFactories();
         assertTrue(factories.containsKey("set"));

server/src/test/java/org/elasticsearch/index/IndexModuleTests.java

@@ -86,6 +86,7 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedInd
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.test.ClusterServiceUtils;
@@ -233,7 +234,8 @@ public class IndexModuleTests extends ESTestCase {
             Collections.emptyMap(),
             () -> true,
             indexNameExpressionResolver,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         module.setReaderWrapper(s -> new Wrapper());
 
@@ -258,7 +260,8 @@ public class IndexModuleTests extends ESTestCase {
             indexStoreFactories,
             () -> true,
             indexNameExpressionResolver,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         final IndexService indexService = newIndexService(module);
@@ -281,7 +284,8 @@ public class IndexModuleTests extends ESTestCase {
             Map.of(),
             () -> true,
             indexNameExpressionResolver,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         module.setDirectoryWrapper(new TestDirectoryWrapper());
@@ -632,7 +636,8 @@ public class IndexModuleTests extends ESTestCase {
             Collections.emptyMap(),
             () -> true,
             indexNameExpressionResolver,
-            recoveryStateFactories
+            recoveryStateFactories,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         final IndexService indexService = newIndexService(module);
@@ -652,7 +657,8 @@ public class IndexModuleTests extends ESTestCase {
             Collections.emptyMap(),
             () -> true,
             indexNameExpressionResolver,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
 
         final AtomicLong lastAcquiredPrimaryTerm = new AtomicLong();
@@ -752,7 +758,8 @@ public class IndexModuleTests extends ESTestCase {
             Collections.emptyMap(),
             () -> true,
             indexNameExpressionResolver,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
     }
 

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -94,7 +95,8 @@ public class CodecTests extends ESTestCase {
             mapperRegistry,
             () -> null,
             settings.getMode().idFieldMapperWithoutFieldData(),
-            ScriptCompiler.NONE
+            ScriptCompiler.NONE,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE);
     }

server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 
@@ -67,7 +68,7 @@ public class DocumentMapperTests extends MapperServiceTestCase {
         assertThat(stage1.mappers().getMapper("age"), nullValue());
         assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue());
         // but merged should
-        DocumentParser documentParser = new DocumentParser(null, null);
+        DocumentParser documentParser = new DocumentParser(null, null, () -> DocumentParsingObserver.EMPTY_INSTANCE);
         DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent());
         assertThat(mergedMapper.mappers().getMapper("age"), notNullValue());
         assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue());

server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java

@@ -702,7 +702,7 @@ public class DynamicTemplatesTests extends MapperServiceTestCase {
             {"foo": "41.12,-71.34", "bar": "41.12,-71.34"}
             """;
         ParsedDocument doc = mapperService.documentMapper()
-            .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point")));
+            .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"), false));
         assertThat(doc.rootDoc().getFields("foo"), hasSize(2));
         assertThat(doc.rootDoc().getFields("bar"), hasSize(1));
     }

server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java

@@ -53,7 +53,8 @@ public class RoutingFieldMapperTests extends MetadataMapperTestCase {
                 BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()),
                 XContentType.JSON,
                 "routing_value",
-                Map.of()
+                Map.of(),
+                false
             )
         );
 

server/src/test/java/org/elasticsearch/index/store/ByteSizeCachingDirectoryTests.java

@@ -46,6 +46,7 @@ public class ByteSizeCachingDirectoryTests extends ESTestCase {
             ByteSizeCachingDirectory cachingDir = new ByteSizeCachingDirectory(countingDir, new TimeValue(0));
             assertEquals(11, cachingDir.estimateSizeInBytes());
             assertEquals(11, cachingDir.estimateSizeInBytes());
+            assertEquals(11, cachingDir.estimateDataSetSizeInBytes());
             assertEquals(1, countingDir.numFileLengthCalls);
 
             try (IndexOutput out = cachingDir.createOutput("foo", IOContext.DEFAULT)) {
@@ -63,6 +64,7 @@ public class ByteSizeCachingDirectoryTests extends ESTestCase {
             assertEquals(7, countingDir.numFileLengthCalls);
             assertEquals(16, cachingDir.estimateSizeInBytes());
             assertEquals(7, countingDir.numFileLengthCalls);
+            assertEquals(16, cachingDir.estimateDataSetSizeInBytes());
 
             try (IndexOutput out = cachingDir.createTempOutput("bar", "baz", IOContext.DEFAULT)) {
                 out.writeBytes(new byte[4], 4);
@@ -79,6 +81,7 @@ public class ByteSizeCachingDirectoryTests extends ESTestCase {
             assertEquals(16, countingDir.numFileLengthCalls);
             assertEquals(20, cachingDir.estimateSizeInBytes());
             assertEquals(16, countingDir.numFileLengthCalls);
+            assertEquals(20, cachingDir.estimateDataSetSizeInBytes());
 
             cachingDir.deleteFile("foo");
 
@@ -87,6 +90,7 @@ public class ByteSizeCachingDirectoryTests extends ESTestCase {
             assertEquals(18, countingDir.numFileLengthCalls);
             assertEquals(15, cachingDir.estimateSizeInBytes());
             assertEquals(18, countingDir.numFileLengthCalls);
+            assertEquals(15, cachingDir.estimateDataSetSizeInBytes());
 
             // Close more than once
             IndexOutput out = cachingDir.createOutput("foo", IOContext.DEFAULT);
@@ -106,6 +110,7 @@ public class ByteSizeCachingDirectoryTests extends ESTestCase {
             }
             out.close();
             assertEquals(20, cachingDir.estimateSizeInBytes());
+            assertEquals(20, cachingDir.estimateDataSetSizeInBytes());
             assertEquals(27, countingDir.numFileLengthCalls);
         }
     }

server/src/test/java/org/elasticsearch/index/store/StoreTests.java

@@ -49,6 +49,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.store.FilterIndexOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.IOUtils;
@@ -82,6 +83,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongUnaryOperator;
 
 import static java.util.Collections.emptyList;
@@ -809,6 +811,108 @@ public class StoreTests extends ESTestCase {
         IOUtils.close(store);
     }
 
+    public void testStoreSizes() throws IOException {
+        // directory that returns total written bytes as the data set size
+        final var directory = new ByteSizeDirectory(StoreTests.newDirectory(random())) {
+
+            final AtomicLong dataSetBytes = new AtomicLong(0L);
+
+            @Override
+            public long estimateSizeInBytes() throws IOException {
+                return estimateSizeInBytes(getDelegate());
+            }
+
+            @Override
+            public long estimateDataSetSizeInBytes() {
+                return dataSetBytes.get();
+            }
+
+            @Override
+            public IndexOutput createOutput(String name, IOContext context) throws IOException {
+                return wrap(super.createOutput(name, context));
+            }
+
+            @Override
+            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+                return wrap(super.createTempOutput(prefix, suffix, context));
+            }
+
+            private IndexOutput wrap(IndexOutput output) {
+                return new FilterIndexOutput("wrapper", output) {
+                    @Override
+                    public void writeByte(byte b) throws IOException {
+                        super.writeByte(b);
+                        dataSetBytes.incrementAndGet();
+                    }
+
+                    @Override
+                    public void writeBytes(byte[] b, int offset, int length) throws IOException {
+                        super.writeBytes(b, offset, length);
+                        dataSetBytes.addAndGet(length);
+                    }
+                };
+            }
+        };
+
+        final ShardId shardId = new ShardId("index", "_na_", 1);
+        final Store store = new Store(
+            shardId,
+            IndexSettingsModule.newIndexSettings(
+                "index",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT)
+                    .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMinutes(0))
+                    .build()
+            ),
+            directory,
+            new DummyShardLock(shardId)
+        );
+        long initialStoreSize = 0L;
+        for (String extraFiles : store.directory().listAll()) {
+            assertTrue("expected extraFS file but got: " + extraFiles, extraFiles.startsWith("extra"));
+            initialStoreSize += store.directory().fileLength(extraFiles);
+        }
+
+        StoreStats stats = store.stats(0L, LongUnaryOperator.identity());
+        assertThat(stats.sizeInBytes(), equalTo(initialStoreSize));
+        assertThat(stats.totalDataSetSizeInBytes(), equalTo(initialStoreSize));
+
+        long additionalStoreSize = 0L;
+
+        int iters = randomIntBetween(1, 10);
+        for (int i = 0; i < iters; i++) {
+            try (IndexOutput output = directory.createOutput(i + ".bar", IOContext.DEFAULT)) {
+                BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
+                output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+                additionalStoreSize += output.getFilePointer();
+            }
+        }
+
+        stats = store.stats(0L, LongUnaryOperator.identity());
+        assertThat(stats.sizeInBytes(), equalTo(initialStoreSize + additionalStoreSize));
+        assertThat(stats.totalDataSetSizeInBytes(), equalTo(initialStoreSize + additionalStoreSize));
+
+        long deletionsStoreSize = 0L;
+
+        var randomFiles = randomSubsetOf(Arrays.asList(directory.listAll()));
+        for (String randomFile : randomFiles) {
+            try {
+                long length = directory.fileLength(randomFile);
+                directory.deleteFile(randomFile);
+                deletionsStoreSize += length;
+            } catch (NoSuchFileException | FileNotFoundException e) {
+                // ignore
+            }
+        }
+
+        stats = store.stats(0L, LongUnaryOperator.identity());
+        assertThat(stats.sizeInBytes(), equalTo(initialStoreSize + additionalStoreSize - deletionsStoreSize));
+        assertThat(stats.totalDataSetSizeInBytes(), equalTo(initialStoreSize + additionalStoreSize));
+
+        deleteContent(store.directory());
+        IOUtils.close(store);
+    }
+
     public static void deleteContent(Directory directory) throws IOException {
         final String[] files = directory.listAll();
         final List<IOException> exceptions = new ArrayList<>();

server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java

@@ -52,6 +52,7 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptModule;
@@ -148,7 +149,8 @@ public class IngestServiceTests extends ESTestCase {
             null,
             List.of(DUMMY_PLUGIN),
             client,
-            null
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         Map<String, Processor.Factory> factories = ingestService.getProcessorFactories();
         assertTrue(factories.containsKey("foo"));
@@ -167,7 +169,8 @@ public class IngestServiceTests extends ESTestCase {
                 null,
                 List.of(DUMMY_PLUGIN, DUMMY_PLUGIN),
                 client,
-                null
+                null,
+                () -> DocumentParsingObserver.EMPTY_INSTANCE
             )
         );
         assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
@@ -183,7 +186,8 @@ public class IngestServiceTests extends ESTestCase {
             null,
             List.of(DUMMY_PLUGIN),
             client,
-            null
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
             .source(Map.of())
@@ -1902,7 +1906,8 @@ public class IngestServiceTests extends ESTestCase {
             null,
             List.of(testPlugin),
             client,
-            null
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
         ingestService.addIngestClusterStateListener(ingestClusterStateListener);
 
@@ -2230,7 +2235,17 @@ public class IngestServiceTests extends ESTestCase {
         Client client = mock(Client.class);
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(clusterState);
-        IngestService ingestService = new IngestService(clusterService, threadPool, null, null, null, List.of(DUMMY_PLUGIN), client, null);
+        IngestService ingestService = new IngestService(
+            clusterService,
+            threadPool,
+            null,
+            null,
+            null,
+            List.of(DUMMY_PLUGIN),
+            client,
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
+        );
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, clusterState));
 
         CountDownLatch latch = new CountDownLatch(1);
@@ -2512,7 +2527,7 @@ public class IngestServiceTests extends ESTestCase {
             public Map<String, Processor.Factory> getProcessors(final Processor.Parameters parameters) {
                 return processors;
             }
-        }), client, null);
+        }), client, null, () -> DocumentParsingObserver.EMPTY_INSTANCE);
     }
 
     private CompoundProcessor mockCompoundProcessor() {

server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java

@@ -144,6 +144,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
             contextWithoutScroll.from(300);
@@ -182,6 +183,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
             context1.from(300);
@@ -255,6 +257,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
 
@@ -290,6 +293,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
@@ -315,6 +319,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
             context4.sliceBuilder(new SliceBuilder(1, 2)).parsedQuery(parsedQuery).preProcess();
@@ -381,6 +386,7 @@ public class DefaultSearchContextTests extends ESTestCase {
                 timeout,
                 randomIntBetween(1, Integer.MAX_VALUE),
                 null,
+                false,
                 false
             );
 

server/src/test/java/org/elasticsearch/search/SearchServiceTests.java

@@ -19,6 +19,7 @@ import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClosePointInTimeAction;
@@ -72,6 +73,7 @@ import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.SearchService.ResultsType;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
@@ -126,6 +128,7 @@ import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
+import static org.elasticsearch.search.SearchModule.SEARCH_CONCURRENCY_ENABLED;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
@@ -515,7 +518,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
                 reader,
                 requestWithDefaultTimeout,
                 mock(SearchShardTask.class),
-                SearchService.ResultsType.NONE,
+                ResultsType.NONE,
                 randomBoolean()
             )
         ) {
@@ -542,7 +545,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
                 reader,
                 requestWithCustomTimeout,
                 mock(SearchShardTask.class),
-                SearchService.ResultsType.NONE,
+                ResultsType.NONE,
                 randomBoolean()
             )
         ) {
@@ -582,13 +585,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         );
         try (
             ReaderContext reader = createReaderContext(indexService, indexShard);
-            SearchContext context = service.createContext(
-                reader,
-                request,
-                mock(SearchShardTask.class),
-                SearchService.ResultsType.NONE,
-                randomBoolean()
-            )
+            SearchContext context = service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
         ) {
             assertNotNull(context);
         }
@@ -596,13 +593,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         searchSourceBuilder.docValueField("unmapped_field");
         try (
             ReaderContext reader = createReaderContext(indexService, indexShard);
-            SearchContext context = service.createContext(
-                reader,
-                request,
-                mock(SearchShardTask.class),
-                SearchService.ResultsType.NONE,
-                randomBoolean()
-            )
+            SearchContext context = service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
         ) {
             assertNotNull(context);
         }
@@ -611,7 +602,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         try (ReaderContext reader = createReaderContext(indexService, indexShard)) {
             IllegalArgumentException ex = expectThrows(
                 IllegalArgumentException.class,
-                () -> service.createContext(reader, request, mock(SearchShardTask.class), SearchService.ResultsType.NONE, randomBoolean())
+                () -> service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
             );
             assertEquals(
                 "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [1] but was [2]. "
@@ -660,7 +651,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
                     reader,
                     request,
                     mock(SearchShardTask.class),
-                    SearchService.ResultsType.NONE,
+                    ResultsType.NONE,
                     randomBoolean()
                 )
             ) {
@@ -709,7 +700,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
                     reader,
                     request,
                     mock(SearchShardTask.class),
-                    SearchService.ResultsType.NONE,
+                    ResultsType.NONE,
                     randomBoolean()
                 )
             ) {
@@ -721,7 +712,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
             );
             IllegalArgumentException ex = expectThrows(
                 IllegalArgumentException.class,
-                () -> service.createContext(reader, request, mock(SearchShardTask.class), SearchService.ResultsType.NONE, randomBoolean())
+                () -> service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
             );
             assertEquals(
                 "Trying to retrieve too many script_fields. Must be less than or equal to: ["
@@ -762,13 +753,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         );
         try (
             ReaderContext reader = createReaderContext(indexService, indexShard);
-            SearchContext context = service.createContext(
-                reader,
-                request,
-                mock(SearchShardTask.class),
-                SearchService.ResultsType.NONE,
-                randomBoolean()
-            )
+            SearchContext context = service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
         ) {
             assertEquals(0, context.scriptFields().fields().size());
         }
@@ -1193,7 +1178,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
                     readerContext,
                     shardRequest,
                     mock(SearchShardTask.class),
-                    SearchService.ResultsType.QUERY,
+                    ResultsType.QUERY,
                     true
                 )
             ) {
@@ -1313,7 +1298,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         try (ReaderContext reader = createReaderContext(indexService, indexService.getShard(shardId.id()))) {
             NullPointerException e = expectThrows(
                 NullPointerException.class,
-                () -> service.createContext(reader, request, mock(SearchShardTask.class), SearchService.ResultsType.NONE, randomBoolean())
+                () -> service.createContext(reader, request, mock(SearchShardTask.class), ResultsType.NONE, randomBoolean())
             );
             assertEquals("expected", e.getMessage());
         }
@@ -1910,6 +1895,44 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         assertTrue(service.freeReaderContext(contextId));
     }
 
+    public void testEnableConcurrentCollection() {
+        createIndex("index", Settings.EMPTY);
+        SearchService service = getInstanceFromNode(SearchService.class);
+        assertTrue(service.isConcurrentCollectionEnabled());
+
+        try {
+            ClusterUpdateSettingsResponse response = client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().put(SEARCH_CONCURRENCY_ENABLED.getKey(), false).build())
+                .get();
+            assertTrue(response.isAcknowledged());
+            assertFalse(service.isConcurrentCollectionEnabled());
+
+        } finally {
+            // reset original default setting
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().putNull(SEARCH_CONCURRENCY_ENABLED.getKey()).build())
+                .get();
+        }
+    }
+
+    public void testConcurrencyConditions() {
+        SearchSourceBuilder searchSourceBuilder = randomBoolean() ? null : new SearchSourceBuilder();
+        if (searchSourceBuilder != null && randomBoolean()) {
+            searchSourceBuilder.aggregation(new TermsAggregationBuilder("terms"));
+        }
+        assertTrue(SearchService.concurrentSearchEnabled(ResultsType.DFS, searchSourceBuilder));
+        assertFalse(
+            SearchService.concurrentSearchEnabled(
+                randomFrom(randomFrom(ResultsType.QUERY, ResultsType.NONE, ResultsType.FETCH)),
+                searchSourceBuilder
+            )
+        );
+    }
+
     private ReaderContext createReaderContext(IndexService indexService, IndexShard indexShard) {
         return new ReaderContext(
             new ShardSearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong()),

server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java

@@ -36,6 +36,8 @@ import org.apache.lucene.search.ConstantScoreWeight;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSearcher.LeafSlice;
+import org.apache.lucene.search.KnnFloatVectorQuery;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -58,6 +60,9 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -78,6 +83,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet;
@@ -175,6 +181,116 @@ public class ContextIndexSearcherTests extends ESTestCase {
         directory.close();
     }
 
+    /**
+     * Check that knn queries rewrite parallelizes on the number of segments if there are enough
+     * threads available.
+     */
+    public void testConcurrentKnnRewrite() throws Exception {
+        final Directory directory = newDirectory();
+        try (
+            IndexWriter iw = new IndexWriter(
+                directory,
+                new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)
+            )
+        ) {
+            final int numDocs = randomIntBetween(100, 200);
+            for (int i = 0; i < numDocs; i++) {
+                Document document = new Document();
+                document.add(new StringField("field", "value", Field.Store.NO));
+                iw.addDocument(document);
+                if (rarely()) {
+                    iw.commit();
+                }
+            }
+        }
+
+        // make sure we have more threads than segments available to check later call to execute method
+        int nThreads = randomIntBetween(2, 5);
+
+        // use an executor that counts calls to its "execute" method
+        AtomicInteger executeCalls = new AtomicInteger(0);
+        ThreadPoolExecutor executor = null;
+        DirectoryReader directoryReader = null;
+        try {
+            executor = new PrioritizedEsThreadPoolExecutor(
+                "test",
+                nThreads,
+                Integer.MAX_VALUE,
+                0L,
+                TimeUnit.MILLISECONDS,
+                EsExecutors.daemonThreadFactory("queuetest"),
+                new ThreadContext(Settings.EMPTY),
+                null
+            ) {
+                @Override
+                public void execute(Runnable command) {
+                    executeCalls.incrementAndGet();
+                    super.execute(command);
+                }
+            };
+
+            directoryReader = DirectoryReader.open(directory);
+            ContextIndexSearcher searcher = new ContextIndexSearcher(
+                directoryReader,
+                IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(),
+                1,
+                randomBoolean(),
+                executor
+            );
+            // check that we calculate one slice per segment
+            int numSegments = directoryReader.getContext().leaves().size();
+            assertEquals(numSegments, searcher.slices(directoryReader.getContext().leaves()).length);
+
+            KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("float_vector", new float[] { 0, 0, 0 }, 10, null);
+            Query rewritenQuery = vectorQuery.rewrite(searcher);
+            // Note: we expect one execute calls less than segments since the last is executed on the caller thread.
+            // For details see QueueSizeBasedExecutor#processTask
+            assertEquals(numSegments - 1, executeCalls.get());
+
+            AtomicInteger collectorCalls = new AtomicInteger(0);
+            searcher.search(rewritenQuery, new CollectorManager<Collector, Object>() {
+
+                @Override
+                public Collector newCollector() {
+                    collectorCalls.incrementAndGet();
+                    return new Collector() {
+                        @Override
+                        public LeafCollector getLeafCollector(LeafReaderContext context) {
+                            return new LeafBucketCollector() {
+                                @Override
+                                public void collect(int doc, long owningBucketOrd) throws IOException {
+                                    // noop
+                                }
+                            };
+                        }
+
+                        @Override
+                        public ScoreMode scoreMode() {
+                            return ScoreMode.COMPLETE;
+                        }
+                    };
+                }
+
+                @Override
+                public Object reduce(Collection<Collector> collectors) throws IOException {
+                    return null;
+                }
+            });
+            LeafSlice[] leafSlices = ContextIndexSearcher.computeSlices(
+                directoryReader.getContext().leaves(),
+                executor.getMaximumPoolSize(),
+                1
+            );
+            assertEquals(leafSlices.length, collectorCalls.get());
+        } finally {
+            directoryReader.close();
+            directory.close();
+            executor.shutdown();
+        }
+    }
+
     public void testConcurrentSearchAllThreadsFinish() throws Exception {
         final Directory directory = newDirectory();
         IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE));
@@ -195,17 +311,41 @@ public class ContextIndexSearcherTests extends ESTestCase {
         AtomicInteger missingDocs = new AtomicInteger();
         AtomicInteger visitDocs = new AtomicInteger(0);
 
+        // determine how many docs are in the first slice for correct, this is how much we are missing by
+        // throwing the exception in the first collector
+        int minDocsPerSlice = 1;
+
+        ContextIndexSearcher searcher = new ContextIndexSearcher(
+            directoryReader,
+            IndexSearcher.getDefaultSimilarity(),
+            IndexSearcher.getDefaultQueryCache(),
+            IndexSearcher.getDefaultQueryCachingPolicy(),
+            minDocsPerSlice,
+            randomBoolean(),
+            executor
+        );
+
+        LeafSlice[] leafSlices = ContextIndexSearcher.computeSlices(
+            directoryReader.getContext().leaves(),
+            executor.getMaximumPoolSize(),
+            minDocsPerSlice
+        );
+        // The test collector manager throws an exception when the first segment gets collected.
+        // All documents in that slice count towards the "missing" docs in the later assertion.
+        int docsFirstSlice = Arrays.stream(leafSlices[0].leaves).map(LeafReaderContext::reader).mapToInt(LeafReader::maxDoc).sum();
+        AtomicInteger collectorCalls = new AtomicInteger(0);
         CollectorManager<Collector, Void> collectorManager = new CollectorManager<>() {
             boolean first = true;
 
             @Override
             public Collector newCollector() {
+                collectorCalls.incrementAndGet();
                 if (first) {
                     first = false;
                     return new Collector() {
                         @Override
                         public LeafCollector getLeafCollector(LeafReaderContext context) {
-                            missingDocs.set(context.reader().numDocs());
+                            missingDocs.set(docsFirstSlice);
                             throw new IllegalArgumentException("fake exception");
                         }
 
@@ -245,26 +385,12 @@ public class ContextIndexSearcherTests extends ESTestCase {
             }
         };
 
-        ContextIndexSearcher searcher = new ContextIndexSearcher(
-            directoryReader,
-            IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(),
-            IndexSearcher.getDefaultQueryCachingPolicy(),
-            1,
-            randomBoolean(),
-            executor
-        ) {
-            @Override
-            protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
-                return slices(leaves, 1, 1);
-            }
-        };
-
         IllegalArgumentException exception = expectThrows(
             IllegalArgumentException.class,
             () -> searcher.search(new MatchAllDocsQuery(), collectorManager)
         );
         assertThat(exception.getMessage(), equalTo("fake exception"));
+        assertEquals(leafSlices.length, collectorCalls.get());
         assertThat(visitDocs.get() + missingDocs.get(), equalTo(numDocs));
         directoryReader.close();
         directory.close();
@@ -396,7 +522,7 @@ public class ContextIndexSearcherTests extends ESTestCase {
         int iter = randomIntBetween(16, 64);
         for (int i = 0; i < iter; i++) {
             int numThreads = randomIntBetween(1, 16);
-            IndexSearcher.LeafSlice[] slices = ContextIndexSearcher.computeSlices(contexts, numThreads, 1);
+            LeafSlice[] slices = ContextIndexSearcher.computeSlices(contexts, numThreads, 1);
             assertSlices(slices, numDocs, numThreads);
         }
         // expect exception for numThreads < 1
@@ -409,12 +535,12 @@ public class ContextIndexSearcherTests extends ESTestCase {
         IOUtils.close(reader, w, dir);
     }
 
-    private void assertSlices(IndexSearcher.LeafSlice[] slices, int numDocs, int numThreads) {
+    private void assertSlices(LeafSlice[] slices, int numDocs, int numThreads) {
         // checks that the number of slices is not bigger than the number of available threads
         // and each slice contains at least 10% of the data (which means the max number of slices is 10)
         int sumDocs = 0;
         assertThat(slices.length, lessThanOrEqualTo(numThreads));
-        for (IndexSearcher.LeafSlice slice : slices) {
+        for (LeafSlice slice : slices) {
             int sliceDocs = Arrays.stream(slice.leaves).mapToInt(l -> l.reader().maxDoc()).sum();
             assertThat(sliceDocs, greaterThanOrEqualTo((int) (0.1 * numDocs)));
             sumDocs += sliceDocs;

server/src/test/java/org/elasticsearch/search/query/PartialHitCountCollectorTests.java

@@ -19,6 +19,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.test.ESTestCase;
@@ -63,7 +64,7 @@ public class PartialHitCountCollectorTests extends ESTestCase {
 
     public void testEarlyTerminatesWithoutCollection() throws IOException {
         Query query = new NonCountingTermQuery(new Term("string", "a1"));
-        PartialHitCountCollector hitCountCollector = new PartialHitCountCollector(0) {
+        PartialHitCountCollector hitCountCollector = new PartialHitCountCollector(new PartialHitCountCollector.HitsThresholdChecker(0)) {
             @Override
             public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
                 return new FilterLeafCollector(super.getLeafCollector(context)) {
@@ -112,7 +113,7 @@ public class PartialHitCountCollectorTests extends ESTestCase {
     public void testCollectedHitCount() throws Exception {
         Query query = new NonCountingTermQuery(new Term("string", "a1"));
         int threshold = randomIntBetween(1, 10000);
-        // there's one doc matching the query: any totalHitsThreshold greater than or equal to 1 will non cause early termination
+        // there's one doc matching the query: any totalHitsThreshold greater than or equal to 1 will not cause early termination
         PartialHitCountCollector.CollectorManager collectorManager = new PartialHitCountCollector.CollectorManager(threshold);
         searcher.search(query, collectorManager);
         assertEquals(1, collectorManager.getTotalHits());
@@ -128,4 +129,42 @@ public class PartialHitCountCollectorTests extends ESTestCase {
         assertEquals(totalHitsThreshold, collectorManager.getTotalHits());
         assertTrue(collectorManager.hasEarlyTerminated());
     }
+
+    public void testCollectedAccurateHitCount() throws Exception {
+        Query query = new NonCountingTermQuery(new Term("string", "a1"));
+        // make sure there is no overhead caused by early termination functionality when performing accurate total hit counting
+        PartialHitCountCollector.CollectorManager collectorManager = new PartialHitCountCollector.CollectorManager(
+            NO_OVERHEAD_HITS_CHECKER
+        );
+        searcher.search(query, collectorManager);
+        assertEquals(1, collectorManager.getTotalHits());
+        assertFalse(collectorManager.hasEarlyTerminated());
+    }
+
+    public void testScoreModeEarlyTermination() {
+        PartialHitCountCollector hitCountCollector = new PartialHitCountCollector(
+            new PartialHitCountCollector.HitsThresholdChecker(randomIntBetween(0, Integer.MAX_VALUE - 1))
+        );
+        assertEquals(ScoreMode.TOP_DOCS, hitCountCollector.scoreMode());
+    }
+
+    public void testScoreModeAccurateHitCount() {
+        PartialHitCountCollector hitCountCollector = new PartialHitCountCollector(
+            new PartialHitCountCollector.HitsThresholdChecker(Integer.MAX_VALUE)
+        );
+        assertEquals(ScoreMode.COMPLETE_NO_SCORES, hitCountCollector.scoreMode());
+    }
+
+    private static final PartialHitCountCollector.HitsThresholdChecker NO_OVERHEAD_HITS_CHECKER =
+        new PartialHitCountCollector.HitsThresholdChecker(Integer.MAX_VALUE) {
+            @Override
+            void incrementHitCount() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            boolean isThresholdReached() {
+                throw new UnsupportedOperationException();
+            }
+        };
 }

server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java

@@ -163,6 +163,7 @@ import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.monitor.StatusInfo;
 import org.elasticsearch.node.ResponseCollectorService;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
@@ -1797,7 +1798,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
                     emptyMap(),
                     List.of(),
                     emptyMap(),
-                    null
+                    null,
+                    () -> DocumentParsingObserver.EMPTY_INSTANCE
                 );
                 final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings);
                 snapshotShardsService = new SnapshotShardsService(
@@ -1937,7 +1939,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
                             new AnalysisModule(environment, Collections.emptyList(), new StablePluginsRegistry()).getAnalysisRegistry(),
                             Collections.emptyList(),
                             client,
-                            null
+                            null,
+                            () -> DocumentParsingObserver.EMPTY_INSTANCE
                         ),
                         client,
                         actionFilters,

test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java

@@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -66,7 +67,8 @@ public class MapperTestUtils {
             mapperRegistry,
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
-            ScriptCompiler.NONE
+            ScriptCompiler.NONE,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
     }
 }

test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java

@@ -22,6 +22,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
@@ -54,7 +55,8 @@ public class TranslogHandler implements Engine.TranslogRecoveryRunner {
             mapperRegistry,
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
-            null
+            null,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
     }
 
@@ -88,7 +90,14 @@ public class TranslogHandler implements Engine.TranslogRecoveryRunner {
                 final Translog.Index index = (Translog.Index) operation;
                 final Engine.Index engineIndex = IndexShard.prepareIndex(
                     mapperService,
-                    new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing(), Map.of()),
+                    new SourceToParse(
+                        index.id(),
+                        index.source(),
+                        XContentHelper.xContentType(index.source()),
+                        index.routing(),
+                        Map.of(),
+                        false
+                    ),
                     index.seqNo(),
                     index.primaryTerm(),
                     index.version(),

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -57,6 +57,7 @@ import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -215,7 +216,8 @@ public abstract class MapperServiceTestCase extends ESTestCase {
                 throw new UnsupportedOperationException();
             },
             indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
-            this::compileScript
+            this::compileScript,
+            () -> DocumentParsingObserver.EMPTY_INSTANCE
         );
     }
 
@@ -274,7 +276,7 @@ public abstract class MapperServiceTestCase extends ESTestCase {
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
         build.accept(builder);
         builder.endObject();
-        return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates);
+        return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates, false);
     }
 
     /**
@@ -711,7 +713,7 @@ public abstract class MapperServiceTestCase extends ESTestCase {
         try (Directory roundTripDirectory = newDirectory()) {
             RandomIndexWriter roundTripIw = new RandomIndexWriter(random(), roundTripDirectory);
             roundTripIw.addDocument(
-                mapper.parse(new SourceToParse("1", new BytesArray(syntheticSource), XContentType.JSON, null, Map.of())).rootDoc()
+                mapper.parse(new SourceToParse("1", new BytesArray(syntheticSource), XContentType.JSON, null, Map.of(), false)).rootDoc()
             );
             roundTripIw.close();
             try (DirectoryReader roundTripReader = DirectoryReader.open(roundTripDirectory)) {

test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java

@@ -947,7 +947,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
             id = UUIDs.base64UUID();
             autoGeneratedTimestamp = System.currentTimeMillis();
         }
-        SourceToParse sourceToParse = new SourceToParse(id, new BytesArray(source), xContentType, routing, Map.of());
+        SourceToParse sourceToParse = new SourceToParse(id, new BytesArray(source), xContentType, routing, Map.of(), false);
         Engine.IndexResult result;
         if (shard.routingEntry().primary()) {
             result = shard.applyIndexOperationOnPrimary(

test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java

@@ -663,6 +663,10 @@ public class MockRepository extends FsRepository {
                 if (failOnIndexLatest && BlobStoreRepository.INDEX_LATEST_BLOB.equals(blobName)) {
                     throw new IOException("Random IOException");
                 }
+                if (shouldFail(blobName, randomControlIOExceptionRate)) {
+                    logger.info("throwing random IOException for atomic write of file [{}] at path [{}]", blobName, path());
+                    throw new IOException("Random IOException");
+                }
                 if (blobName.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)) {
                     if (blockAndFailOnWriteIndexFile) {
                         blockExecutionAndFail(blobName);

test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java

@@ -65,6 +65,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.MockScriptService;
@@ -470,7 +471,8 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
                 mapperRegistry,
                 () -> createShardContext(null, clusterService.getClusterSettings()),
                 idxSettings.getMode().idFieldMapperWithoutFieldData(),
-                ScriptCompiler.NONE
+                ScriptCompiler.NONE,
+                () -> DocumentParsingObserver.EMPTY_INSTANCE
             );
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
             });

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -2037,7 +2037,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
             mockPlugins,
             getClientWrapper(),
             forbidPrivateIndexSettings(),
-            forceSingleDataPath()
+            forceSingleDataPath(),
+            autoManageVotingExclusions()
         );
     }
 
@@ -2441,6 +2442,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
      */
     protected void setupSuiteScopeCluster() throws Exception {}
 
+    protected boolean autoManageVotingExclusions() {
+        // Temporary workaround until #98055 is tackled
+        return true;
+    }
+
     private static boolean isSuiteScopedTest(Class<?> clazz) {
         return clazz.getAnnotation(SuiteScopeTestCase.class) != null;
     }

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -244,6 +244,7 @@ public final class InternalTestCluster extends TestCluster {
 
     private ServiceDisruptionScheme activeDisruptionScheme;
     private final Function<Client, Client> clientWrapper;
+    private final boolean autoManageVotingExclusions;
 
     /**
      * Default value of bootstrapMasterNodeIndex, indicating that bootstrapping should happen automatically.
@@ -286,7 +287,8 @@ public final class InternalTestCluster extends TestCluster {
             mockPlugins,
             clientWrapper,
             true,
-            false
+            false,
+            true
         );
     }
 
@@ -304,7 +306,8 @@ public final class InternalTestCluster extends TestCluster {
         final Collection<Class<? extends Plugin>> mockPlugins,
         final Function<Client, Client> clientWrapper,
         final boolean forbidPrivateIndexSettings,
-        final boolean forceSingleDataPath
+        final boolean forceSingleDataPath,
+        final boolean autoManageVotingExclusions
     ) {
         super(clusterSeed);
         this.autoManageMasterNodes = autoManageMasterNodes;
@@ -312,6 +315,7 @@ public final class InternalTestCluster extends TestCluster {
         this.forbidPrivateIndexSettings = forbidPrivateIndexSettings;
         this.baseDir = baseDir;
         this.clusterName = clusterName;
+        this.autoManageVotingExclusions = autoManageVotingExclusions;
         if (minNumDataNodes < 0 || maxNumDataNodes < 0) {
             throw new IllegalArgumentException("minimum and maximum number of data nodes must be >= 0");
         }
@@ -1887,7 +1891,7 @@ public final class InternalTestCluster extends TestCluster {
     private Set<String> excludeMasters(Collection<NodeAndClient> nodeAndClients) {
         assert Thread.holdsLock(this);
         final Set<String> excludedNodeNames = new HashSet<>();
-        if (autoManageMasterNodes && nodeAndClients.size() > 0) {
+        if (autoManageVotingExclusions && autoManageMasterNodes && nodeAndClients.size() > 0) {
 
             final long currentMasters = nodes.values().stream().filter(NodeAndClient::isMasterEligible).count();
             final long stoppingMasters = nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).count();
@@ -1917,7 +1921,7 @@ public final class InternalTestCluster extends TestCluster {
 
     private void removeExclusions(Set<String> excludedNodeIds) {
         assert Thread.holdsLock(this);
-        if (excludedNodeIds.isEmpty() == false) {
+        if (autoManageVotingExclusions && excludedNodeIds.isEmpty() == false) {
             logger.info("removing voting config exclusions for {} after restart/shutdown", excludedNodeIds);
             try {
                 Client client = getRandomNodeAndClient(node -> excludedNodeIds.contains(node.name) == false).client();