Add Watcher APIs for updating/retrieving settings (#95342)

The `.watches` index is a system index, which means that its settings
cannot be modified by the user. This commit adds APIs (`PUT
/_watcher/settings` and `GET /_watcher/settings`) that allow modifying
and retrieving a subset of index settings for the `.watches` index.

The settings that are currently allowed are `index.number_of_replicas`
and `index.auto_expand_replicas`, though more may be added in the
future.

Resolves https://github.com/elastic/elasticsearch/issues/92991

docs/changelog/95342.yaml

@@ -0,0 +1,6 @@
+pr: 95342
+summary: Add Watcher APIs for updating/retrieving settings
+area: Watcher
+type: enhancement
+issues:
+ - 92991

rest-api-spec/src/main/resources/rest-api-spec/api/watcher.get_settings.json

@@ -0,0 +1,23 @@
+{
+  "watcher.get_settings":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-get-settings.html",
+      "description":"Retrieve settings for the watcher system index"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_watcher/settings",
+          "methods":["GET"]
+        }
+      ]
+    },
+    "params":{}
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/watcher.update_settings.json

@@ -0,0 +1,27 @@
+{
+  "watcher.update_settings":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-update-settings.html",
+      "description":"Update settings for the watcher system index"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_watcher/settings",
+          "methods":["PUT"]
+        }
+      ]
+    },
+    "params":{},
+    "body":{
+      "description": "An object with the new index settings",
+      "required": true
+    }
+  }
+}

x-pack/docs/en/rest-api/watcher.asciidoc

@@ -10,6 +10,8 @@
 * <<watcher-api-ack-watch>>
 * <<watcher-api-activate-watch>>
 * <<watcher-api-deactivate-watch>>
+* <<watcher-api-update-settings>>
+* <<watcher-api-get-settings>>
 * <<watcher-api-stats>>
 * <<watcher-api-stop>>
 * <<watcher-api-start>>
@@ -31,6 +33,10 @@ include::watcher/stats.asciidoc[]
 include::watcher/query-watches.asciidoc[]
 //PUT
 include::watcher/put-watch.asciidoc[]
+// UPDATE-SETTINGS
+include::watcher/update-settings.asciidoc[]
+// GET-SETTINGS
+include::watcher/get-settings.asciidoc[]
 //START
 include::watcher/start.asciidoc[]
 //STOP

x-pack/docs/en/rest-api/watcher/get-settings.asciidoc

@@ -0,0 +1,20 @@
+[role="xpack"]
+[[watcher-api-get-settings]]
+=== Get Watcher index settings
+++++
+<titleabbrev>Get Watcher settings</titleabbrev>
+++++
+
+This API allows a user to retrieve the user-configurable settings for the Watcher internal index (`.watches`). Only a subset of the index settings—those that are user-configurable—will be shown. This includes:
+
+- `index.auto_expand_replicas`
+- `index.number_of_replicas`
+
+An example of retrieving the Watcher settings:
+
+[source,console]
+-----------------------------------------------------------
+GET /_watcher/settings
+-----------------------------------------------------------
+
+The configurable settings can be modified using the <<watcher-api-update-settings,Update Watcher index settings>> API.

x-pack/docs/en/rest-api/watcher/update-settings.asciidoc

@@ -0,0 +1,48 @@
+[role="xpack"]
+[[watcher-api-update-settings]]
+=== Update Watcher index settings
+++++
+<titleabbrev>Update Watcher settings</titleabbrev>
+++++
+
+This API allows a user to modify the settings for the Watcher internal index (`.watches`). Only a subset of settings are allowed to by modified. This includes:
+
+- `index.auto_expand_replicas`
+- `index.number_of_replicas`
+
+An example of modifying the Watcher settings:
+
+[source,console]
+----------------------------------------------------------------
+PUT /_watcher/watch/test_watch
+{
+  "trigger": {
+    "schedule": {
+      "hourly": {
+        "minute": [ 0, 5 ]
+        }
+      }
+  },
+  "input": {
+    "simple": {
+      "payload": {
+        "send": "yes"
+      }
+    }
+  },
+  "condition": {
+    "always": {}
+  }
+}
+----------------------------------------------------------------
+// TESTSETUP
+
+[source,console]
+-----------------------------------------------------------
+PUT /_watcher/settings
+{
+  "index.auto_expand_replicas": "0-4"
+}
+-----------------------------------------------------------
+
+The configurable settings can be retrieved using the <<watcher-api-get-settings,Get Watcher index settings>> API.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.watcher.transport.actions.put;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public class GetWatcherSettingsAction extends ActionType<GetWatcherSettingsAction.Response> {
+
+    public static final GetWatcherSettingsAction INSTANCE = new GetWatcherSettingsAction();
+    public static final String NAME = "cluster:admin/xpack/watcher/settings/get";
+
+    public GetWatcherSettingsAction() {
+        super(NAME, GetWatcherSettingsAction.Response::new);
+    }
+
+    public static class Request extends MasterNodeReadRequest<Request> {
+
+        public Request() {}
+
+        public Request(StreamInput in) throws IOException {}
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {}
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private final Settings settings;
+
+        public Response(Settings settings) {
+            this.settings = settings;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            this.settings = Settings.readSettingsFromStream(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            this.settings.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            this.settings.toXContent(builder, params);
+            builder.endObject();
+            return builder;
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.watcher.transport.actions.put;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.set.Sets;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public class UpdateWatcherSettingsAction extends ActionType<AcknowledgedResponse> {
+
+    public static final UpdateWatcherSettingsAction INSTANCE = new UpdateWatcherSettingsAction();
+    public static final String NAME = "cluster:admin/xpack/watcher/settings/update";
+
+    public static final Set<String> ALLOWED_SETTING_KEYS = Set.of(
+        IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
+        IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS
+    );
+
+    public UpdateWatcherSettingsAction() {
+        super(NAME, AcknowledgedResponse::readFrom);
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> {
+        private final Map<String, Object> settings;
+
+        public Request(Map<String, Object> settings) {
+            this.settings = settings;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            this.settings = in.readMap();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeGenericMap(this.settings);
+        }
+
+        public Map<String, Object> settings() {
+            return this.settings;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            Set<String> forbiddenSettings = Sets.difference(settings.keySet(), ALLOWED_SETTING_KEYS);
+            if (forbiddenSettings.size() > 0) {
+                return ValidateActions.addValidationError(
+                    "illegal settings: "
+                        + forbiddenSettings
+                        + ", these settings may not be configured. Only the following settings may be configured: "
+                        + ALLOWED_SETTING_KEYS,
+                    null
+                );
+            }
+            return null;
+        }
+    }
+}

x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java

@@ -244,6 +244,8 @@ public class Constants {
         "cluster:admin/xpack/watcher/watch/activate",
         "cluster:admin/xpack/watcher/watch/delete",
         "cluster:admin/xpack/watcher/watch/execute",
+        "cluster:admin/xpack/watcher/settings/get",
+        "cluster:admin/xpack/watcher/settings/update",
         "cluster:admin/xpack/watcher/watch/put",
         "cluster:internal/remote_cluster/nodes",
         "cluster:internal/xpack/ml/datafeed/isolate",

x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/update_settings/10_update_watcher_settings.yml

@@ -0,0 +1,66 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+
+---
+"Test update and get watch settings api":
+  - do:
+      watcher.put_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "trigger": {
+              "schedule": {
+                "hourly": {
+                  "minute": [ 0, 5 ]
+                  }
+                }
+            },
+            "input": {
+              "simple": {
+                "payload": {
+                  "send": "yes"
+                }
+              }
+            },
+            "condition": {
+              "always": {}
+            },
+            "actions": {
+              "test_index": {
+                "index": {
+                  "index": "test"
+                }
+              }
+            }
+          }
+  - match: { _id: "my_watch" }
+
+  - do:
+      watcher.get_settings: {}
+
+  -  match: { index.auto_expand_replicas: "0-1" }
+  -  match: { index.number_of_replicas: "0" }
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: "0-all"
+
+  - do:
+      watcher.get_settings: {}
+
+  -  match: { index.auto_expand_replicas: "0-all" }
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: null
+          index.number_of_replicas: 1
+
+  - do:
+      watcher.get_settings: {}
+
+  -  match: { index.number_of_replicas: "1" }

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -81,7 +81,9 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateW
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
@@ -150,8 +152,10 @@ import org.elasticsearch.xpack.watcher.rest.action.RestActivateWatchAction.Deact
 import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestExecuteWatchAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestGetWatchAction;
+import org.elasticsearch.xpack.watcher.rest.action.RestGetWatcherSettingsAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestPutWatchAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestQueryWatchesAction;
+import org.elasticsearch.xpack.watcher.rest.action.RestUpdateWatcherSettingsAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestWatchServiceAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestWatcherStatsAction;
 import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry;
@@ -166,8 +170,10 @@ import org.elasticsearch.xpack.watcher.transport.actions.TransportActivateWatchA
 import org.elasticsearch.xpack.watcher.transport.actions.TransportDeleteWatchAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportExecuteWatchAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportGetWatchAction;
+import org.elasticsearch.xpack.watcher.transport.actions.TransportGetWatcherSettingsAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportPutWatchAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportQueryWatchesAction;
+import org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportWatcherServiceAction;
 import org.elasticsearch.xpack.watcher.transport.actions.TransportWatcherStatsAction;
 import org.elasticsearch.xpack.watcher.trigger.TriggerEngine;
@@ -675,6 +681,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin,
             new ActionHandler<>(WatcherServiceAction.INSTANCE, TransportWatcherServiceAction.class),
             new ActionHandler<>(ExecuteWatchAction.INSTANCE, TransportExecuteWatchAction.class),
             new ActionHandler<>(QueryWatchesAction.INSTANCE, TransportQueryWatchesAction.class),
+            new ActionHandler<>(UpdateWatcherSettingsAction.INSTANCE, TransportUpdateWatcherSettingsAction.class),
+            new ActionHandler<>(GetWatcherSettingsAction.INSTANCE, TransportGetWatcherSettingsAction.class),
             usageAction,
             infoAction
         );
@@ -704,7 +712,9 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin,
             new RestActivateWatchAction(),
             new DeactivateRestHandler(),
             new RestExecuteWatchAction(),
-            new RestQueryWatchesAction()
+            new RestQueryWatchesAction(),
+            new RestUpdateWatcherSettingsAction(),
+            new RestGetWatcherSettingsAction()
         );
     }
 

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.rest.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Allows retrieving a subset of index settings (those use-settable) for the .watches index.
+ * See {@link RestUpdateWatcherSettingsAction} for the setting counterpart.
+ */
+public class RestGetWatcherSettingsAction extends BaseRestHandler {
+    @Override
+    public String getName() {
+        return "watcher_get_settings";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(Route.builder(RestRequest.Method.GET, "/_watcher/settings").build());
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        GetWatcherSettingsAction.Request req = new GetWatcherSettingsAction.Request();
+        return channel -> client.execute(GetWatcherSettingsAction.INSTANCE, req, new RestToXContentListener<>(channel));
+    }
+}

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.rest.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Allows setting a subset of index settings for the .watches index.
+ * See {@link RestGetWatcherSettingsAction} for the retrieval counterpart.
+ */
+public class RestUpdateWatcherSettingsAction extends BaseRestHandler {
+    @Override
+    public String getName() {
+        return "watcher_update_settings";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(Route.builder(RestRequest.Method.PUT, "/_watcher/settings").build());
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        UpdateWatcherSettingsAction.Request req = new UpdateWatcherSettingsAction.Request(request.contentParser().map());
+        return channel -> client.execute(UpdateWatcherSettingsAction.INSTANCE, req, new RestToXContentListener<>(channel));
+    }
+}

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.transport.actions;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
+
+import static org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction.WATCHER_INDEX_NAME;
+
+public class TransportGetWatcherSettingsAction extends TransportMasterNodeAction<
+    GetWatcherSettingsAction.Request,
+    GetWatcherSettingsAction.Response> {
+
+    @Inject
+    public TransportGetWatcherSettingsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver
+    ) {
+        super(
+            GetWatcherSettingsAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            GetWatcherSettingsAction.Request::new,
+            indexNameExpressionResolver,
+            GetWatcherSettingsAction.Response::new,
+            ThreadPool.Names.SAME
+        );
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        GetWatcherSettingsAction.Request request,
+        ClusterState state,
+        ActionListener<GetWatcherSettingsAction.Response> listener
+    ) {
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        // Stashing and un-stashing the context allows warning headers about accessing a system index to be ignored.
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            IndexMetadata metadata = state.metadata().index(WATCHER_INDEX_NAME);
+            if (metadata == null) {
+                listener.onResponse(new GetWatcherSettingsAction.Response(Settings.EMPTY));
+            } else {
+                listener.onResponse(new GetWatcherSettingsAction.Response(filterSettableSettings(metadata.getSettings())));
+            }
+        }
+    }
+
+    /**
+     * Filters the settings to only those settable by the user (using the update watcher settings API).
+     */
+    private static Settings filterSettableSettings(Settings settings) {
+        Settings.Builder builder = Settings.builder();
+        for (String settingName : UpdateWatcherSettingsAction.ALLOWED_SETTING_KEYS) {
+            if (settings.hasValue(settingName)) {
+                builder.put(settingName, settings.get(settingName));
+            }
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(GetWatcherSettingsAction.Request request, ClusterState state) {
+        ClusterBlockException globalBlock = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+        if (globalBlock != null) {
+            return globalBlock;
+        }
+        return state.blocks()
+            .indicesBlockedException(
+                ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, WATCHER_INDEX_NAME)
+            );
+    }
+}

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.transport.actions;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
+
+public class TransportUpdateWatcherSettingsAction extends TransportMasterNodeAction<
+    UpdateWatcherSettingsAction.Request,
+    AcknowledgedResponse> {
+
+    public static final String WATCHER_INDEX_NAME = ".watches";
+
+    private static final Logger logger = LogManager.getLogger(TransportUpdateWatcherSettingsAction.class);
+    private final MetadataUpdateSettingsService updateSettingsService;
+
+    @Inject
+    public TransportUpdateWatcherSettingsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        MetadataUpdateSettingsService updateSettingsService,
+        IndexNameExpressionResolver indexNameExpressionResolver
+    ) {
+        super(
+            UpdateWatcherSettingsAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            UpdateWatcherSettingsAction.Request::new,
+            indexNameExpressionResolver,
+            AcknowledgedResponse::readFrom,
+            ThreadPool.Names.SAME
+        );
+        this.updateSettingsService = updateSettingsService;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        UpdateWatcherSettingsAction.Request request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> listener
+    ) {
+        final IndexMetadata watcherIndexMd = state.metadata().index(WATCHER_INDEX_NAME);
+        if (watcherIndexMd == null) {
+            // Index does not exist, so fail fast
+            listener.onFailure(new ResourceNotFoundException("no Watches found on which to modify settings"));
+            return;
+        }
+        final Settings newSettings = Settings.builder().loadFromMap(request.settings()).build();
+        final UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest().indices(
+            new Index[] { watcherIndexMd.getIndex() }
+        ).settings(newSettings).ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout());
+
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        // Stashing and un-stashing the context allows warning headers about accessing a system index to be ignored.
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            updateSettingsService.updateSettings(clusterStateUpdateRequest, new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    if (acknowledgedResponse.isAcknowledged()) {
+                        logger.info("successfully updated Watcher service settings to {}", request.settings());
+                    } else {
+                        logger.warn("updating Watcher service settings to {} was not acknowledged", request.settings());
+                    }
+                    listener.onResponse(acknowledgedResponse);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    logger.debug(() -> "failed to update settings for Watcher service", e);
+                    listener.onFailure(e);
+                }
+            });
+        }
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(UpdateWatcherSettingsAction.Request request, ClusterState state) {
+        ClusterBlockException globalBlock = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        if (globalBlock != null) {
+            return globalBlock;
+        }
+        return state.blocks()
+            .indicesBlockedException(
+                ClusterBlockLevel.METADATA_WRITE,
+                indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, WATCHER_INDEX_NAME)
+            );
+    }
+}