Browse Source

Adding manage_dlm privilege (#95512)

This adds a new index privilege called `manage_dlm`. The `manage_dlm`
has permission to perform all DLM actions on an index, including put and
delete. It also adds the ability to call DLM get and explain to the
`view_index_metadata` existing index privilege.
Keith Massey 2 years ago
parent
commit
322805858f

+ 5 - 0
docs/changelog/95512.yaml

@@ -0,0 +1,5 @@
+pr: 95512
+summary: Adding `manage_dlm` index privilege and expanding `view_index_metadata` for access to data lifecycle APIs
+area: DLM
+type: enhancement
+issues: []

+ 6 - 0
docs/reference/dlm/apis/delete-lifecycle.asciidoc

@@ -8,6 +8,12 @@ experimental::[]
 
 Deletes the lifecycle from a set of data streams.
 
+[[delete-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to
+use this API. For more information, see <<security-privileges>>.
+
 [[dlm-delete-lifecycle-request]]
 ==== {api-request-title}
 

+ 8 - 0
docs/reference/dlm/apis/explain-data-lifecycle.asciidoc

@@ -8,6 +8,14 @@ experimental::[]
 
 Retrieves the current data lifecycle status for one or more data stream backing indices.
 
+[[explain-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* Nit: would rephrase as:
+
+If the {es} {security-features} are enabled, you must have at least the `manage_dlm` index privilege or
+`view_index_metadata` index privilege to use this API. For more information, see <<security-privileges>>.
+
 [[dlm-explain-lifecycle-request]]
 ==== {api-request-title}
 

+ 7 - 0
docs/reference/dlm/apis/get-lifecycle.asciidoc

@@ -8,6 +8,13 @@ experimental::[]
 
 Gets the lifecycle of a set of data streams.
 
+[[get-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have at least one of the `manage`
+<<privileges-list-indices,index privilege>>, the `manage_dlm` index privilege, or the
+`view_index_metadata` privilege to use this API. For more information, see <<security-privileges>>.
+
 [[dlm-get-lifecycle-request]]
 ==== {api-request-title}
 

+ 6 - 0
docs/reference/dlm/apis/put-lifecycle.asciidoc

@@ -8,6 +8,12 @@ experimental::[]
 
 Configures the data lifecycle for the targeted data streams.
 
+[[put-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to use this API.
+For more information, see <<security-privileges>>.
+
 [[dlm-put-lifecycle-request]]
 ==== {api-request-title}
 

+ 0 - 0
modules/dlm/qa/build.gradle


+ 19 - 0
modules/dlm/qa/with-security/build.gradle

@@ -0,0 +1,19 @@
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.authenticated-testclusters'
+
+dependencies {
+  javaRestTestImplementation project(":client:rest-high-level")
+}
+
+testClusters.configureEach {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  rolesFile file('roles.yml')
+  user username: "test_dlm", password: "x-pack-test-password", role: "manage_dlm"
+  user username: "test_non_privileged", password: "x-pack-test-password", role: "not_privileged"
+  requiresFeature 'es.dlm_feature_flag_enabled', Version.fromString("8.9.0")
+}

+ 18 - 0
modules/dlm/qa/with-security/roles.yml

@@ -0,0 +1,18 @@
+manage_dlm:
+  cluster:
+    - monitor
+  indices:
+    - names: [ 'dlm-*' ]
+      privileges:
+        - read
+        - write
+        - manage_dlm
+not_privileged:
+  cluster:
+    - monitor
+  indices:
+    - names: [ 'dlm-*' ]
+      privileges:
+        - read
+        - write
+        - view_index_metadata

+ 165 - 0
modules/dlm/qa/with-security/src/javaRestTest/java/org/elasticsearch/dlm/PermissionsIT.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.dlm;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PermissionsIT extends ESRestTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        // Note: This user is defined in build.gradle, and assigned the role "manage_dlm". That role is defined in roles.yml.
+        String token = basicAuthHeaderValue("test_dlm", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    private Settings restUnprivilegedClientSettings() {
+        // Note: This user is defined in build.gradle, and assigned the role "not_privileged". That role is defined in roles.yml.
+        String token = basicAuthHeaderValue("test_non_privileged", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testManageDLM() throws Exception {
+        {
+            /*
+             * This test checks that a user with the "manage_dlm" index privilege on "dlm-*" data streams can delete and put a lifecycle
+             * on the "dlm-test" data stream, while a user with who does not have that privilege (but does have all of the other same
+             * "dlm-*" privileges) cannot delete or put a lifecycle on that datastream.
+             */
+            String dataStreamName = "dlm-test"; // Needs to match the pattern of the names in roles.yml
+            createDataStreamAsAdmin(dataStreamName);
+            Response getDatastreamRepsonse = adminClient().performRequest(new Request("GET", "/_data_stream/" + dataStreamName));
+            final List<Map<String, Object>> nodes = ObjectPath.createFromResponse(getDatastreamRepsonse).evaluate("data_streams");
+            String index = (String) ((List<Map<String, Object>>) nodes.get(0).get("indices")).get(0).get("index_name");
+
+            Request explainLifecycleRequest = new Request("GET", "/" + randomFrom("_all", "*", index) + "/_lifecycle/explain");
+            Request getLifecycleRequest = new Request("GET", "_data_stream/" + randomFrom("_all", "*", dataStreamName) + "/_lifecycle");
+            Request deleteLifecycleRequest = new Request(
+                "DELETE",
+                "_data_stream/" + randomFrom("_all", "*", dataStreamName) + "/_lifecycle"
+            );
+            Request putLifecycleRequest = new Request("PUT", "_data_stream/" + randomFrom("_all", "*", dataStreamName) + "/_lifecycle");
+            putLifecycleRequest.setJsonEntity("{}");
+
+            makeRequest(client(), explainLifecycleRequest, true);
+            makeRequest(client(), getLifecycleRequest, true);
+            makeRequest(client(), deleteLifecycleRequest, true);
+            makeRequest(client(), putLifecycleRequest, true);
+
+            try (
+                RestClient nonDlmManagerClient = buildClient(restUnprivilegedClientSettings(), getClusterHosts().toArray(new HttpHost[0]))
+            ) {
+                makeRequest(nonDlmManagerClient, explainLifecycleRequest, true);
+                makeRequest(nonDlmManagerClient, getLifecycleRequest, true);
+                makeRequest(nonDlmManagerClient, deleteLifecycleRequest, false);
+                makeRequest(nonDlmManagerClient, putLifecycleRequest, false);
+            }
+        }
+        {
+            // Now test that the user who has the manage_dlm privilege on dlm-* data streams cannot manage other data streams:
+            String otherDataStreamName = "other-dlm-test";
+            createDataStreamAsAdmin(otherDataStreamName);
+            Response getOtherDataStreamResponse = adminClient().performRequest(new Request("GET", "/_data_stream/" + otherDataStreamName));
+            final List<Map<String, Object>> otherNodes = ObjectPath.createFromResponse(getOtherDataStreamResponse).evaluate("data_streams");
+            String otherIndex = (String) ((List<Map<String, Object>>) otherNodes.get(0).get("indices")).get(0).get("index_name");
+            Request putOtherLifecycleRequest = new Request("PUT", "_data_stream/" + otherDataStreamName + "/_lifecycle");
+            putOtherLifecycleRequest.setJsonEntity("{}");
+            makeRequest(client(), new Request("GET", "/" + otherIndex + "/_lifecycle/explain"), false);
+            makeRequest(client(), new Request("GET", "_data_stream/" + otherDataStreamName + "/_lifecycle"), false);
+            makeRequest(client(), new Request("DELETE", "_data_stream/" + otherDataStreamName + "/_lifecycle"), false);
+            makeRequest(client(), putOtherLifecycleRequest, false);
+        }
+    }
+
+    /*
+     * This makes the given request with the given client. It asserts a 200 response if expectSuccess is true, and asserts an exception
+     * with a 403 response if expectStatus is false.
+     */
+    private void makeRequest(RestClient client, Request request, boolean expectSuccess) throws IOException {
+        if (expectSuccess) {
+            Response response = client.performRequest(request);
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+        } else {
+            ResponseException exception = expectThrows(ResponseException.class, () -> client.performRequest(request));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.FORBIDDEN.getStatus()));
+        }
+    }
+
+    private void createDataStreamAsAdmin(String name) throws IOException {
+        String mappingsTemplateName = name + "_mappings";
+        Request mappingsRequest = new Request("PUT", "/_component_template/" + mappingsTemplateName);
+        mappingsRequest.setJsonEntity("""
+            {
+              "template": {
+                "mappings": {
+                  "properties": {
+                    "@timestamp": {
+                      "type": "date",
+                      "format": "date_optional_time||epoch_millis"
+                    },
+                    "message": {
+                      "type": "wildcard"
+                    }
+                  }
+                }
+              }
+            }""");
+        assertOK(adminClient().performRequest(mappingsRequest));
+
+        String settingsTemplateName = name + "_settings";
+        Request settingsRequest = new Request("PUT", "/_component_template/" + settingsTemplateName);
+        settingsRequest.setJsonEntity("""
+            {
+              "template": {
+                  "settings": {
+                    "number_of_shards": 1,
+                    "number_of_replicas": 0
+                  }
+              }
+            }""");
+        assertOK(adminClient().performRequest(settingsRequest));
+
+        Request indexTemplateRequest = new Request("PUT", "/_index_template/" + name + "_template");
+        indexTemplateRequest.setJsonEntity(Strings.format("""
+            {
+                "index_patterns": ["%s*"],
+                "data_stream": { },
+                "composed_of": [ "%s", "%s" ]
+            }""", name, mappingsTemplateName, settingsTemplateName));
+        assertOK(adminClient().performRequest(indexTemplateRequest));
+
+        Request request = new Request("PUT", "/_data_stream/" + name);
+        assertOK(adminClient().performRequest(request));
+    }
+
+}

+ 6 - 0
x-pack/docs/en/security/authorization/privileges.asciidoc

@@ -303,6 +303,12 @@ All {Ilm} operations relating to managing the execution of policies of an index
 or data stream. This includes operations such as retrying policies and removing
 a policy from an index or data stream.
 
+ifeval::["{release-state}"!="released"]
+`manage_dlm`::
+All {Dlm} operations relating to reading and managing the lifecycle of a data stream.
+This includes operations such as adding and removing a lifecycle from a data stream.
+endif::[]
+
 `manage_leader_index`::
 All actions that are required to manage the lifecycle of a leader index, which
 includes <<ccr-post-forget-follower,forgetting a follower>>. This

+ 6 - 0
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java

@@ -30,6 +30,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.datastreams.PromoteDataStreamAction;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
 import org.elasticsearch.action.search.SearchShardsAction;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
 import org.elasticsearch.transport.TcpTransport;
@@ -118,6 +119,8 @@ public final class IndexPrivilege extends Privilege {
         ValidateQueryAction.NAME + "*",
         GetSettingsAction.NAME,
         ExplainLifecycleAction.NAME,
+        "indices:admin/dlm/get",
+        "indices:admin/dlm/explain",
         GetDataStreamAction.NAME,
         ResolveIndexAction.NAME,
         FieldCapabilitiesAction.NAME + "*",
@@ -133,6 +136,7 @@ public final class IndexPrivilege extends Privilege {
     );
     private static final Automaton MANAGE_LEADER_INDEX_AUTOMATON = patterns(ForgetFollowerAction.NAME + "*");
     private static final Automaton MANAGE_ILM_AUTOMATON = patterns("indices:admin/ilm/*");
+    private static final Automaton MANAGE_DLM_AUTOMATON = patterns("indices:admin/dlm/*");
     private static final Automaton MAINTENANCE_AUTOMATON = patterns(
         "indices:admin/refresh*",
         "indices:admin/flush*",
@@ -173,6 +177,7 @@ public final class IndexPrivilege extends Privilege {
     public static final IndexPrivilege MANAGE_FOLLOW_INDEX = new IndexPrivilege("manage_follow_index", MANAGE_FOLLOW_INDEX_AUTOMATON);
     public static final IndexPrivilege MANAGE_LEADER_INDEX = new IndexPrivilege("manage_leader_index", MANAGE_LEADER_INDEX_AUTOMATON);
     public static final IndexPrivilege MANAGE_ILM = new IndexPrivilege("manage_ilm", MANAGE_ILM_AUTOMATON);
+    public static final IndexPrivilege MANAGE_DLM = new IndexPrivilege("manage_dlm", MANAGE_DLM_AUTOMATON);
     public static final IndexPrivilege MAINTENANCE = new IndexPrivilege("maintenance", MAINTENANCE_AUTOMATON);
     public static final IndexPrivilege AUTO_CONFIGURE = new IndexPrivilege("auto_configure", AUTO_CONFIGURE_AUTOMATON);
     public static final IndexPrivilege CROSS_CLUSTER_REPLICATION = new IndexPrivilege(
@@ -204,6 +209,7 @@ public final class IndexPrivilege extends Privilege {
             entry("manage_follow_index", MANAGE_FOLLOW_INDEX),
             entry("manage_leader_index", MANAGE_LEADER_INDEX),
             entry("manage_ilm", MANAGE_ILM),
+            DataLifecycle.isEnabled() ? entry("manage_dlm", MANAGE_DLM) : null,
             entry("maintenance", MAINTENANCE),
             entry("auto_configure", AUTO_CONFIGURE),
             TcpTransport.isUntrustedRemoteClusterEnabled() ? entry("cross_cluster_replication", CROSS_CLUSTER_REPLICATION) : null,

+ 30 - 0
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java

@@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TcpTransport;
@@ -465,6 +466,35 @@ public class PrivilegeTests extends ESTestCase {
         }
     }
 
+    public void testDlmPrivileges() {
+        assumeTrue("feature flag required", DataLifecycle.isEnabled());
+        {
+            Predicate<String> predicate = IndexPrivilege.MANAGE_DLM.predicate();
+            // check indices actions
+            assertThat(predicate.test("indices:admin/dlm/explain"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/get"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/delete"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/put"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/brand_new_api"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/brand_new_api"), is(true));
+            // check non-dlm action
+            assertThat(predicate.test("indices:admin/whatever"), is(false));
+        }
+
+        {
+            Predicate<String> predicate = IndexPrivilege.VIEW_METADATA.predicate();
+            // check indices actions
+            assertThat(predicate.test("indices:admin/dlm/explain"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/get"), is(true));
+            assertThat(predicate.test("indices:admin/dlm/delete"), is(false));
+            assertThat(predicate.test("indices:admin/dlm/put"), is(false));
+            assertThat(predicate.test("indices:admin/dlm/brand_new_api"), is(false));
+            assertThat(predicate.test("indices:admin/dlm/brand_new_api"), is(false));
+            // check non-dlm action
+            assertThat(predicate.test("indices:admin/whatever"), is(false));
+        }
+    }
+
     public void testIngestPipelinePrivileges() {
         {
             verifyClusterActionAllowed(

+ 1 - 1
x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/privileges/11_builtin.yml

@@ -16,4 +16,4 @@ setup:
   # I would much prefer we could just check that specific entries are in the array, but we don't have
   # an assertion for that
   - length: { "cluster" : 48 }
-  - length: { "index" : 21 }
+  - length: { "index" : 22 }

+ 1 - 1
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java

@@ -348,7 +348,7 @@ public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     }
 
     private static RoleDescriptor randomRoleDescriptor(boolean includeRemoteIndices) {
-        final Set<String> excludedPrivileges = Set.of("cross_cluster_replication", "cross_cluster_replication_internal");
+        final Set<String> excludedPrivileges = Set.of("cross_cluster_replication", "cross_cluster_replication_internal", "manage_dlm");
         return new RoleDescriptor(
             randomAlphaOfLengthBetween(3, 90),
             randomSubsetOf(Set.of("all", "monitor", "none")).toArray(String[]::new),