Run X-Pack YAML tests on Multi Project (MP-1731)

This runs the X-Pack YAML test suite against a cluster with multiple
projects

x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle

@@ -0,0 +1,153 @@
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.util.GradleUtils
+
+dependencies {
+  testImplementation project(xpackModule('core'))
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation project(':test:yaml-rest-runner')
+  testImplementation(testArtifact(project(":x-pack:plugin:security:qa:service-account"), "javaRestTest"))
+}
+
+// let the yamlRestTests see the classpath of test
+GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest"))
+
+restResources {
+  restTests {
+    includeXpack '*'
+  }
+}
+
+tasks.named("yamlRestTest").configure {
+  usesDefaultDistribution()
+  ArrayList<String> blacklist = [
+    /* These tests don't work on multi-project yet - we need to go through each of them and make them work */
+    'aggregate-metrics/*/*',
+    'analytics/*/*',
+    'api_key/*/*',
+    'async_search/*/*',
+    'authenticate/*/*',
+    'change_password/*/*',
+    'constant_keyword/*/*',
+    'counted_keyword/*/*',
+    'data_streams/*/*',
+    'deprecation/*/*',
+    'dlm/*/*',
+    'enrich/*/*',
+    'esql/*/*',
+    'graph/*/*',
+    'health/*/*',
+    'ml/3rd_party_deployment/*',
+    'ml/bucket_correlation_agg/*',
+    'ml/bucket_count_ks_test_agg/*',
+    'ml/calendar_crud/*',
+    'ml/categorization_agg/*',
+    'ml/change_point_agg/*',
+    'ml/custom_all_field/*',
+    'ml/data_frame_analytics_cat_apis/*',
+    'ml/data_frame_analytics_crud/*',
+    'ml/data_frame_analytics_crud/*/*',
+    'ml/datafeed_cat_apis/*',
+    'ml/datafeeds_crud/*',
+    'ml/delete_expired_data/*',
+    'ml/delete_job_force/*',
+    'ml/evaluate_data_frame/*',
+    'ml/explain_data_frame_analytics/*',
+    'ml/filter_crud/*',
+    'ml/forecast/*',
+    'ml/frequent_item_sets_agg/*',
+    'ml/get_datafeed_stats/*',
+    'ml/get_datafeeds/*',
+    'ml/get_memory_stats/*',
+    'ml/get_model_snapshots/*',
+    'ml/get_model_snapshots/*/*',
+    'ml/get_trained_model_stats/*',
+    'ml/inference_crud/*',
+    'ml/inference_processor/*',
+    'ml/job_cat_apis/*',
+    'ml/job_groups/*',
+    'ml/jobs_crud/*',
+    'ml/jobs_get/*',
+    'ml/jobs_get_result_buckets/*',
+    'ml/jobs_get_result_categories/*',
+    'ml/jobs_get_result_influencers/*',
+    'ml/jobs_get_result_overall_buckets/*',
+    'ml/jobs_get_result_records/*',
+    'ml/jobs_get_stats/*',
+    'ml/learning_to_rank_rescorer/*',
+    'ml/ml_anomalies_default_mappings/*',
+    'ml/ml_info/*',
+    'ml/p_value_significant_term_score/*',
+    'ml/pipeline_inference/*',
+    'ml/post_data/*',
+    'ml/preview_data_frame_analytics/*',
+    'ml/preview_datafeed/*',
+    'ml/reset_job/*',
+    'ml/revert_model_snapshot/*',
+    'ml/search_knn_query_vector_builder/*',
+    'ml/set_upgrade_mode/*',
+    'ml/sparse_vector_search/*',
+    'ml/start_data_frame_analytics/*',
+    'ml/start_stop_datafeed/*',
+    'ml/stop_data_frame_analytics/*',
+    'ml/text_embedding_search/*',
+    'ml/text_expansion_search/*',
+    'ml/text_expansion_search_rank_features/*',
+    'ml/text_expansion_search_sparse_vector/*',
+    'ml/trained_model_cat_apis/*',
+    'ml/update_trained_model_deployment/*',
+    'ml/upgrade_job_snapshot/*',
+    'monitoring/bulk/*/*',
+    'privileges/10_basic/*',
+    'privileges/20_has_application_privs/*',
+    'privileges/30_superuser/*',
+    'privileges/40_get_user_privs/*',
+    'profiling/*/*',
+    'redact/*/*',
+    'role_mapping/*/*',
+    'roles/10_basic/*',
+    'roles/11_idx_arrays/*',
+    'roles/40_global_privileges/*',
+    'roles/50_remote_only/*',
+    'roles/60_bulk_roles/*',
+    'rollup/*/*',
+    'search-business-rules/*/*',
+    'searchable_snapshots/*/*',
+    'security/authz/*/*',
+    'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices',
+    'security/authz_api_keys/*/*',
+    'security/hidden-index/*/*',
+    'security/settings/*/*',
+    'service_accounts/*/*',
+    'set_security_user/*/*',
+    'snapshot/10_basic/*',
+    'snapshot/20_operator_privileges_disabled/*',
+    'spatial/*/*',
+    'sql/sql/*',
+    'sql/translate/*',
+    'terms_enum/*/*',
+    'text_structure/find_field_structure/*',
+    'token/*/*',
+    'transform/*/*',
+    'transform/transforms_start_stop/Test start/stop only starts/stops specified transform',
+    'transform/transforms_start_stop/Test start/stop with field alias',
+    'transform/transforms_start_stop/Test start/stop/start continuous transform',
+    'transform/transforms_start_stop/Test start/stop/start transform',
+    'user_profile/*/*',
+    'users/10_basic/*',
+    'users/15_overwrite_user/*',
+    'users/16_update_user/*',
+    'users/30_enable_disable/*',
+    'users/31_create_disabled/*',
+    'users/40_query/*',
+    'vector-tile/*/*',
+    'versionfield/*/*',
+    'voting_only_node/*/*',
+    'wildcard/*/*',
+  ];
+  if (BuildParams.isSnapshotBuild() == false) {
+    blacklist += [];
+  }
+  systemProperty 'tests.rest.blacklist', blacklist.join(',')
+}

x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/src/yamlRestTest/java/org/elasticsearch/multiproject/test/XpackWithMultipleProjectsClientYamlTestSuiteIT.java

@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.multiproject.test;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+import org.apache.lucene.tests.util.TimeUnits;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.FixForMultiProject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
+public class XpackWithMultipleProjectsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin"));
+    private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password", "test-password"));
+
+    // The active project-id is slightly longer, and has a fixed suffix so that its easier to pick in error messages etc.
+    private final String activeProject = randomAlphaOfLength(8).toLowerCase(Locale.ROOT) + "00active";
+    private final Set<String> extraProjects = randomSet(1, 3, () -> randomAlphaOfLength(12).toLowerCase(Locale.ROOT));
+
+    /* This should be true, but there are listeners running in X-Pack that prevent us from creating an index */
+    @FixForMultiProject
+    private static boolean firstRun = false;
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .name("yamlRestTest")
+        .setting("xpack.ml.enabled", "true")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.watcher.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.authc.token.enabled", "true")
+        .setting("xpack.security.authc.api_key.enabled", "true")
+        .setting("xpack.security.transport.ssl.enabled", "true")
+        .setting("xpack.security.transport.ssl.key", "testnode.pem")
+        .setting("xpack.security.transport.ssl.certificate", "testnode.crt")
+        .setting("xpack.security.transport.ssl.verification_mode", "certificate")
+        .setting("xpack.security.audit.enabled", "true")
+        .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode")
+        .configFile("testnode.pem", Resource.fromClasspath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+        .configFile("testnode.crt", Resource.fromClasspath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+        .configFile("service_tokens", Resource.fromClasspath("service_tokens"))
+        .user(USER, PASS)
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.FAILURE_STORE_ENABLED)
+        .build();
+
+    public XpackWithMultipleProjectsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @Before
+    public void configureProjects() throws Exception {
+        super.initClient();
+        createProject(activeProject);
+        for (var project : extraProjects) {
+            createProject(project);
+        }
+
+        if (firstRun) {
+            firstRun = false;
+            // To verify everything is configured correctly, we create an index in the default project (using the admin client)
+            // And check it's not visible by the regular (project) client
+            // This ensures that the regular client (used by the yaml tests) is correctly pointing to a non-default project
+            createIndex(adminClient(), "test-index", indexSettings(1, 1).build());
+            assertThat(indexExists(adminClient(), "test-index"), equalTo(true));
+            assertThat(indexExists(client(), "test-index"), equalTo(false));
+        }
+    }
+
+    @After
+    public void removeProjects() throws Exception {
+        for (var project : extraProjects) {
+            deleteProject(project);
+        }
+        deleteProject(activeProject);
+    }
+
+    private void createProject(String project) throws IOException {
+        RestClient client = adminClient();
+        final Request request = new Request("PUT", "/_project/" + project);
+        try {
+            final Response response = client.performRequest(request);
+            logger.info("Created project {} : {}", project, response.getStatusLine());
+        } catch (ResponseException e) {
+            logger.error("Failed to create project: {}", project);
+            throw e;
+        }
+    }
+
+    private void deleteProject(String project) throws IOException {
+        var client = adminClient();
+        final Request request = new Request("DELETE", "/_project/" + project);
+        try {
+            final Response response = client.performRequest(request);
+            logger.info("Deleted project {} : {}", project, response.getStatusLine());
+        } catch (ResponseException e) {
+            logger.error("Failed to delete project: {}", project);
+            throw e;
+        }
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        return clientSettings(true);
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        return clientSettings(false);
+    }
+
+    private Settings clientSettings(boolean projectScoped) {
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        final Settings.Builder builder = Settings.builder()
+            .put(super.restClientSettings())
+            .put(ThreadContext.PREFIX + ".Authorization", token);
+        if (projectScoped) {
+            builder.put(ThreadContext.PREFIX + "." + Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, activeProject);
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}