Migrate x-pack-enrich legacy rest tests to new test framework (#131743) (#131906)

(cherry picked from commit 71130a6d32604bd86d323e4cce8ab65ef4b36d45)

# Conflicts:
#	build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
#	x-pack/plugin/enrich/build.gradle
#	x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle
#	x-pack/plugin/enrich/qa/rest-with-security/build.gradle
#	x-pack/plugin/enrich/qa/rest/build.gradle

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -58,9 +58,6 @@ public abstract class RestrictedBuildApiService implements BuildService<Restrict
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest-with-advanced-security");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");

x-pack/plugin/enrich/build.gradle

@@ -1,9 +1,12 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
 esplugin {
   name = 'x-pack-enrich'
   description = 'Elasticsearch Expanded Pack Plugin - Enrich'
-  classname ='org.elasticsearch.xpack.enrich.EnrichPlugin'
+  classname = 'org.elasticsearch.xpack.enrich.EnrichPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
@@ -19,6 +22,27 @@ dependencies {
   testImplementation project(path: ':modules:legacy-geo')
   testImplementation project(xpackModule('spatial'))
   testImplementation(testArtifact(project(xpackModule('monitoring'))))
+
+  clusterModules project(':modules:analysis-common')
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':modules:mapper-extras')
+
+  clusterModules project(xpackModule('monitoring'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('wildcard'))
+}
+
+tasks.named('yamlRestTest') {
+  // single tests can be run using the same cluster configuration as used for Java REST tests, but cleanup inbetween tests fails
+  usesDefaultDistribution("cleanUpCluster fails if not using the default distribution")
+}
+
+restResources {
+  restApi {
+    include '_common', 'bulk', 'indices', 'index', 'ingest.delete_pipeline', 'ingest.put_pipeline', 'enrich', 'get', 'capabilities'
+  }
+  restTests {
+    includeXpack 'enrich'
+  }
 }
 
-addQaCheckDependencies(project)

x-pack/plugin/enrich/qa/common/build.gradle

@@ -1,5 +0,0 @@
-apply plugin: 'elasticsearch.java'
-
-dependencies {
-  api project(':test:framework')
-}

x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle

@@ -1,18 +0,0 @@
-apply plugin: 'elasticsearch.legacy-java-rest-test'
-
-import org.elasticsearch.gradle.internal.info.BuildParams
-
-dependencies {
-  javaRestTestImplementation project(path: xpackModule('core'))
-  javaRestTestImplementation project(path: xpackModule('enrich:qa:common'))
-}
-
-testClusters.configureEach {
-  testDistribution = 'DEFAULT'
-  extraConfigFile 'roles.yml', file('roles.yml')
-  user username: "test_admin", password: "x-pack-test-password", role: "superuser"
-  user username: "test_enrich", password: "x-pack-test-password", role: "integ_test_role"
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.monitoring.collection.enabled', 'true'
-}

x-pack/plugin/enrich/qa/rest-with-security/build.gradle

@@ -1,23 +0,0 @@
-apply plugin: 'elasticsearch.legacy-java-rest-test'
-
-import org.elasticsearch.gradle.internal.info.BuildParams
-
-dependencies {
-  javaRestTestImplementation project(path: xpackModule('core'))
-  javaRestTestImplementation project(path: xpackModule('enrich:qa:common'))
-}
-if (buildParams.inFipsJvm){
-  // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
-  tasks.named("javaRestTest").configure{enabled = false }
-}
-
-testClusters.configureEach {
-  testDistribution = 'DEFAULT'
-  extraConfigFile 'roles.yml', file('roles.yml')
-  user username: "test_admin", password: "x-pack-test-password", role: "superuser"
-  user username: "test_enrich", password: "x-pack-test-password", role: "enrich_user,integ_test_role"
-  user username: "test_enrich_no_privs", password: "x-pack-test-password", role: "enrich_no_privs"
-  setting 'xpack.license.self_generated.type', 'basic'
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.monitoring.collection.enabled', 'true'
-}

x-pack/plugin/enrich/qa/rest/build.gradle

@@ -1,35 +0,0 @@
-apply plugin: 'elasticsearch.legacy-java-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
-
-
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.info.BuildParams
-
-restResources {
-  restApi {
-    include '_common', 'bulk', 'indices', 'index', 'ingest.delete_pipeline', 'ingest.put_pipeline', 'enrich', 'get', 'capabilities'
-  }
-  restTests {
-    includeXpack 'enrich'
-  }
-}
-
-dependencies {
-  javaRestTestImplementation project(path: xpackModule('enrich:qa:common'))
-}
-
-if (buildParams.inFipsJvm){
-  // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
-  tasks.named("javaRestTest").configure{enabled = false }
-  tasks.named("yamlRestTest").configure{enabled = false }
-}
-
-testClusters.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.license.self_generated.type', 'basic'
-  setting 'xpack.monitoring.collection.enabled', 'true'
-  setting 'xpack.security.enabled', 'false'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.4.0")
-}
-

x-pack/plugin/enrich/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichIT.java

@@ -1,11 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.enrich;
-
-import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
-
-public class EnrichIT extends CommonEnrichRestTestCase {}

x-pack/plugin/enrich/qa/common/src/main/java/org/elasticsearch/test/enrich/CommonEnrichRestTestCase.java → x-pack/plugin/enrich/src/javaRestTest/java/org/elasticsearch/test/enrich/CommonEnrichRestTestCase.java

@@ -11,10 +11,14 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -34,6 +38,27 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public abstract class CommonEnrichRestTestCase extends ESRestTestCase {
 
+    static LocalClusterSpecBuilder<ElasticsearchCluster> enrichCluster(String license, boolean isSecurityEnabled) {
+        return ElasticsearchCluster.local()
+            .module("analysis-common")
+            .module("ingest-common")
+            .module("mapper-extras")
+            .module("x-pack-enrich")
+            .module("x-pack-monitoring")
+            .module("x-pack-ilm")
+            .module("wildcard")
+            .setting("xpack.security.enabled", Boolean.toString(isSecurityEnabled))
+            .setting("xpack.license.self_generated.type", license)
+            // silence stats collector errors (we don't want to add all xpack modules here)
+            .setting("logger.org.elasticsearch.xpack.monitoring.collector", "fatal")
+            .setting("xpack.monitoring.collection.enabled", "true");
+    }
+
+    static Settings authRequestHeaderSetting(String user, String password) {
+        String token = basicAuthHeaderValue(user, new SecureString(password.toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
     private List<String> cleanupPipelines = new ArrayList<>();
 
     /**

x-pack/plugin/enrich/qa/rest-with-advanced-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichAdvancedSecurityIT.java → x-pack/plugin/enrich/src/javaRestTest/java/org/elasticsearch/test/enrich/EnrichAdvancedSecurityIT.java

@@ -4,13 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.enrich;
+package org.elasticsearch.test.enrich;
 
 import org.elasticsearch.client.Request;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.Map;
@@ -20,16 +20,29 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class EnrichAdvancedSecurityIT extends CommonEnrichRestTestCase {
 
+    public static final String ADMIN_USER = "test_admin";
+    public static final String ENRICH_USER = "test_enrich";
+    public static final String TEST_PASSWORD = "x-pack-test-password";
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = enrichCluster("trial", true).rolesFile(Resource.fromClasspath("advanced_roles.yml"))
+        .user(ADMIN_USER, TEST_PASSWORD, "superuser", true)
+        .user(ENRICH_USER, TEST_PASSWORD, "integ_test_role", false)
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     @Override
     protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("test_enrich", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return authRequestHeaderSetting(ENRICH_USER, TEST_PASSWORD);
     }
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return authRequestHeaderSetting(ADMIN_USER, TEST_PASSWORD);
     }
 
     public void testEnrichEnforcesDLS() throws IOException {

x-pack/plugin/enrich/src/javaRestTest/java/org/elasticsearch/test/enrich/EnrichIT.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.test.enrich;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.junit.ClassRule;
+
+public class EnrichIT extends CommonEnrichRestTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = enrichCluster("basic", false).build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityFailureIT.java → x-pack/plugin/enrich/src/javaRestTest/java/org/elasticsearch/test/enrich/EnrichSecurityFailureIT.java

@@ -4,30 +4,42 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.enrich;
+package org.elasticsearch.test.enrich;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import org.junit.ClassRule;
 
 public class EnrichSecurityFailureIT extends ESRestTestCase {
 
+    public static final String ADMIN_USER = "test_admin";
+    public static final String ENRICH_USER = "test_enrich_no_privs";
+    public static final String TEST_PASSWORD = "x-pack-test-password";
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = CommonEnrichRestTestCase.enrichCluster("basic", true)
+        .rolesFile(Resource.fromClasspath("roles.yml"))
+        .user(ADMIN_USER, TEST_PASSWORD, "superuser", true)
+        .user(ENRICH_USER, TEST_PASSWORD, "enrich_no_privs", false)
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     @Override
     protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("test_enrich_no_privs", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return CommonEnrichRestTestCase.authRequestHeaderSetting(ENRICH_USER, TEST_PASSWORD);
     }
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return CommonEnrichRestTestCase.authRequestHeaderSetting(ADMIN_USER, TEST_PASSWORD);
     }
 
     public void testFailure() throws Exception {

x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java → x-pack/plugin/enrich/src/javaRestTest/java/org/elasticsearch/test/enrich/EnrichSecurityIT.java

@@ -4,30 +4,43 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.enrich;
+package org.elasticsearch.test.enrich;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Strings;
-import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.junit.ClassRule;
 
 import static org.hamcrest.CoreMatchers.containsString;
 
 public class EnrichSecurityIT extends CommonEnrichRestTestCase {
 
+    public static final String ADMIN_USER = "test_admin";
+    public static final String ENRICH_USER = "test_enrich";
+    public static final String TEST_PASSWORD = "x-pack-test-password";
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = enrichCluster("basic", true).rolesFile(Resource.fromClasspath("roles.yml"))
+        .user(ADMIN_USER, TEST_PASSWORD, "superuser", true)
+        .user(ENRICH_USER, TEST_PASSWORD, "enrich_user,integ_test_role", false)
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     @Override
     protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("test_enrich", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return authRequestHeaderSetting(ENRICH_USER, TEST_PASSWORD);
     }
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        return authRequestHeaderSetting(ADMIN_USER, TEST_PASSWORD);
     }
 
     public void testInsufficientPermissionsOnNonExistentIndex() throws Exception {

x-pack/plugin/enrich/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/enrich/EnrichRestIT.java → x-pack/plugin/enrich/src/yamlRestTest/java/org/elasticsearch/xpack/enrich/EnrichRestIT.java

@@ -9,11 +9,26 @@ package org.elasticsearch.xpack.enrich;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class EnrichRestIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "basic")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     public EnrichRestIT(final ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -22,5 +37,4 @@ public class EnrichRestIT extends ESClientYamlSuiteTestCase {
     public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
-
 }