Convert remaining module projects to new test clusters framework (#125613)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -29,15 +29,7 @@ public abstract class RestrictedBuildApiService implements BuildService<Restrict
         ListMultimap<Class<?>, String> map = ArrayListMultimap.create(1, 200);
         map.put(LegacyRestTestBasePlugin.class, ":docs");
         map.put(LegacyRestTestBasePlugin.class, ":distribution:docker");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
         map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-nori");

modules/lang-expression/build.gradle

@@ -6,8 +6,8 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java

@@ -12,11 +12,16 @@ package org.elasticsearch.script.expression;
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-expression").build();
+
     public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -25,4 +30,9 @@ public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

modules/lang-mustache/build.gradle

@@ -6,9 +6,9 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
-apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java

@@ -11,7 +11,9 @@ package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 
@@ -19,6 +21,9 @@ import static org.hamcrest.CoreMatchers.containsString;
 
 public class SearchTemplateWithoutContentIT extends ESRestTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").build();
+
     public void testSearchTemplateMissingBody() throws IOException {
         ResponseException responseException = expectThrows(
             ResponseException.class,
@@ -36,4 +41,9 @@ public class SearchTemplateWithoutContentIT extends ESRestTestCase {
         assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
         assertThat(responseException.getMessage(), containsString("request body or source parameter is required"));
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java

@@ -12,11 +12,16 @@ package org.elasticsearch.script.mustache;
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").build();
+
     public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -25,4 +30,9 @@ public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

modules/rank-eval/build.gradle

@@ -6,8 +6,8 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -21,7 +21,6 @@ restResources {
   }
 }
 
-testClusters.configureEach {
-  // Modules who's integration is explicitly tested in integration tests
-  module ':modules:lang-mustache'
+dependencies {
+  clusterModules project(':modules:lang-mustache')
 }

modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java

@@ -12,10 +12,16 @@ package org.elasticsearch.index.rankeval;
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class RankEvalYamlIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").module("rank-eval").build();
+
     public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -24,4 +30,9 @@ public class RankEvalYamlIT extends ESClientYamlSuiteTestCase {
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

modules/transport-netty4/build.gradle

@@ -7,13 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+
+import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin
-import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
-apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.publish'
 
@@ -48,6 +49,8 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
 
   testImplementation project(':modules:rest-root')
+
+  clusterModules project(':modules:rest-root')
 }
 
 restResources {
@@ -77,28 +80,12 @@ TaskProvider<Test> pooledInternalClusterTest = tasks.register("pooledInternalClu
   setClasspath(internalTestSourceSet.getRuntimeClasspath())
 }
 
-TaskProvider<RestIntegTestTask> pooledJavaRestTest = tasks.register("pooledJavaRestTest", RestIntegTestTask) {
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet javaRestTestSourceSet = sourceSets.getByName(LegacyJavaRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(javaRestTestSourceSet.getRuntimeClasspath())
-
-
-  testClusters.pooledJavaRestTest {
-    systemProperty 'es.use_unpooled_allocator', 'false'
-  }
-}
-
 tasks.named("internalClusterTest").configure {
   systemProperty 'es.insecure_network_trace_enabled', 'true'
 }
 
-testClusters.configureEach {
-  module ':modules:rest-root'
-}
-
 tasks.named("check").configure {
-  dependsOn(pooledTest, pooledJavaRestTest, pooledInternalClusterTest)
+  dependsOn(pooledTest, pooledInternalClusterTest)
 }
 
 tasks.named("thirdPartyAudit").configure {

modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/AbstractNetty4IT.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.MutableSystemPropertyProvider;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.List;
+
+public abstract class AbstractNetty4IT extends ESRestTestCase {
+    private static final MutableSystemPropertyProvider clusterSettings = new MutableSystemPropertyProvider();
+    private final boolean usePooledAllocator;
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("transport-netty4")
+        .module("rest-root")
+        .systemProperties(clusterSettings)
+        .build();
+
+    public AbstractNetty4IT(@Name("pooled") boolean pooledAllocator) {
+        this.usePooledAllocator = pooledAllocator;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
+    @Before
+    public void maybeRestart() throws IOException {
+        // Restart the cluster to pick up the new setting if necessary
+        String current = clusterSettings.get(null).get("es.use_unpooled_allocator");
+        if (current == null || current.equals(Boolean.toString(usePooledAllocator))) {
+            clusterSettings.get(null).put("es.use_unpooled_allocator", Boolean.toString(usePooledAllocator == false));
+            cluster.restart(false);
+            closeClients();
+            initClient();
+        }
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.rest;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -17,7 +19,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.http.HttpTransportSettings;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 
 import java.io.IOException;
@@ -31,7 +32,11 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.matchesRegex;
 
-public class Netty4BadRequestIT extends ESRestTestCase {
+public class Netty4BadRequestIT extends AbstractNetty4IT {
+
+    public Netty4BadRequestIT(@Name("pooled") boolean pooledAllocator) {
+        super(pooledAllocator);
+    }
 
     public void testBadRequest() throws IOException {
         final Response response = client().performRequest(new Request("GET", "/_nodes/settings"));

modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java

@@ -9,11 +9,12 @@
 
 package org.elasticsearch.rest;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matcher;
 
@@ -28,7 +29,12 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.nullValue;
 
-public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
+public class Netty4HeadBodyIsEmptyIT extends AbstractNetty4IT {
+
+    public Netty4HeadBodyIsEmptyIT(@Name("pooled") boolean pooledAllocator) {
+        super(pooledAllocator);
+    }
+
     public void testHeadRoot() throws IOException {
         headTestCase("/", emptyMap(), greaterThan(0));
         headTestCase("/", singletonMap("pretty", ""), greaterThan(0));

modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java

@@ -14,8 +14,10 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.lucene.tests.util.TimeUnits;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 
@@ -23,6 +25,9 @@ import java.io.IOException;
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
 public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("transport-netty4").build();
+
     public Netty4ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -37,4 +42,9 @@ public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         assumeFalse("FIPS JVMs are configured to use the 'security4' transport rather than 'netty4'", inFipsJvm());
         super.test();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSystemPropertyProvider.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.LocalClusterSpec;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MutableSystemPropertyProvider implements SystemPropertyProvider {
+    private final Map<String, String> settings = new HashMap<>();
+
+    @Override
+    public Map<String, String> get(LocalClusterSpec.LocalNodeSpec nodeSpec) {
+        return settings;
+    }
+
+    public void put(String setting, String value) {
+        settings.put(setting, value);
+    }
+
+    public void remove(String setting) {
+        settings.remove(setting);
+    }
+
+    public void clear() {
+        settings.clear();
+    }
+}