Fixes for module projects in new tests clusters and auto security config (#92533)

Fix an issue where the build cannot resolve a module dependency for the
current module project. Also add partial support for security auto-
configuration in test clusters.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -18,6 +18,7 @@ import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
 import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin;
 import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
 import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
@@ -41,7 +42,8 @@ import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.util.PatternFilterable;
 
 import java.util.Collection;
-import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -89,19 +91,18 @@ public class RestTestBasePlugin implements Plugin<Project> {
         });
 
         // Create configures for module and plugin dependencies
-        Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true);
-        Configuration pluginsConfiguration = createPluginConfiguration(project, PLUGINS_CONFIGURATION, false);
-        Configuration extractedPluginsConfiguration = createPluginConfiguration(project, EXTRACTED_PLUGINS_CONFIGURATION, true);
+        Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true, false);
+        Configuration pluginsConfiguration = createPluginConfiguration(project, PLUGINS_CONFIGURATION, false, false);
+        Configuration extractedPluginsConfiguration = createPluginConfiguration(project, EXTRACTED_PLUGINS_CONFIGURATION, true, true);
         extractedPluginsConfiguration.extendsFrom(pluginsConfiguration);
         configureArtifactTransforms(project);
 
         // For plugin and module projects, register the current project plugin bundle as a dependency
         project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
             if (GradleUtils.isModuleProject(project.getPath())) {
-                project.getDependencies()
-                    .add(modulesConfiguration.getName(), project.getDependencies().project(Map.of("path", project.getPath())));
+                project.getDependencies().add(MODULES_CONFIGURATION, getExplodedBundleDependency(project, project.getPath()));
             } else {
-                project.getDependencies().add(pluginsConfiguration.getName(), project.files(project.getTasks().named("bundlePlugin")));
+                project.getDependencies().add(PLUGINS_CONFIGURATION, getBundleZipTaskDependency(project, project.getPath()));
             }
 
         });
@@ -158,15 +159,15 @@ public class RestTestBasePlugin implements Plugin<Project> {
     }
 
     private void registerConfigurationInputs(Task task, Configuration configuration) {
-        task.getInputs()
-            .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
-            .withPropertyName(configuration.getName() + "-classpath")
-            .withNormalizer(ClasspathNormalizer.class);
-
         task.getInputs()
             .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false)))
             .withPropertyName(configuration.getName() + "-files")
             .withPathSensitivity(PathSensitivity.RELATIVE);
+
+        task.getInputs()
+            .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
+            .withPropertyName(configuration.getName() + "-classpath")
+            .withNormalizer(ClasspathNormalizer.class);
     }
 
     private void registerDistributionInputs(Task task, ElasticsearchDistribution distribution) {
@@ -192,36 +193,64 @@ public class RestTestBasePlugin implements Plugin<Project> {
             .map(Project::getPath);
     }
 
-    private Configuration createPluginConfiguration(Project project, String name, boolean useExploded) {
+    private Configuration createPluginConfiguration(Project project, String name, boolean useExploded, boolean isExtended) {
         return project.getConfigurations().create(name, c -> {
             if (useExploded) {
                 c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE));
             } else {
                 c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE));
             }
-            c.withDependencies(dependencies -> {
-                // Add dependencies of any modules
-                Collection<Dependency> additionalDependencies = new HashSet<>();
-                for (Dependency dependency : dependencies) {
-                    if (dependency instanceof ProjectDependency projectDependency) {
-                        List<String> extendedPlugins = projectDependency.getDependencyProject()
-                            .getExtensions()
-                            .getByType(PluginPropertiesExtension.class)
-                            .getExtendedPlugins();
-
-                        for (String extendedPlugin : extendedPlugins) {
-                            findModulePath(project, extendedPlugin).ifPresent(
-                                modulePath -> additionalDependencies.add(project.getDependencies().project(Map.of("path", modulePath)))
-                            );
+            if (isExtended == false) {
+                c.withDependencies(dependencies -> {
+                    // Add dependencies of any modules
+                    Collection<Dependency> additionalDependencies = new LinkedHashSet<>();
+                    for (Iterator<Dependency> iterator = dependencies.iterator(); iterator.hasNext();) {
+                        Dependency dependency = iterator.next();
+                        if (dependency instanceof ProjectDependency projectDependency) {
+                            Project dependencyProject = projectDependency.getDependencyProject();
+                            List<String> extendedPlugins = dependencyProject.getExtensions()
+                                .getByType(PluginPropertiesExtension.class)
+                                .getExtendedPlugins();
+
+                            // Replace project dependency with explicit dependency on exploded configuration to workaround variant bug
+                            if (projectDependency.getTargetConfiguration() == null) {
+                                iterator.remove();
+                                additionalDependencies.add(
+                                    useExploded
+                                        ? getExplodedBundleDependency(project, dependencyProject.getPath())
+                                        : getBundleZipTaskDependency(project, dependencyProject.getPath())
+                                );
+                            }
+
+                            for (String extendedPlugin : extendedPlugins) {
+                                findModulePath(project, extendedPlugin).ifPresent(
+                                    modulePath -> additionalDependencies.add(
+                                        useExploded
+                                            ? getExplodedBundleDependency(project, modulePath)
+                                            : getBundleZipTaskDependency(project, modulePath)
+                                    )
+                                );
+                            }
                         }
                     }
-                }
 
-                dependencies.addAll(additionalDependencies);
-            });
+                    dependencies.addAll(additionalDependencies);
+                });
+            }
         });
     }
 
+    private Dependency getExplodedBundleDependency(Project project, String projectPath) {
+        return project.getDependencies()
+            .project(Map.of("path", projectPath, "configuration", BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG));
+    }
+
+    private Dependency getBundleZipTaskDependency(Project project, String projectPath) {
+        Project dependencyProject = project.findProject(projectPath);
+        return project.getDependencies()
+            .create(project.files(dependencyProject.getTasks().named(BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME)));
+    }
+
     private void configureArtifactTransforms(Project project) {
         project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
             transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);

modules/aggregations/build.gradle

@@ -1,4 +1,3 @@
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 /*
@@ -8,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -52,11 +51,7 @@ artifacts {
   restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
 }
 
-testClusters.configureEach {
-  module ':modules:lang-painless'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-}
-
 dependencies {
   compileOnly(project(':modules:lang-painless:spi'))
+  clusterModules(project(':modules:lang-painless'))
 }

modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java

@@ -10,10 +10,20 @@ package org.elasticsearch.aggregations;
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class AggregationsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("aggregations")
+        .module("lang-painless")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
     public AggregationsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -22,4 +32,9 @@ public class AggregationsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java

@@ -26,6 +26,7 @@ public class DefaultSettingsProvider implements SettingsProvider {
         settings.put("node.portsfile", "true");
         settings.put("http.port", "0");
         settings.put("transport.port", "0");
+        settings.put("network.host", "_local_");
 
         if (nodeSpec.getDistributionType() == DistributionType.INTEG_TEST) {
             settings.put("xpack.security.enabled", "false");

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java

@@ -341,7 +341,7 @@ public class LocalClusterFactory implements ClusterFactory<LocalClusterSpec, Loc
                                         + project
                                         + "':\n\n"
                                         + "dependencies {\n"
-                                        + "  clusterModules "
+                                        + "  clusterPlugins "
                                         + "project(':plugins:"
                                         + pluginName
                                         + "')"

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java

@@ -31,6 +31,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class LocalClusterHandle implements ClusterHandle {
     private static final Logger LOGGER = LogManager.getLogger(LocalClusterHandle.class);
@@ -124,11 +125,17 @@ public class LocalClusterHandle implements ClusterHandle {
         try {
             Retry.retryUntilTrue(CLUSTER_UP_TIMEOUT, Duration.ZERO, () -> {
                 Node node = nodes.get(0);
-                String scheme = node.getSpec().isSettingTrue("xpack.security.http.ssl.enabled") ? "https" : "http";
+                boolean securityEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.enabled", "true"));
+                boolean sslEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.http.ssl.enabled", "false"));
+                boolean securityAutoConfigured = isSecurityAutoConfigured(node);
+                String scheme = securityEnabled && (sslEnabled || securityAutoConfigured) ? "https" : "http";
                 WaitForHttpResource wait = new WaitForHttpResource(scheme, node.getHttpAddress(), nodes.size());
                 User credentials = node.getSpec().getUsers().get(0);
                 wait.setUsername(credentials.getUsername());
                 wait.setPassword(credentials.getPassword());
+                if (securityAutoConfigured) {
+                    wait.setCertificateAuthorities(node.getWorkingDir().resolve("config/certs/http_ca.crt").toFile());
+                }
                 return wait.wait(500);
             });
         } catch (TimeoutException e) {
@@ -138,6 +145,15 @@ public class LocalClusterHandle implements ClusterHandle {
         }
     }
 
+    private boolean isSecurityAutoConfigured(Node node) {
+        Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml");
+        try (Stream<String> lines = Files.lines(configFile)) {
+            return lines.anyMatch(l -> l.contains("BEGIN SECURITY AUTO CONFIGURATION"));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     private void writeUnicastHostsFile() {
         String transportUris = execute(() -> nodes.parallelStream().map(Node::getTransportEndpoint).collect(Collectors.joining("\n")));
         nodes.forEach(node -> {

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java

@@ -147,8 +147,8 @@ public class LocalClusterSpec implements ClusterSpec {
             );
         }
 
-        public boolean isSettingTrue(String setting) {
-            return Boolean.parseBoolean(resolveSettings().getOrDefault(setting, "false"));
+        public String getSetting(String setting, String defaultValue) {
+            return resolveSettings().getOrDefault(setting, defaultValue);
         }
 
         /**

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java

@@ -37,7 +37,7 @@ public class LocalDistributionResolver implements DistributionResolver {
                             + "':\n\n"
                             + "tasks.named('"
                             + taskName
-                            + "').configure {\n"
+                            + "') {\n"
                             + "  usesDefaultDistribution()\n"
                             + "}"
                     );