Remove cross project support in TestFixturesPlugin (#109077)

- One step closer to configuration cache support
- Crossproject support has been replaced by using testcontainer based fixtures

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureExtension.java

@@ -1,112 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.gradle.internal.testfixtures;
-
-import org.gradle.api.GradleException;
-import org.gradle.api.NamedDomainObjectContainer;
-import org.gradle.api.Project;
-
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-
-public class TestFixtureExtension {
-
-    private final Project project;
-    final NamedDomainObjectContainer<Project> fixtures;
-    final Map<String, String> serviceToProjectUseMap = new HashMap<>();
-
-    public TestFixtureExtension(Project project) {
-        this.project = project;
-        this.fixtures = project.container(Project.class);
-    }
-
-    public void useFixture() {
-        useFixture(this.project.getPath());
-    }
-
-    public void useFixture(String path) {
-        addFixtureProject(path);
-        serviceToProjectUseMap.put(path, this.project.getPath());
-    }
-
-    public void useFixture(String path, String serviceName) {
-        addFixtureProject(path);
-        String key = getServiceNameKey(path, serviceName);
-        serviceToProjectUseMap.put(key, this.project.getPath());
-
-        Optional<String> otherProject = this.findOtherProjectUsingService(key);
-        if (otherProject.isPresent()) {
-            throw new GradleException(
-                String.format(
-                    Locale.ROOT,
-                    "Projects %s and %s both claim the %s service defined in the docker-compose.yml of "
-                        + "%sThis is not supported because it breaks running in parallel. Configure dedicated "
-                        + "services for each project and use those instead.",
-                    otherProject.get(),
-                    this.project.getPath(),
-                    serviceName,
-                    path
-                )
-            );
-        }
-    }
-
-    private String getServiceNameKey(String fixtureProjectPath, String serviceName) {
-        return fixtureProjectPath + "::" + serviceName;
-    }
-
-    private Optional<String> findOtherProjectUsingService(String serviceName) {
-        return this.project.getRootProject()
-            .getAllprojects()
-            .stream()
-            .filter(p -> p.equals(this.project) == false)
-            .filter(p -> p.getExtensions().findByType(TestFixtureExtension.class) != null)
-            .map(project -> project.getExtensions().getByType(TestFixtureExtension.class))
-            .flatMap(ext -> ext.serviceToProjectUseMap.entrySet().stream())
-            .filter(entry -> entry.getKey().equals(serviceName))
-            .map(Map.Entry::getValue)
-            .findAny();
-    }
-
-    private void addFixtureProject(String path) {
-        Project fixtureProject = this.project.findProject(path);
-        if (fixtureProject == null) {
-            throw new IllegalArgumentException("Could not find test fixture " + fixtureProject);
-        }
-        if (fixtureProject.file(TestFixturesPlugin.DOCKER_COMPOSE_YML).exists() == false) {
-            throw new IllegalArgumentException(
-                "Project " + path + " is not a valid test fixture: missing " + TestFixturesPlugin.DOCKER_COMPOSE_YML
-            );
-        }
-        fixtures.add(fixtureProject);
-        // Check for exclusive access
-        Optional<String> otherProject = this.findOtherProjectUsingService(path);
-        if (otherProject.isPresent()) {
-            throw new GradleException(
-                String.format(
-                    Locale.ROOT,
-                    "Projects %s and %s both claim all services from %s. This is not supported because it"
-                        + " breaks running in parallel. Configure specific services in docker-compose.yml "
-                        + "for each and add the service name to `useFixture`",
-                    otherProject.get(),
-                    this.project.getPath(),
-                    path
-                )
-            );
-        }
-    }
-
-    boolean isServiceRequired(String serviceName, String fixtureProject) {
-        if (serviceToProjectUseMap.containsKey(fixtureProject)) {
-            return true;
-        }
-        return serviceToProjectUseMap.containsKey(getServiceNameKey(fixtureProject, serviceName));
-    }
-}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -70,7 +70,6 @@ public class TestFixturesPlugin implements Plugin<Project> {
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
 
         TaskContainer tasks = project.getTasks();
-        TestFixtureExtension extension = project.getExtensions().create("testFixtures", TestFixtureExtension.class, project);
         Provider<DockerComposeThrottle> dockerComposeThrottle = project.getGradle()
             .getSharedServices()
             .registerIfAbsent(DOCKER_COMPOSE_THROTTLE, DockerComposeThrottle.class, spec -> spec.getMaxParallelUsages().set(1));
@@ -84,73 +83,63 @@ public class TestFixturesPlugin implements Plugin<Project> {
         File testFixturesDir = project.file("testfixtures_shared");
         ext.set("testFixturesDir", testFixturesDir);
 
-        if (project.file(DOCKER_COMPOSE_YML).exists()) {
-            project.getPluginManager().apply(BasePlugin.class);
-            project.getPluginManager().apply(DockerComposePlugin.class);
-            TaskProvider<TestFixtureTask> preProcessFixture = project.getTasks().register("preProcessFixture", TestFixtureTask.class, t -> {
-                t.getFixturesDir().set(testFixturesDir);
-                t.doFirst(task -> {
-                    try {
-                        Files.createDirectories(testFixturesDir.toPath());
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                });
-            });
-            TaskProvider<Task> buildFixture = project.getTasks()
-                .register("buildFixture", t -> t.dependsOn(preProcessFixture, tasks.named("composeUp")));
-
-            TaskProvider<TestFixtureTask> postProcessFixture = project.getTasks()
-                .register("postProcessFixture", TestFixtureTask.class, task -> {
-                    task.getFixturesDir().set(testFixturesDir);
-                    task.dependsOn(buildFixture);
-                    configureServiceInfoForTask(
-                        task,
-                        project,
-                        false,
-                        (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
-                    );
-                });
-
-            maybeSkipTask(dockerSupport, preProcessFixture);
-            maybeSkipTask(dockerSupport, postProcessFixture);
-            maybeSkipTask(dockerSupport, buildFixture);
-
-            ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
-            composeExtension.setProjectName(project.getName());
-            composeExtension.getUseComposeFiles().addAll(Collections.singletonList(DOCKER_COMPOSE_YML));
-            composeExtension.getRemoveContainers().set(true);
-            composeExtension.getCaptureContainersOutput()
-                .set(EnumSet.of(LogLevel.INFO, LogLevel.DEBUG).contains(project.getGradle().getStartParameter().getLogLevel()));
-            composeExtension.getUseDockerComposeV2().set(false);
-            composeExtension.getExecutable().set(this.providerFactory.provider(() -> {
-                String composePath = dockerSupport.get().getDockerAvailability().dockerComposePath();
-                LOGGER.debug("Docker Compose path: {}", composePath);
-                return composePath != null ? composePath : "/usr/bin/docker-compose";
-            }));
-
-            tasks.named("composeUp").configure(t -> {
-                // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
-                if (BuildParams.isCi()) {
-                    t.usesService(dockerComposeThrottle);
+        if (project.file(DOCKER_COMPOSE_YML).exists() == false) {
+            // if only one fixture is used, that's this one, but without a compose file that's not a valid configuration
+            throw new IllegalStateException("No " + DOCKER_COMPOSE_YML + " found for " + project.getPath() + ".");
+        }
+        project.getPluginManager().apply(BasePlugin.class);
+        project.getPluginManager().apply(DockerComposePlugin.class);
+        TaskProvider<TestFixtureTask> preProcessFixture = project.getTasks().register("preProcessFixture", TestFixtureTask.class, t -> {
+            t.getFixturesDir().set(testFixturesDir);
+            t.doFirst(task -> {
+                try {
+                    Files.createDirectories(testFixturesDir.toPath());
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
                 }
-                t.mustRunAfter(preProcessFixture);
             });
-            tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
-            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
-        } else {
-            project.afterEvaluate(spec -> {
-                if (extension.fixtures.isEmpty()) {
-                    // if only one fixture is used, that's this one, but without a compose file that's not a valid configuration
-                    throw new IllegalStateException(
-                        "No " + DOCKER_COMPOSE_YML + " found for " + project.getPath() + " nor does it use other fixtures."
-                    );
-                }
+        });
+        TaskProvider<Task> buildFixture = project.getTasks()
+            .register("buildFixture", t -> t.dependsOn(preProcessFixture, tasks.named("composeUp")));
+
+        TaskProvider<TestFixtureTask> postProcessFixture = project.getTasks()
+            .register("postProcessFixture", TestFixtureTask.class, task -> {
+                task.getFixturesDir().set(testFixturesDir);
+                task.dependsOn(buildFixture);
+                configureServiceInfoForTask(
+                    task,
+                    project,
+                    false,
+                    (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
+                );
             });
-        }
 
-        extension.fixtures.matching(fixtureProject -> fixtureProject.equals(project) == false)
-            .all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath()));
+        maybeSkipTask(dockerSupport, preProcessFixture);
+        maybeSkipTask(dockerSupport, postProcessFixture);
+        maybeSkipTask(dockerSupport, buildFixture);
+
+        ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
+        composeExtension.setProjectName(project.getName());
+        composeExtension.getUseComposeFiles().addAll(Collections.singletonList(DOCKER_COMPOSE_YML));
+        composeExtension.getRemoveContainers().set(true);
+        composeExtension.getCaptureContainersOutput()
+            .set(EnumSet.of(LogLevel.INFO, LogLevel.DEBUG).contains(project.getGradle().getStartParameter().getLogLevel()));
+        composeExtension.getUseDockerComposeV2().set(false);
+        composeExtension.getExecutable().set(this.providerFactory.provider(() -> {
+            String composePath = dockerSupport.get().getDockerAvailability().dockerComposePath();
+            LOGGER.debug("Docker Compose path: {}", composePath);
+            return composePath != null ? composePath : "/usr/bin/docker-compose";
+        }));
+
+        tasks.named("composeUp").configure(t -> {
+            // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
+            if (BuildParams.isCi()) {
+                t.usesService(dockerComposeThrottle);
+            }
+            t.mustRunAfter(preProcessFixture);
+        });
+        tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
+        tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
 
         // Skip docker compose tasks if it is unavailable
         maybeSkipTasks(tasks, dockerSupport, Test.class);
@@ -161,17 +150,18 @@ public class TestFixturesPlugin implements Plugin<Project> {
         maybeSkipTasks(tasks, dockerSupport, ComposePull.class);
         maybeSkipTasks(tasks, dockerSupport, ComposeDown.class);
 
-        tasks.withType(Test.class).configureEach(task -> extension.fixtures.all(fixtureProject -> {
-            task.dependsOn(fixtureProject.getTasks().named("postProcessFixture"));
-            task.finalizedBy(fixtureProject.getTasks().named("composeDown"));
+        tasks.withType(Test.class).configureEach(testTask -> {
+            testTask.dependsOn(postProcessFixture);
+            testTask.finalizedBy(tasks.named("composeDown"));
             configureServiceInfoForTask(
-                task,
-                fixtureProject,
+                testTask,
+                project,
                 true,
-                (name, host) -> task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class).systemProperty(name, host)
+                (name, host) -> testTask.getExtensions()
+                    .getByType(SystemPropertyCommandLineArgumentProvider.class)
+                    .systemProperty(name, host)
             );
-        }));
-
+        });
     }
 
     private void maybeSkipTasks(TaskContainer tasks, Provider<DockerSupportService> dockerSupport, Class<? extends DefaultTask> taskClass) {
@@ -203,28 +193,20 @@ public class TestFixturesPlugin implements Plugin<Project> {
         task.doFirst(new Action<Task>() {
             @Override
             public void execute(Task theTask) {
-                TestFixtureExtension extension = theTask.getProject().getExtensions().getByType(TestFixtureExtension.class);
-
-                fixtureProject.getExtensions()
-                    .getByType(ComposeExtension.class)
-                    .getServicesInfos()
-                    .entrySet()
-                    .stream()
-                    .filter(entry -> enableFilter == false || extension.isServiceRequired(entry.getKey(), fixtureProject.getPath()))
-                    .forEach(entry -> {
-                        String service = entry.getKey();
-                        ServiceInfo infos = entry.getValue();
-                        infos.getTcpPorts().forEach((container, host) -> {
-                            String name = "test.fixtures." + service + ".tcp." + container;
-                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                            consumer.accept(name, host);
-                        });
-                        infos.getUdpPorts().forEach((container, host) -> {
-                            String name = "test.fixtures." + service + ".udp." + container;
-                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                            consumer.accept(name, host);
-                        });
+                fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos().entrySet().stream().forEach(entry -> {
+                    String service = entry.getKey();
+                    ServiceInfo infos = entry.getValue();
+                    infos.getTcpPorts().forEach((container, host) -> {
+                        String name = "test.fixtures." + service + ".tcp." + container;
+                        theTask.getLogger().info("port mapping property: {}={}", name, host);
+                        consumer.accept(name, host);
                     });
+                    infos.getUdpPorts().forEach((container, host) -> {
+                        String name = "test.fixtures." + service + ".udp." + container;
+                        theTask.getLogger().info("port mapping property: {}={}", name, host);
+                        consumer.accept(name, host);
+                    });
+                });
             }
         });
     }

distribution/docker/build.gradle

@@ -72,8 +72,6 @@ if (useDra == false) {
   }
 }
 
-testFixtures.useFixture()
-
 configurations {
   aarch64DockerSource {
     attributes {

qa/apm/build.gradle

@@ -16,8 +16,6 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 
-testFixtures.useFixture()
-
 dockerCompose {
   environment.put 'STACK_VERSION', BuildParams.snapshotBuild ? VersionProperties.elasticsearch : VersionProperties.elasticsearch + "-SNAPSHOT"
 }

qa/remote-clusters/build.gradle

@@ -15,8 +15,6 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 
-testFixtures.useFixture()
-
 tasks.register("copyNodeKeyMaterial", Sync) {
   from project(':x-pack:plugin:core')
     .files(