Fix deprecated unsave project outputs resolution (#59088)

- Fixes how libs in distribution are resolved
- Related to #57920
- Required minor rework on common repository setup to allow distribution projects
to resolve thirdparty artifacts
- Use Default configurations when resolving tools for distribution packaging

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -24,6 +24,7 @@ import groovy.transform.CompileStatic
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.ElasticsearchJavaPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.elasticsearch.gradle.RepositoriesSetupPlugin
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
@@ -59,9 +60,9 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RepositoriesSetupPlugin)
 
         project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
-        ElasticsearchJavaPlugin.configureRepositories(project)
         ElasticsearchJavaPlugin.configureTestTasks(project)
         ElasticsearchJavaPlugin.configureInputNormalization(project)
         ElasticsearchJavaPlugin.configureCompile(project)

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java

@@ -36,9 +36,6 @@ import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ModuleDependency;
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.ResolutionStrategy;
-import org.gradle.api.artifacts.dsl.RepositoryHandler;
-import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
-import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
 import org.gradle.api.execution.TaskActionListener;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.BasePlugin;
@@ -59,16 +56,10 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 import static org.elasticsearch.gradle.util.Util.toStringable;
@@ -83,11 +74,11 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         // apply global test task failure listener
         project.getRootProject().getPluginManager().apply(TestFailureReportingPlugin.class);
-
+        // common repositories setup
+        project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(JavaLibraryPlugin.class);
 
         configureConfigurations(project);
-        configureRepositories(project);
         configureCompile(project);
         configureInputNormalization(project);
         configureTestTasks(project);
@@ -149,77 +140,6 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
         disableTransitiveDeps.accept(JavaPlugin.TEST_IMPLEMENTATION_CONFIGURATION_NAME);
     }
 
-    private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
-
-    /**
-     * Adds repositories used by ES dependencies
-     */
-    public static void configureRepositories(Project project) {
-        // ensure all repositories use secure urls
-        // TODO: remove this with gradle 7.0, which no longer allows insecure urls
-        project.getRepositories().all(repository -> {
-            if (repository instanceof MavenArtifactRepository) {
-                final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
-                assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
-                for (URI uri : maven.getArtifactUrls()) {
-                    assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
-                }
-            } else if (repository instanceof IvyArtifactRepository) {
-                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
-                assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
-            }
-        });
-        RepositoryHandler repos = project.getRepositories();
-        if (System.getProperty("repos.mavenLocal") != null) {
-            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
-            // useful for development ie. bwc tests where we install stuff in the local repository
-            // such that we don't have to pass hardcoded files to gradle
-            repos.mavenLocal();
-        }
-        repos.jcenter();
-
-        String luceneVersion = VersionProperties.getLucene();
-        if (luceneVersion.contains("-snapshot")) {
-            // extract the revision number from the version with a regex matcher
-            Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
-            if (matcher.find() == false) {
-                throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
-            }
-            String revision = matcher.group(1);
-            MavenArtifactRepository luceneRepo = repos.maven(repo -> {
-                repo.setName("lucene-snapshots");
-                repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
-            });
-            repos.exclusiveContent(exclusiveRepo -> {
-                exclusiveRepo.filter(
-                    descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
-                );
-                exclusiveRepo.forRepositories(luceneRepo);
-            });
-        }
-    }
-
-    private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
-
-    private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
-        if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
-            String url;
-            try {
-                url = uri.toURL().toString();
-            } catch (MalformedURLException e) {
-                throw new IllegalStateException(e);
-            }
-            final String message = String.format(
-                Locale.ROOT,
-                "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
-                repositoryName,
-                projectPath,
-                url
-            );
-            throw new GradleException(message);
-        }
-    }
-
     /**
      * Adds compiler settings to the project
      */

buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.GradleException;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.dsl.RepositoryHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
+
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RepositoriesSetupPlugin implements Plugin<Project> {
+
+    private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
+    private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
+
+    @Override
+    public void apply(Project project) {
+        configureRepositories(project);
+    }
+
+    /**
+     * Adds repositories used by ES projects and dependencies
+     */
+    public static void configureRepositories(Project project) {
+        // ensure all repositories use secure urls
+        // TODO: remove this with gradle 7.0, which no longer allows insecure urls
+        project.getRepositories().all(repository -> {
+            if (repository instanceof MavenArtifactRepository) {
+                final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
+                assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
+                for (URI uri : maven.getArtifactUrls()) {
+                    assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
+                }
+            } else if (repository instanceof IvyArtifactRepository) {
+                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
+                assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
+            }
+        });
+        RepositoryHandler repos = project.getRepositories();
+        if (System.getProperty("repos.mavenLocal") != null) {
+            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
+            // useful for development ie. bwc tests where we install stuff in the local repository
+            // such that we don't have to pass hardcoded files to gradle
+            repos.mavenLocal();
+        }
+        repos.jcenter();
+
+        String luceneVersion = VersionProperties.getLucene();
+        if (luceneVersion.contains("-snapshot")) {
+            // extract the revision number from the version with a regex matcher
+            Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
+            if (matcher.find() == false) {
+                throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
+            }
+            String revision = matcher.group(1);
+            MavenArtifactRepository luceneRepo = repos.maven(repo -> {
+                repo.setName("lucene-snapshots");
+                repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
+            });
+            repos.exclusiveContent(exclusiveRepo -> {
+                exclusiveRepo.filter(
+                    descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
+                );
+                exclusiveRepo.forRepositories(luceneRepo);
+            });
+        }
+    }
+
+    private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
+        if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
+            String url;
+            try {
+                url = uri.toURL().toString();
+            } catch (MalformedURLException e) {
+                throw new IllegalStateException(e);
+            }
+            final String message = String.format(
+                Locale.ROOT,
+                "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
+                repositoryName,
+                projectPath,
+                url
+            );
+            throw new GradleException(message);
+        }
+    }
+
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.RepositoriesSetupPlugin

distribution/build.gradle

@@ -259,6 +259,7 @@ copyModule(processSystemdOutputs, project(':modules:systemd'))
 configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   apply plugin: 'elasticsearch.jdk-download'
+  apply plugin: 'elasticsearch.repositories'
 
   // Setup all required JDKs
   project.jdks {
@@ -278,6 +279,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
   /*****************************************************************************
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
+  configurations {
+    ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli'].each {
+      create(it) {
+        canBeConsumed = false
+        canBeResolved = true
+        attributes {
+          attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY))
+          attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME))
+          attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL))
+        }
+      }
+    }
+  }
+
+  dependencies {
+    libs project(':server')
+    libs project(':libs:elasticsearch-plugin-classloader')
+    libs project(':distribution:tools:java-version-checker')
+    libs project(':distribution:tools:launchers')
+
+    libsPluginCli project(':distribution:tools:plugin-cli')
+    libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
+    libsSecurityCli project(':x-pack:plugin:security:cli')
+  }
+
   project.ext {
 
     /*****************************************************************************
@@ -286,22 +312,16 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libFiles = { oss ->
       copySpec {
         // delay by using closures, since they have not yet been configured, so no jar task exists yet
-        from { project(':server').jar }
-        from { project(':server').configurations.runtimeClasspath }
-        from { project(':libs:elasticsearch-plugin-classloader').jar }
-        from { project(':distribution:tools:java-version-checker').jar }
-        from { project(':distribution:tools:launchers').jar }
+        from(configurations.libs)
         into('tools/plugin-cli') {
-          from { project(':distribution:tools:plugin-cli').jar }
-          from { project(':distribution:tools:plugin-cli').configurations.runtimeClasspath }
+          from(configurations.libsPluginCli)
         }
         into('tools/keystore-cli') {
-          from { project(':distribution:tools:keystore-cli').jar }
+          from(configurations.libsKeystoreCli)
         }
         if (oss == false) {
           into('tools/security-cli') {
-            from { project(':x-pack:plugin:security:cli').jar }
-            from { project(':x-pack:plugin:security:cli').configurations.runtimeClasspath }
+            from(configurations.libsSecurityCli)
           }
         }
       }
@@ -612,14 +632,14 @@ subprojects {
 }
 
 ['archives:windows-zip', 'archives:oss-windows-zip',
-  'archives:darwin-tar', 'archives:oss-darwin-tar',
-  'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
-  'archives:linux-tar', 'archives:oss-linux-tar',
-  'archives:integ-test-zip',
-  'packages:rpm', 'packages:deb',
-  'packages:aarch64-rpm', 'packages:aarch64-deb',
-  'packages:oss-rpm', 'packages:oss-deb',
-  'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
+ 'archives:darwin-tar', 'archives:oss-darwin-tar',
+ 'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
+ 'archives:linux-tar', 'archives:oss-linux-tar',
+ 'archives:integ-test-zip',
+ 'packages:rpm', 'packages:deb',
+ 'packages:aarch64-rpm', 'packages:aarch64-deb',
+ 'packages:oss-rpm', 'packages:oss-deb',
+ 'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
 ].forEach { subName ->
   Project subproject = project("${project.path}:${subName}")
   Configuration configuration = configurations.create(subproject.name)

distribution/tools/launchers/build.gradle

@@ -21,7 +21,7 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  api parent.project('java-version-checker')
+  compileOnly project(':distribution:tools:java-version-checker')
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"