Add DRA snapshot build (#91018)

* Add support for building against defined build-ids of dependent artifacts
* Introduce DraResolvePlugin and functional tests
* Declare initial dra snapshot ci job for elasticsearch

We run the DRA builds against ubuntu as other release related ci jobs have in the past. Also we only have libs2xmlutills available on our ubuntu based ci workers

.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml

@@ -0,0 +1,47 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
+    description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
+    node: "ubuntu-18.04 || ubuntu-20.04"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+            #!/usr/local/bin/runbld --redirect-stderr
+            ES_VERSION=$(cat build-tools-internal/version.properties \
+                  | grep elasticsearch \
+                  | sed "s/elasticsearch//g" \
+                  | sed "s/ //g" \
+                  | sed "s/=//")
+            BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats master)"
+            ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp master)"
+            set -euo pipefail
+            set +x
+            VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+            export VAULT_TOKEN
+            $WORKSPACE/.ci/scripts/run-gradle.sh assemble -Ddra.artifacts=true -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID}
+            unset VAULT_TOKEN
+            set -x
+            $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh
+            # Artifacts should be generated
+            docker run --rm \
+                   --name release-manager \
+                   -e VAULT_ADDR \
+                   -e VAULT_ROLE_ID \
+                   -e VAULT_SECRET_ID \
+                   --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+                   docker.elastic.co/infra/release-manager:latest \
+                   cli collect \
+                   --project elasticsearch \
+                   --branch "master" \
+                   --commit "$GIT_COMMIT" \
+                   --workflow "snapshot" \
+                   --version "$ES_VERSION" \
+                   --artifact-set main \
+                   --dependency https://artifacts-snapshot.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json \
+                   --dependency https://artifacts-snapshot.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json

.ci/scripts/resolve-dra-manifest.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+ARTIFACT="${ARTIFACT:-$1}"
+BRANCH="${BRANCH:-$2}"
+
+curl -sS https://artifacts-snapshot.elastic.co/$ARTIFACT/latest/$BRANCH.json \
+  | jq -r '.build_id'

build-tools-internal/build.gradle

@@ -51,6 +51,10 @@ gradlePlugin {
       id = 'elasticsearch.docs-test'
       implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
     }
+    draArtifacts {
+      id = 'elasticsearch.dra-artifacts'
+      implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin'
+    }
     globalBuildInfo {
       id = 'elasticsearch.global-build-info'
       implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy

@@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource {
 
     private TemporaryFolder temporaryFolder
 
-    LocalRepositoryFixture(){
+    LocalRepositoryFixture() {
         this.temporaryFolder = new TemporaryFolder()
     }
 
@@ -33,22 +33,27 @@ class LocalRepositoryFixture extends ExternalResource {
         temporaryFolder.after()
     }
 
-    void generateJar(String group, String module, String version, String... clazzNames){
+    void generateJar(String group, String module, String version, String... clazzNames) {
         def baseGroupFolderPath = group.replace('.', '/')
         def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version")
         targetFolder.mkdirs()
 
         def jarFile = new File(targetFolder, "${module}-${version}.jar")
-        clazzNames.each {clazzName ->
-            DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
-                    .name(clazzName)
-                    .make()
-            if(jarFile.exists()) {
-                dynamicType.inject(jarFile);
-            }else {
-                dynamicType.toJar(jarFile);
+        if (clazzNames.size() == 0) {
+            jarFile.write("blubb")
+        } else {
+            clazzNames.each { clazzName ->
+                DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
+                        .name(clazzName)
+                        .make()
+                if (jarFile.exists()) {
+                    dynamicType.inject(jarFile);
+                } else {
+                    dynamicType.toJar(jarFile);
+                }
             }
         }
+
     }
 
     void configureBuild(File buildFile) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.dra
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture
+import org.elasticsearch.gradle.fixtures.WiremockFixture
+import org.gradle.testkit.runner.TaskOutcome
+import org.junit.ClassRule
+import spock.lang.Shared
+
+class DraResolvePluginFuncTest extends AbstractGradleFuncTest {
+
+    @Shared
+    @ClassRule
+    public LocalRepositoryFixture repository = new LocalRepositoryFixture()
+
+    def setup() {
+        configurationCacheCompatible = false
+
+        buildFile << """
+        plugins {
+            id 'elasticsearch.dra-artifacts'
+        }
+        
+        repositories.all {
+            // for supporting http testing repos here
+            allowInsecureProtocol = true
+        }
+        """
+    }
+
+    def "provides flag indicating dra usage"() {
+        setup:
+        repository.generateJar("org.acme", "ml-cpp", "8.6.0-SNAPSHOT")
+        buildFile << """
+        if(useDra == false) {
+            repositories {
+              maven {
+                name = "local-test"
+                url = "${repository.getRepoDir().toURI()}"
+                metadataSources {
+                  artifact()
+                }
+              }
+            }  
+        }
+        """
+
+        buildFile << """
+        configurations {
+            someConfig
+        }
+        
+        dependencies {
+            someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT"
+        }
+        
+        tasks.register('resolveArtifacts') {
+            doLast {
+                configurations.someConfig.files.each { println it }
+            }
+        }
+        """
+
+        when:
+        def result = gradleRunner("resolveArtifacts").build()
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
+
+        when:
+        result = gradleRunner("resolveArtifacts", "-Ddra.artifacts=true").buildAndFail()
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.FAILED
+        result.output.contains("Cannot resolve external dependency org.acme:ml-cpp:8.6.0-SNAPSHOT because no repositories are defined.")
+    }
+
+    def "configures repositories to resolve #draKey like dra #artifactType artifacts"() {
+        setup:
+        repository.generateJar("some.group", "bar", "1.0.0")
+        repository.generateJar("some.group", "baz", "1.0.0-SNAPSHOT")
+        repository.configureBuild(buildFile)
+        buildFile << """
+        configurations {
+            someConfig
+        }
+        
+        dependencies {
+            someConfig "some.group:bar:1.0.0"
+            someConfig "some.group:baz:1.0.0-SNAPSHOT"
+            someConfig "org.acme:$draArtifact:$draVersion:deps@zip"
+        }
+        
+        tasks.register('resolveArtifacts') {
+            doLast {
+                configurations.someConfig.files.each { println it }
+            }
+        }
+        """
+
+        when:
+        def result = WiremockFixture.withWireMock(expectedRequest, "content".getBytes('UTF-8')) { server ->
+            gradleRunner("resolveArtifacts",
+                    '-Ddra.artifacts=true',
+                    "-Ddra.artifacts.dependency.${draKey}=$buildId",
+                    "-Ddra.artifacts.url.repo.${artifactType}.prefix=${server.baseUrl()}").build()
+        }
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
+
+        where:
+        artifactType | buildId          | draVersion       | draKey   | draArtifact  | expectedRequest
+        "snapshot"   | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "ml-cpp" | "ml-cpp"     | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}-deps.zip"
+        "release"    | '8.6.0-f633b1d7' | "8.6.0"          | "ml-cpp" | "ml-cpp"     | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}-deps.zip"
+        "snapshot"   | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "beats"  | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}-deps.zip"
+        "release"    | '8.6.0-f633b1d7' | "8.6.0"          | "beats"  | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}-deps.zip"
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.dra;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.initialization.layout.BuildLayout;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+import static java.util.Map.Entry;
+
+public class DraResolvePlugin implements Plugin<Project> {
+
+    public static final String USE_DRA_ARTIFACTS_FLAG = "dra.artifacts";
+    public static final String DRA_ARTIFACTS_DEPENDENCY_PREFIX = "dra.artifacts.dependency";
+    private final ProviderFactory providerFactory;
+    private BuildLayout buildLayout;
+
+    private final Provider<String> snapshotRepositoryPrefix;
+    private final Provider<String> releaseRepositoryPrefix;
+
+    @Inject
+    public DraResolvePlugin(ProviderFactory providerFactory, BuildLayout buildLayout) {
+        this.providerFactory = providerFactory;
+        this.buildLayout = buildLayout;
+        this.snapshotRepositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.snapshot.prefix");
+        this.releaseRepositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.release.prefix");
+    }
+
+    @Override
+    public void apply(Project project) {
+        boolean useDra = providerFactory.systemProperty(USE_DRA_ARTIFACTS_FLAG).map(Boolean::parseBoolean).getOrElse(false);
+        project.getExtensions().getExtraProperties().set("useDra", useDra);
+        if (useDra) {
+            resolveBuildIdProperties().get().forEach((key, buildId) -> {
+                configureDraRepository(
+                    project,
+                    "dra-snapshot-artifacts-" + key,
+                    key,
+                    buildId,
+                    snapshotRepositoryPrefix.orElse("https://artifacts-snapshot.elastic.co/"),
+                    ".*SNAPSHOT"
+                );
+                configureDraRepository(
+                    project,
+                    "dra-release-artifacts-" + key,
+                    key,
+                    buildId,
+                    releaseRepositoryPrefix.orElse("https://artifacts.elastic.co/"),
+                    "^(.(?!SNAPSHOT))*$"
+                );
+            });
+        }
+    }
+
+    private void configureDraRepository(
+        Project project,
+        String repositoryName,
+        String draKey,
+        String buildId,
+        Provider<String> repoPrefix,
+        String includeVersionRegex
+    ) {
+        project.getRepositories().ivy(repo -> {
+            repo.setName(repositoryName);
+            repo.setUrl(repoPrefix.get());
+            repo.patternLayout(patternLayout -> {
+                patternLayout.artifact(
+                    String.format("/%s/%s/downloads/%s/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
+                );
+                patternLayout.artifact(
+                    String.format("/%s/%s/downloads/%s/[module]/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
+                );
+            });
+            repo.metadataSources(metadataSources -> metadataSources.artifact());
+            repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex));
+        });
+    }
+
+    private Provider<Map<String, String>> resolveBuildIdProperties() {
+        return providerFactory.systemPropertiesPrefixedBy(DRA_ARTIFACTS_DEPENDENCY_PREFIX)
+            .map(
+                stringStringMap -> stringStringMap.entrySet()
+                    .stream()
+                    .collect(
+                        Collectors.toMap(entry -> entry.getKey().substring(DRA_ARTIFACTS_DEPENDENCY_PREFIX.length() + 1), Entry::getValue)
+                    )
+            );
+    }
+}

distribution/docker/build.gradle

@@ -8,6 +8,7 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
 import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
+import org.elasticsearch.gradle.internal.dra.DraResolvePlugin
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
@@ -17,9 +18,10 @@ import java.time.temporal.ChronoUnit
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
+apply plugin: 'elasticsearch.dra-artifacts'
 
 String buildId = providers.systemProperty('build.id').getOrNull()
-boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false
+boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false
 
 repositories {
   // Define a repository that allows Gradle to fetch a resource from GitHub. This
@@ -35,29 +37,33 @@ repositories {
     metadataSources { artifact() }
     content { includeGroup 'krallin' }
   }
+}
 
-  // Cloud builds bundle some beats
-  ivy {
-    name = 'beats'
-    if (useLocalArtifacts) {
-      url "file://${buildDir}/artifacts/"
-      patternLayout {
-        artifact '/[organisation]/[module]-[revision]-[classifier].[ext]'
-      }
-    } else {
-      url "https://artifacts-snapshot.elastic.co/"
-      patternLayout {
-        if (VersionProperties.isElasticsearchSnapshot()) {
-          artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
-        } else {
-          // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
-          // Release builds are always done with a local repo.
-          artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+if(useDra == false) {
+  repositories {
+    // Cloud builds bundle some beats
+    ivy {
+      name = 'beats'
+      if (useLocalArtifacts) {
+        url "file://${buildDir}/artifacts/"
+        patternLayout {
+          artifact '/[organisation]/[module]-[revision]-[classifier].[ext]'
+        }
+      } else {
+        url "https://artifacts-snapshot.elastic.co/"
+        patternLayout {
+          if (VersionProperties.isElasticsearchSnapshot()) {
+            artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
+          } else {
+            // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
+            // Release builds are always done with a local repo.
+            artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+          }
         }
       }
+      metadataSources { artifact() }
+      content { includeGroup 'beats' }
     }
-    metadataSources { artifact() }
-    content { includeGroup 'beats' }
   }
 }
 

x-pack/plugin/ml/build.gradle

@@ -1,8 +1,10 @@
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.dra.DraResolvePlugin
 
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.dra-artifacts'
 
 esplugin {
   name 'x-pack-ml'
@@ -13,39 +15,41 @@ esplugin {
 }
 
 def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull
-
-repositories {
-  exclusiveContent {
-    filter {
-      includeGroup 'org.elasticsearch.ml'
-    }
-    forRepository {
-      ivy {
-        name "ml-cpp"
-        metadataSources {
-          // no repository metadata, look directly for the artifact
-          artifact()
-        }
-        if (localRepo) {
-          url localRepo
-          patternLayout {
-            artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
+if (useDra == false) {
+  repositories {
+    exclusiveContent {
+      filter {
+        includeGroup 'org.elasticsearch.ml'
+      }
+      forRepository {
+        ivy {
+          name "ml-cpp"
+          metadataSources {
+            // no repository metadata, look directly for the artifact
+            artifact()
           }
-        } else {
-          url "https://artifacts-snapshot.elastic.co/"
-          patternLayout {
-            if (VersionProperties.isElasticsearchSnapshot()) {
-              artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision]-[classifier].[ext]'
-            } else {
-              // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
-              // Release builds are always done with a local repo.
-              artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+          if (localRepo) {
+            url localRepo
+            patternLayout {
+              artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
+            }
+          } else {
+            url "https://artifacts-snapshot.elastic.co/"
+            patternLayout {
+              if (VersionProperties.isElasticsearchSnapshot()) {
+                artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision]-[classifier].[ext]'
+              } else {
+                // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
+                // Release builds are always done with a local repo.
+                artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
+              }
             }
           }
         }
       }
     }
   }
+
 }
 
 configurations {
@@ -126,4 +130,4 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
 
-addQaCheckDependencies(project)
+addQaCheckDependencies(project)