Add elasticsearch distribution plugin (#43247)

Several types of distributions are built and tested in elasticsearch,
ranging from the current version, to building or downloading snapshot or
released versions. Currently tests relying on these have to contain
logic deciding where and how to pull down these distributions.

This commit adds an distributiond download plugin for each project to
manage which versions and variants the project needs. It abstracts away
all need for knowing where a particular version comes from, like a local
distribution or bwc project, or pulling from the elastic download
service. This will be used in a followup PR by the testclusters and
vagrant tests.

buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java

@@ -109,7 +109,7 @@ public class BwcVersions {
     }
 
     protected BwcVersions(List<String> versionLines, Version currentVersionProperty) {
-        SortedSet<Version> allVersions = versionLines.stream()
+        this(versionLines.stream()
             .map(LINE_PATTERN::matcher)
             .filter(Matcher::matches)
             .map(match -> new Version(
@@ -117,8 +117,11 @@ public class BwcVersions {
                 Integer.parseInt(match.group(2)),
                 Integer.parseInt(match.group(3))
             ))
-            .collect(Collectors.toCollection(TreeSet::new));
+            .collect(Collectors.toCollection(TreeSet::new)), currentVersionProperty);
+    }
 
+    // for testkit tests, until BwcVersions is extracted into an extension
+    public BwcVersions(SortedSet<Version> allVersions, Version currentVersionProperty) {
         if (allVersions.isEmpty()) {
             throw new IllegalArgumentException("Could not parse any versions");
         }

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -0,0 +1,273 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.gradle.api.GradleException;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.UnknownTaskException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.credentials.HttpHeaderCredentials;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.authentication.http.HttpHeaderAuthentication;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.function.Supplier;
+
+/**
+ * A plugin to manage getting and extracting distributions of Elasticsearch.
+ *
+ * The source of the distribution could be from a local snapshot, a locally built
+ * bwc snapshot, or the Elastic downloads service.
+ */
+public class DistributionDownloadPlugin implements Plugin<Project> {
+
+    private static final String FAKE_GROUP = "elasticsearch-distribution";
+    private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
+
+    private BwcVersions bwcVersions;
+    private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
+
+    @Override
+    public void apply(Project project) {
+        distributionsContainer = project.container(ElasticsearchDistribution.class, name -> new ElasticsearchDistribution(name, project));
+        project.getExtensions().add("elasticsearch_distributions", distributionsContainer);
+
+        setupDownloadServiceRepo(project);
+
+        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
+        this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
+        // TODO: setup snapshot dependency instead of pointing to bwc distribution projects for external projects
+
+        project.afterEvaluate(this::setupDistributions);
+    }
+
+    // pkg private for tests
+    void setupDistributions(Project project) {
+        for (ElasticsearchDistribution distribution : distributionsContainer) {
+            distribution.finalizeValues();
+
+            DependencyHandler dependencies = project.getDependencies();
+            // for the distribution as a file, just depend on the artifact directly
+            dependencies.add(distribution.configuration.getName(), dependencyNotation(project, distribution));
+
+            // no extraction allowed for rpm or deb
+            if (distribution.getType() != Type.RPM && distribution.getType() != Type.DEB) {
+                // for the distribution extracted, add a root level task that does the extraction, and depend on that
+                // extracted configuration as an artifact consisting of the extracted distribution directory
+                dependencies.add(distribution.getExtracted().configuration.getName(),
+                    projectDependency(project, ":", configName("extracted_elasticsearch", distribution)));
+                // ensure a root level download task exists
+                setupRootDownload(project.getRootProject(), distribution);
+            }
+        }
+    }
+
+    private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) {
+        String extractTaskName = extractTaskName(distribution);
+        // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
+        try {
+            rootProject.getTasks().named(extractTaskName);
+            // already setup this version
+            return;
+        } catch (UnknownTaskException e) {
+            // fall through: register the task
+        }
+        setupDownloadServiceRepo(rootProject);
+
+        final ConfigurationContainer configurations = rootProject.getConfigurations();
+        String downloadConfigName = configName("elasticsearch", distribution);
+        String extractedConfigName = "extracted_" + downloadConfigName;
+        final Configuration downloadConfig = configurations.create(downloadConfigName);
+        configurations.create(extractedConfigName);
+        Object distroDep = dependencyNotation(rootProject, distribution);
+        rootProject.getDependencies().add(downloadConfigName, distroDep);
+
+        // add task for extraction, delaying resolving config until runtime
+        if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
+            Supplier<File> archiveGetter = downloadConfig::getSingleFile;
+            String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString();
+            TaskProvider<Copy> extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
+                copyTask.dependsOn(downloadConfig);
+                copyTask.doFirst(t -> rootProject.delete(extractDir));
+                copyTask.into(extractDir);
+                copyTask.from((Callable<FileTree>)() -> {
+                    File archiveFile = archiveGetter.get();
+                    String archivePath = archiveFile.toString();
+                    if (archivePath.endsWith(".zip")) {
+                        return rootProject.zipTree(archiveFile);
+                    } else if (archivePath.endsWith(".tar.gz")) {
+                        return rootProject.tarTree(rootProject.getResources().gzip(archiveFile));
+                    }
+                    throw new IllegalStateException("unexpected file extension on [" + archivePath + "]");
+                });
+            });
+            rootProject.getArtifacts().add(extractedConfigName,
+                rootProject.getLayout().getProjectDirectory().dir(extractDir),
+                artifact -> artifact.builtBy(extractTask));
+        }
+    }
+
+    private static void setupDownloadServiceRepo(Project project) {
+        if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
+            return;
+        }
+        project.getRepositories().ivy(ivyRepo -> {
+            ivyRepo.setName(DOWNLOAD_REPO_NAME);
+            ivyRepo.setUrl("https://artifacts.elastic.co");
+            ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+            // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
+            ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
+                creds.setName("X-Elastic-No-KPI");
+                creds.setValue("1");
+            });
+            ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
+            ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
+            ivyRepo.content(content -> content.includeGroup(FAKE_GROUP));
+        });
+        project.getRepositories().all(repo -> {
+            if (repo.getName().equals(DOWNLOAD_REPO_NAME) == false) {
+                // all other repos should ignore the special group name
+                repo.content(content -> content.excludeGroup(FAKE_GROUP));
+            }
+        });
+        // TODO: need maven repo just for integ-test-zip, but only in external cases
+    }
+
+    /**
+     * Returns a dependency object representing the given distribution.
+     *
+     * The returned object is suitable to be passed to {@link DependencyHandler}.
+     * The concrete type of the object will either be a project {@link Dependency} or
+     * a set of maven coordinates as a {@link String}. Project dependencies point to
+     * a project in the Elasticsearch repo either under `:distribution:bwc`,
+     * `:distribution:archives` or :distribution:packages`. Maven coordinates point to
+     * either the integ-test-zip coordinates on maven central, or a set of artificial
+     * coordinates that resolve to the Elastic download service through an ivy repository.
+     */
+    private Object dependencyNotation(Project project, ElasticsearchDistribution distribution) {
+
+        if (Version.fromString(VersionProperties.getElasticsearch()).equals(distribution.getVersion())) {
+            return projectDependency(project, distributionProjectPath(distribution), "default");
+            // TODO: snapshot dep when not in ES repo
+        }
+        BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(distribution.getVersion());
+        if (unreleasedInfo != null) {
+            assert distribution.getBundledJdk();
+            return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+        }
+
+        if (distribution.getType() == Type.INTEG_TEST_ZIP) {
+            return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion();
+        }
+
+        String extension = distribution.getType().toString();
+        String classifier = "x86_64";
+        if (distribution.getType() == Type.ARCHIVE) {
+            extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
+            classifier = distribution.getPlatform() + "-" + classifier;
+        }
+        return FAKE_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
+            + distribution.getVersion() + ":" + classifier + "@" + extension;
+    }
+
+    private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
+
+        if (project.findProject(projectPath) == null) {
+            throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
+        }
+        Map<String, Object> depConfig = new HashMap<>();
+        depConfig.put("path", projectPath);
+        depConfig.put("configuration", projectConfig);
+        return project.getDependencies().project(depConfig);
+    }
+
+    private static String distributionProjectPath(ElasticsearchDistribution distribution) {
+        String projectPath = ":distribution";
+        if (distribution.getType() == Type.INTEG_TEST_ZIP) {
+            projectPath += ":archives:integ-test-zip";
+        } else {
+            projectPath += distribution.getType() == Type.ARCHIVE ? ":archives:" : ":packages:";
+            projectPath += distributionProjectName(distribution);
+        }
+        return projectPath;
+    }
+
+    private static String distributionProjectName(ElasticsearchDistribution distribution) {
+        String projectName = "";
+        if (distribution.getFlavor() == Flavor.OSS) {
+            projectName += "oss-";
+        }
+        if (distribution.getBundledJdk() == false) {
+            projectName += "no-jdk-";
+        }
+        if (distribution.getType() == Type.ARCHIVE) {
+            Platform platform = distribution.getPlatform();
+            projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar");
+        } else {
+            projectName += distribution.getType();
+        }
+        return projectName;
+    }
+
+    private static String configName(String prefix, ElasticsearchDistribution distribution) {
+        return prefix + "_" + distribution.getVersion() + "_" + distribution.getType() + "_" +
+            (distribution.getPlatform() == null ? "" : distribution.getPlatform() + "_")
+            + distribution.getFlavor() + (distribution.getBundledJdk() ? "" : "_nojdk");
+    }
+
+    private static String capitalize(String s) {
+        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
+    }
+
+    private static String extractTaskName(ElasticsearchDistribution distribution) {
+        String taskName = "extractElasticsearch";
+        if (distribution.getType() != Type.INTEG_TEST_ZIP) {
+            if (distribution.getFlavor() == Flavor.OSS) {
+                taskName += "Oss";
+            }
+            if (distribution.getBundledJdk() == false) {
+                taskName += "NoJdk";
+            }
+        }
+        if (distribution.getType() == Type.ARCHIVE) {
+            taskName += capitalize(distribution.getPlatform().toString());
+        } else if (distribution.getType() != Type.INTEG_TEST_ZIP) {
+            taskName += capitalize(distribution.getType().toString());
+        }
+        taskName += distribution.getVersion();
+        return taskName;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -0,0 +1,229 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Buildable;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.TaskDependency;
+
+import java.io.File;
+import java.util.Iterator;
+import java.util.Locale;
+
+public class ElasticsearchDistribution implements Buildable {
+
+    public enum Platform {
+        LINUX,
+        WINDOWS,
+        DARWIN;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public enum Type {
+        INTEG_TEST_ZIP,
+        ARCHIVE,
+        RPM,
+        DEB;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public enum Flavor {
+        DEFAULT,
+        OSS;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    // package private to tests can use
+    static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
+        .onLinux(() -> Platform.LINUX)
+        .onWindows(() -> Platform.WINDOWS)
+        .onMac(() -> Platform.DARWIN)
+        .supply();
+
+    public static final class Extracted implements Buildable, Iterable<File> {
+
+        // pkg private so plugin can configure
+        final Configuration configuration;
+
+        private Extracted(Configuration configuration) {
+            this.configuration = configuration;
+        }
+
+        @Override
+        public Iterator<File> iterator() {
+            return configuration.iterator();
+        }
+
+        @Override
+        public TaskDependency getBuildDependencies() {
+            return configuration.getBuildDependencies();
+        }
+
+        @Override
+        public String toString() {
+            return configuration.getSingleFile().toString();
+        }
+    }
+
+    private final String name;
+    // pkg private so plugin can configure
+    final Configuration configuration;
+    private final Extracted extracted;
+
+    private final Property<Version> version;
+    private final Property<Type> type;
+    private final Property<Platform> platform;
+    private final Property<Flavor> flavor;
+    private final Property<Boolean> bundledJdk;
+
+    ElasticsearchDistribution(String name, Project project) {
+        this.name = name;
+        this.configuration = project.getConfigurations().create("es_distro_file_" + name);
+        this.version = project.getObjects().property(Version.class);
+        this.version.convention(Version.fromString(VersionProperties.getElasticsearch()));
+        this.type = project.getObjects().property(Type.class);
+        this.type.convention(Type.ARCHIVE);
+        this.platform = project.getObjects().property(Platform.class);
+        this.flavor = project.getObjects().property(Flavor.class);
+        this.bundledJdk = project.getObjects().property(Boolean.class);
+        this.extracted = new Extracted(project.getConfigurations().create("es_distro_extracted_" + name));
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Version getVersion() {
+        return version.get();
+    }
+
+    public void setVersion(String version) {
+        this.version.set(Version.fromString(version));
+    }
+
+    public Platform getPlatform() {
+        return platform.getOrNull();
+    }
+
+    public void setPlatform(Platform platform) {
+        this.platform.set(platform);
+    }
+
+    public Type getType() {
+        return type.get();
+    }
+
+    public void setType(Type type) {
+        this.type.set(type);
+    }
+
+    public Flavor getFlavor() {
+        return flavor.getOrNull();
+    }
+
+    public void setFlavor(Flavor flavor) {
+        this.flavor.set(flavor);
+    }
+
+    public boolean getBundledJdk() {
+        return bundledJdk.getOrElse(true);
+    }
+
+    public void setBundledJdk(boolean bundledJdk) {
+        this.bundledJdk.set(bundledJdk);
+    }
+
+    @Override
+    public String toString() {
+        return configuration.getSingleFile().toString();
+    }
+
+    public Extracted getExtracted() {
+        if (getType() == Type.RPM || getType() == Type.DEB) {
+            throw new UnsupportedOperationException("distribution type [" + getType() + "] for " +
+                "elasticsearch distribution [" + name + "] cannot be extracted");
+        }
+        return extracted;
+    }
+
+    @Override
+    public TaskDependency getBuildDependencies() {
+        return configuration.getBuildDependencies();
+    }
+
+    // internal, make this distribution's configuration unmodifiable
+    void finalizeValues() {
+
+        if (getType() == Type.INTEG_TEST_ZIP) {
+            if (platform.isPresent()) {
+                throw new IllegalArgumentException(
+                    "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            if (flavor.isPresent()) {
+                throw new IllegalArgumentException(
+                    "flavor not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            if (bundledJdk.isPresent()) {
+                throw new IllegalArgumentException(
+                    "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            return;
+        }
+
+        if (getType() == Type.ARCHIVE) {
+            // defaults for archive, set here instead of via convention so integ-test-zip can verify they are not set
+            if (platform.isPresent() == false) {
+                platform.set(CURRENT_PLATFORM);
+            }
+        } else { // rpm or deb
+            if (platform.isPresent()) {
+                throw new IllegalArgumentException("platform not allowed for elasticsearch distribution ["
+                    + name + "] of type [" + getType() + "]");
+            }
+        }
+
+        if (flavor.isPresent() == false) {
+            flavor.set(Flavor.DEFAULT);
+        }
+        if (bundledJdk.isPresent() == false) {
+            bundledJdk.set(true);
+        }
+
+        version.finalizeValue();
+        platform.finalizeValue();
+        type.finalizeValue();
+        flavor.finalizeValue();
+        bundledJdk.finalizeValue();
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.DistributionDownloadPlugin

buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.head;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+
+public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
+    // TODO: check reuse of root task across projects MOVE TO UNIT TEST
+    // TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
+
+    public void testCurrent() throws Exception {
+        String projectName = ":distribution:archives:linux-tar";
+        assertExtractedDistro(VersionProperties.getElasticsearch(), "archive", "linux", null, null,
+            "tests.local_distro.config", "default",
+            "tests.local_distro.project", projectName);
+    }
+
+    public void testBwc() throws Exception {
+        assertExtractedDistro("1.1.0", "archive", "linux", null, null,
+            "tests.local_distro.config", "linux-tar",
+            "tests.local_distro.project", ":distribution:bwc:minor",
+            "tests.current_version", "2.0.0");
+    }
+
+    public void testReleased() throws Exception {
+        WireMockServer wireMock = new WireMockServer(0);
+        try {
+            final byte[] filebytes;
+            try (InputStream stream =
+                     Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
+                filebytes = stream.readAllBytes();
+            }
+            String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip";
+            wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
+            wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
+            wireMock.start();
+
+            assertExtractedDistro("1.0.0", "archive", "windows", null, null,
+                "tests.download_service", wireMock.baseUrl());
+        } catch (Exception e) {
+            // for debugging
+            System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());
+            throw e;
+        } finally {
+            wireMock.stop();
+        }
+    }
+
+    private void assertFileDistro(String version, String type, String platform, String flavor, Boolean bundledJdk,
+                                  String... sysProps) throws IOException {
+        List<String> finalSysProps = new ArrayList<>();
+        addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
+        finalSysProps.addAll(Arrays.asList(sysProps));
+        runBuild(":subproj:assertDistroFile", finalSysProps.toArray(new String[0]));
+    }
+
+    private void assertExtractedDistro(String version, String type, String platform, String flavor, Boolean bundledJdk,
+                                       String... sysProps) throws IOException {
+        List<String> finalSysProps = new ArrayList<>();
+        addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
+        finalSysProps.addAll(Arrays.asList(sysProps));
+        runBuild(":subproj:assertDistroExtracted", finalSysProps.toArray(new String[0]));
+    }
+
+    private BuildResult runBuild(String taskname, String... sysProps) throws IOException {
+        assert sysProps.length % 2 == 0;
+        List<String> args = new ArrayList<>();
+        args.add(taskname);
+        args.add("-Dlocal.repo.path=" + getLocalTestRepoPath());
+        for (int i = 0; i < sysProps.length; i += 2) {
+            args.add("-D" + sysProps[i] + "=" + sysProps[i + 1]);
+        }
+        args.add("-i");
+        GradleRunner runner = getGradleRunner("distribution-download").withArguments(args);
+
+        BuildResult result = runner.build();
+        System.out.println(result.getOutput());
+        return result;
+    }
+
+    private void addDistroSysProps(List<String> sysProps, String version, String type, String platform, String flavor, Boolean bundledJdk) {
+        if (version != null) {
+            sysProps.add("tests.distro.version");
+            sysProps.add(version);
+        }
+        if (type != null) {
+            sysProps.add("tests.distro.type");
+            sysProps.add(type);
+        }
+        if (platform != null) {
+            sysProps.add("tests.distro.platform");
+            sysProps.add(platform);
+        }
+        if (flavor != null) {
+            sysProps.add("tests.distro.flavor");
+            sysProps.add(flavor);
+        }
+        if (bundledJdk != null) {
+            sysProps.add("tests.distro.bundledJdk");
+            sysProps.add(bundledJdk.toString());
+        }
+    }
+}

buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java

@@ -0,0 +1,260 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Project;
+import org.gradle.testfixtures.ProjectBuilder;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.TreeSet;
+
+import static org.hamcrest.core.StringContains.containsString;
+
+public class DistributionDownloadPluginTests extends GradleUnitTestCase {
+    private static Project rootProject;
+    private static Project archivesProject;
+    private static Project packagesProject;
+    private static Project bwcProject;
+
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("2.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("1.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1");
+    private static final BwcVersions BWC_MINOR =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_STAGED =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_BUGFIX =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_MAINTENANCE =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION)), BWC_MINOR_VERSION);
+
+    public void testVersionDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", null, Type.ARCHIVE, Platform.LINUX, Flavor.OSS, true);
+        assertEquals(distro.getVersion(), Version.fromString(VersionProperties.getElasticsearch()));
+    }
+
+    public void testBadVersionFormat() {
+        assertDistroError(createProject(null), "testdistro", "badversion", Type.ARCHIVE, Platform.LINUX, Flavor.OSS, true,
+            "Invalid version format: 'badversion'");
+    }
+
+    public void testTypeDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", null, Platform.LINUX, Flavor.OSS, true);
+        assertEquals(distro.getType(), Type.ARCHIVE);
+    }
+
+    public void testPlatformDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, null, Flavor.OSS, true);
+        assertEquals(distro.getPlatform(), ElasticsearchDistribution.CURRENT_PLATFORM);
+    }
+
+    public void testPlatformForIntegTest() {
+        assertDistroError(createProject(null), "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, Platform.LINUX, null, null,
+            "platform not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testFlavorDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, null, true);
+        assertEquals(distro.getFlavor(), Flavor.DEFAULT);
+    }
+
+    public void testFlavorForIntegTest() {
+        assertDistroError(createProject(null),
+            "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, null, Flavor.OSS, null,
+            "flavor not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testBundledJdkDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, null, true);
+        assertTrue(distro.getBundledJdk());
+    }
+
+    public void testBundledJdkForIntegTest() {
+        assertDistroError(createProject(null), "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, null, null, true,
+            "bundledJdk not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testCurrentVersionIntegTestZip() {
+        Project project = createProject(null);
+        Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName("integ-test-zip").build();
+        archiveProject.getConfigurations().create("default");
+        archiveProject.getArtifacts().add("default", new File("doesnotmatter"));
+        createDistro(project, "distro",
+            VersionProperties.getElasticsearch(), Type.INTEG_TEST_ZIP, null, null, null);
+        checkPlugin(project);
+    }
+
+    public void testCurrentVersionArchives() {
+        for (Platform platform : Platform.values()) {
+            for (Flavor flavor : Flavor.values()) {
+                for (boolean bundledJdk : new boolean[] { true, false}) {
+                    // create a new project in each iteration, so that we know we are resolving the only additional project being created
+                    Project project = createProject(null);
+                    String projectName = projectName(platform.toString(), flavor, bundledJdk);
+                    projectName += (platform == Platform.WINDOWS ? "-zip" : "-tar");
+                    Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName(projectName).build();
+                    archiveProject.getConfigurations().create("default");
+                    archiveProject.getArtifacts().add("default", new File("doesnotmatter"));
+                    createDistro(project, "distro",
+                        VersionProperties.getElasticsearch(), Type.ARCHIVE, platform, flavor, bundledJdk);
+                    checkPlugin(project);
+                }
+            }
+        }
+    }
+
+    public void testCurrentVersionPackages() {
+        for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
+            for (Flavor flavor : Flavor.values()) {
+                for (boolean bundledJdk : new boolean[] { true, false}) {
+                    Project project = createProject(null);
+                    String projectName = projectName(packageType.toString(), flavor, bundledJdk);
+                    Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build();
+                    packageProject.getConfigurations().create("default");
+                    packageProject.getArtifacts().add("default", new File("doesnotmatter"));
+                    createDistro(project, "distro",
+                        VersionProperties.getElasticsearch(), packageType, null, flavor, bundledJdk);
+                    checkPlugin(project);
+                }
+            }
+        }
+    }
+
+    public void testLocalBwcArchives() {
+        for (Platform platform : Platform.values()) {
+            for (Flavor flavor : Flavor.values()) {
+                // note: no non bundled jdk for bwc
+                String configName = projectName(platform.toString(), flavor, true);
+                configName += (platform == Platform.WINDOWS ? "-zip" : "-tar");
+
+                checkBwc("minor", configName, BWC_MINOR_VERSION, BWC_MINOR, Type.ARCHIVE, platform, flavor);
+                checkBwc("staged", configName, BWC_STAGED_VERSION, BWC_STAGED, Type.ARCHIVE, platform, flavor);
+                checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, BWC_BUGFIX, Type.ARCHIVE, platform, flavor);
+                checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, BWC_MAINTENANCE, Type.ARCHIVE, platform, flavor);
+            }
+        }
+    }
+
+    public void testLocalBwcPackages() {
+        for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
+            for (Flavor flavor : Flavor.values()) {
+                // note: no non bundled jdk for bwc
+                String configName = projectName(packageType.toString(), flavor, true);
+
+                checkBwc("minor", configName, BWC_MINOR_VERSION, BWC_MINOR, packageType, null, flavor);
+                checkBwc("staged", configName, BWC_STAGED_VERSION, BWC_STAGED, packageType, null, flavor);
+                checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, BWC_BUGFIX, packageType, null, flavor);
+                checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, BWC_MAINTENANCE, packageType, null, flavor);
+            }
+        }
+    }
+
+    private void assertDistroError(Project project, String name, String version, Type type, Platform platform,
+                                   Flavor flavor, Boolean bundledJdk, String message) {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> checkDistro(project, name, version, type, platform, flavor, bundledJdk));
+        assertThat(e.getMessage(), containsString(message));
+    }
+
+    private ElasticsearchDistribution createDistro(Project project, String name, String version, Type type,
+                              Platform platform, Flavor flavor, Boolean bundledJdk) {
+        @SuppressWarnings("unchecked")
+        NamedDomainObjectContainer<ElasticsearchDistribution> distros =
+            (NamedDomainObjectContainer<ElasticsearchDistribution>) project.getExtensions().getByName("elasticsearch_distributions");
+        return distros.create(name, distro -> {
+            if (version != null) {
+                distro.setVersion(version);
+            }
+            if (type != null) {
+                distro.setType(type);
+            }
+            if (platform != null) {
+                distro.setPlatform(platform);
+            }
+            if (flavor != null) {
+                distro.setFlavor(flavor);
+            }
+            if (bundledJdk != null) {
+                distro.setBundledJdk(bundledJdk);
+            }
+        });
+    }
+
+    // create a distro and finalize its configuration
+    private ElasticsearchDistribution checkDistro(Project project, String name, String version, Type type,
+                                                  Platform platform, Flavor flavor, Boolean bundledJdk) {
+        ElasticsearchDistribution distribution = createDistro(project, name, version, type, platform, flavor, bundledJdk);
+        distribution.finalizeValues();
+        return distribution;
+    }
+
+    // check the download plugin can be fully configured
+    private void checkPlugin(Project project) {
+        DistributionDownloadPlugin plugin = project.getPlugins().getPlugin(DistributionDownloadPlugin.class);
+        plugin.setupDistributions(project);
+    }
+
+    private void checkBwc(String projectName, String config, Version version, BwcVersions bwcVersions,
+                          Type type, Platform platform, Flavor flavor) {
+        Project project = createProject(bwcVersions);
+        Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build();
+        archiveProject.getConfigurations().create(config);
+        archiveProject.getArtifacts().add(config, new File("doesnotmatter"));
+        createDistro(project, "distro", version.toString(), type, platform, flavor, true);
+        checkPlugin(project);
+    }
+
+    private Project createProject(BwcVersions bwcVersions) {
+        rootProject = ProjectBuilder.builder().build();
+        Project distributionProject = ProjectBuilder.builder().withParent(rootProject).withName("distribution").build();
+        archivesProject = ProjectBuilder.builder().withParent(distributionProject).withName("archives").build();
+        packagesProject = ProjectBuilder.builder().withParent(distributionProject).withName("packages").build();
+        bwcProject = ProjectBuilder.builder().withParent(distributionProject).withName("bwc").build();
+        Project project = ProjectBuilder.builder().withParent(rootProject).build();
+        project.getExtensions().getExtraProperties().set("bwcVersions", bwcVersions);
+        project.getPlugins().apply("elasticsearch.distribution-download");
+        return project;
+    }
+
+    private static String projectName(String base, Flavor flavor, boolean bundledJdk) {
+        String prefix = "";
+        if (flavor == Flavor.OSS) {
+            prefix += "oss-";
+        }
+        if (bundledJdk == false) {
+            prefix += "no-jdk-";
+        }
+
+        return prefix + base;
+    }
+}

buildSrc/src/testKit/distribution-download/build.gradle

@@ -0,0 +1,44 @@
+import org.elasticsearch.gradle.BwcVersions
+import org.elasticsearch.gradle.Version
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// bring in build-tools onto the classpath
+plugins {
+    id 'elasticsearch.global-build-info' apply false
+}
+project.gradle.projectsEvaluated {
+    // wire the download service url to wiremock
+    String fakeDownloadService = System.getProperty('tests.download_service')
+    if (fakeDownloadService != null) {
+        println rootProject.repositories.asMap.keySet()
+        IvyArtifactRepository repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-downloads")
+        repository.setUrl(fakeDownloadService)
+        repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-downloads")
+        repository.setUrl(fakeDownloadService)
+    }
+}
+
+Version currentVersion = Version.fromString("2.0.0")
+BwcVersions versions = new BwcVersions(new TreeSet<>(
+        Arrays.asList(Version.fromString("1.0.0"), Version.fromString("1.0.1"), Version.fromString("1.1.0"), currentVersion)),
+        currentVersion)
+allprojects {
+    ext.bwcVersions = versions
+}

buildSrc/src/testKit/distribution-download/distribution/build.gradle

@@ -0,0 +1,35 @@
+
+String distroConfig = System.getProperty('tests.local_distro.config')
+if (distroConfig != null) {
+  // setup the test distribution as an artifact of this project
+  String distroType = System.getProperty('tests.distro.type')
+
+  Task buildDistro
+  File buildFile
+  if (['rpm', 'deb'].contains(distroType)) {
+    buildDistro = project.tasks.create("build" + distroType.capitalize(), Copy) {
+      from 'files'
+      into 'build/files'
+      include 'fake_elasticsearch.tar.gz'
+      // this shouldn't be extracted so we just rename the file so it is a dummy package
+      rename { filename -> filename.replace('tar.gz', distroType) }
+    }
+    buildFile = project.file("build/files/fake_elasticsearch." + distroType)
+  } else {
+    String distroPlatform = System.getProperty('tests.distro.platform')
+    String extension = "tar.gz"
+    if (distroType == 'archive' && distroPlatform == 'windows' || distroType == 'integ-test-zip') {
+      extension = "zip"
+    }
+    // copy file as is
+    buildDistro = project.tasks.create("buildArchive", Copy) {
+      from 'files'
+      include "fake_elasticsearch.${extension}"
+      into 'build/files'
+    }
+    buildFile = project.file("build/files/fake_elasticsearch.${extension}")
+  }
+
+  configurations.create(distroConfig)
+  artifacts.add(distroConfig, [file: buildFile, builtBy: buildDistro])
+}

buildSrc/src/testKit/distribution-download/settings.gradle

@@ -0,0 +1,8 @@
+include 'subproj'
+
+String distroProject = System.getProperty('tests.local_distro.project')
+System.out.println("local distro project: " + distroProject);
+if (distroProject != null) {
+  include distroProject
+  project(distroProject).projectDir = new File(rootDir, 'distribution')
+}

buildSrc/src/testKit/distribution-download/subproj/build.gradle

@@ -0,0 +1,58 @@
+
+plugins {
+    id 'elasticsearch.distribution-download'
+}
+
+String distroVersion = System.getProperty('tests.distro.version')
+String distroType = System.getProperty('tests.distro.type')
+String distroPlatform = System.getProperty('tests.distro.platform')
+String distroFlavor = System.getProperty('tests.distro.flavor')
+String distroBundledJdk = System.getProperty('tests.distro.bundledJdk')
+
+elasticsearch_distributions {
+  test_distro {
+    if (distroVersion != null) {
+      version = distroVersion
+    }
+    if (distroType != null) {
+      type = distroType
+    }
+    if (distroPlatform != null) {
+      platform = distroPlatform
+    }
+    if (distroFlavor != null) {
+      flavor = distroFlavor
+    }
+    if (distroBundledJdk != null) {
+      bundledJdk = Boolean.parseBoolean(distroBundledJdk)
+    }
+  }
+}
+
+task assertDistroFile {
+  dependsOn elasticsearch_distributions.test_distro
+  doLast {
+    File distroFile = new File(elasticsearch_distributions.test_distro.toString())
+    if (distroFile.exists() == false) {
+      throw new GradleException("distro file does not exist: ${distroFile}")
+    }
+    if (distroFile.isFile() == false) {
+      throw new GradleException("distro file is not a regular file: ${distroFile}")
+    }
+  }
+}
+
+if (['rpm', 'deb'].contains(distroType) == false) {
+  task assertDistroExtracted {
+    dependsOn elasticsearch_distributions.test_distro.extracted, assertDistroFile
+    doLast {
+      File distroExtracted = new File(elasticsearch_distributions.test_distro.extracted.toString())
+      if (distroExtracted.exists() == false) {
+        throw new GradleException("extracted does not exist: ${distroExtracted}")
+      }
+      if (distroExtracted.isDirectory() == false) {
+        throw new GradleException("extracted distro is not a directory: ${distroExtracted}")
+      }
+    }
+  }
+}