Dependency Graph gradle Task (#63641)

This change adds a gradle task that builds a simplified dependency graph
of our runtime dependencies and pushes that to be monitored by a
software composition analysis service.

benchmarks/build.gradle

@@ -55,6 +55,8 @@ disableTasks('forbiddenApisMain')
 // No licenses for our benchmark deps (we don't ship benchmarks)
 tasks.named("dependencyLicenses").configure { enabled = false }
 tasks.named("dependenciesInfo").configure {  enabled = false }
+tasks.named("dependenciesGraph").configure {  enabled = false }
+
 
 tasks.named("thirdPartyAudit").configure {
   ignoreViolations(

build.gradle

@@ -419,6 +419,9 @@ gradle.projectsEvaluated {
       maybeConfigure(project.tasks, 'dependenciesInfo') {
         it.enabled = false
       }
+      maybeConfigure(project.tasks, 'dependenciesGraph') {
+        it.enabled = false
+      }
     }
   }
   // Having the same group and name for distinct projects causes Gradle to consider them equal when resolving

buildSrc/build.gradle

@@ -104,6 +104,8 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.13.4'
   api 'org.apache.maven:maven-model:3.6.2'
   api 'com.networknt:json-schema-validator:1.0.36'
+  api "org.apache.httpcomponents:httpclient:${props.getProperty('httpclient')}"
+  api "org.apache.httpcomponents:httpcore:${props.getProperty('httpcore')}"
   compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testImplementation "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
@@ -155,6 +157,7 @@ if (project != rootProject) {
   // build-tools is not ready for primetime with these...
   tasks.named("dependencyLicenses").configure { enabled = false }
   tasks.named("dependenciesInfo").configure {enabled = false }
+  tasks.named("dependenciesGraph").configure {enabled = false }
   disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
   tasks.named("jarHell").configure {
     enabled = false

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -49,6 +49,7 @@ class BuildPlugin implements Plugin<Project> {
         configureLicenseAndNotice(project)
         project.pluginManager.apply('elasticsearch.publish')
         project.pluginManager.apply(DependenciesInfoPlugin)
+        project.pluginManager.apply(DependenciesGraphPlugin)
 
         PrecommitTasks.create(project, true)
     }

buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphPlugin.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.GradleException;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class DependenciesGraphPlugin implements Plugin<Project> {
+
+    public void apply(Project project) {
+        project.getRootProject().getPluginManager().apply(DependenciesGraphHookPlugin.class);
+        final String url = System.getenv("SCA_URL");
+        final String token = System.getenv("SCA_TOKEN");
+        TaskProvider<DependenciesGraphTask> depsGraph = project.getTasks().register("dependenciesGraph", DependenciesGraphTask.class);
+        depsGraph.configure(t -> {
+            project.getPlugins().withType(JavaPlugin.class, p -> {
+                t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
+                t.setToken(token);
+                t.setUrl(url);
+            });
+        });
+    }
+
+    static class DependenciesGraphHookPlugin implements Plugin<Project> {
+
+        @Override
+        public void apply(Project project) {
+            if (project != project.getRootProject()) {
+                throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
+            }
+            final String url = System.getenv("SCA_URL");
+            final String token = System.getenv("SCA_TOKEN");
+            project.getGradle().getTaskGraph().whenReady(graph -> {
+                List<String> depGraphTasks = graph.getAllTasks()
+                    .stream()
+                    .filter(t -> t instanceof DependenciesGraphTask)
+                    .map(Task::getPath)
+                    .collect(Collectors.toList());
+                if (depGraphTasks.size() > 0) {
+                    if (url == null || token == null) {
+                        // If there are more than one DependenciesGraphTasks to run, print the message only for one of
+                        // them as the resolving action is the same for all
+                        throw new GradleException(
+                            "The environment variables SCA_URL and SCA_TOKEN need to be set before task "
+                                + depGraphTasks.get(0)
+                                + " can run"
+                        );
+                    }
+                }
+            });
+
+        }
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphTask.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.gradle.api.GradleException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.DependencySet;
+import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.TaskAction;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A task to generate a dependency graph of our runtime dependencies and push that via
+ * an API call to a given endpoint of a SCA tool/service.
+ * The graph is built according to the specification in https://github.com/snyk/dep-graph#depgraphdata
+ *
+ * Due to the nature of our dependency resolution in gradle, we are abusing the aforementioned graph definition as
+ * the graph we construct has a single root ( the subproject ) and all dependencies are children of that root,
+ * irrespective of if they are direct dependencies or transitive ones ( that should be children of other children ).
+ * Although we end up lacking some contextual information, this allows us to scan and monitor only the dependencies
+ * that are bundled and used in runtime.
+ */
+public class DependenciesGraphTask extends DefaultTask {
+
+    private Configuration runtimeConfiguration;
+    private String token;
+    private String url;
+
+    @Input
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    @Input
+    public String getToken() {
+        return token;
+    }
+
+    public void setToken(String token) {
+        this.token = token;
+    }
+
+    @InputFiles
+    public Configuration getRuntimeConfiguration() {
+        return runtimeConfiguration;
+    }
+
+    public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
+        this.runtimeConfiguration = runtimeConfiguration;
+    }
+
+    @TaskAction
+    void generateDependenciesGraph() {
+
+        if (getProject().getGradle().getStartParameter().isOffline()) {
+            throw new GradleException("Must run in online mode in order to submit the dependency graph to the SCA service");
+        }
+
+        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
+        final Set<String> packages = new HashSet<>();
+        final Set<String> nodes = new HashSet<>();
+        final Set<String> nodeIds = new HashSet<>();
+        for (final Dependency dependency : runtimeDependencies) {
+            final String id = dependency.getGroup() + ":" + dependency.getName();
+            final String versionedId = id + "@" + dependency.getVersion();
+            final StringBuilder packageString = new StringBuilder();
+            final StringBuilder nodeString = new StringBuilder();
+            if (dependency instanceof ProjectDependency) {
+                continue;
+            }
+            packageString.append("{\"id\": \"")
+                .append(versionedId)
+                .append("\",\"info\": {\"name\": \"")
+                .append(id)
+                .append("\",\"version\": \"")
+                .append(dependency.getVersion())
+                .append("\"}}");
+            packages.add(packageString.toString());
+            nodeString.append("{\"nodeId\": \"")
+                .append(versionedId)
+                .append("\",\"pkgId\": \"")
+                .append(versionedId)
+                .append("\",\"deps\": []}");
+            nodes.add(nodeString.toString());
+            nodeIds.add("{\"nodeId\": \"" + versionedId + "\"}");
+        }
+        // We add one package and one node for each dependency, it suffices to check packages.
+        if (packages.size() > 0) {
+            final String projectName = "elastic/elasticsearch" + getProject().getPath();
+            final StringBuilder output = new StringBuilder();
+            output.append("{\"depGraph\": {\"schemaVersion\": \"1.2.0\",\"pkgManager\": {\"name\": \"gradle\"},\"pkgs\": [")
+                .append("{\"id\": \"")
+                .append(projectName)
+                .append("@0.0.0")
+                .append("\", \"info\": {\"name\": \"")
+                .append(projectName)
+                .append("\", \"version\": \"0.0.0\"}},")
+                .append(String.join(",", packages))
+                .append("],\"graph\": {\"rootNodeId\": \"")
+                .append(projectName)
+                .append("@0.0.0")
+                .append("\",\"nodes\": [")
+                .append("{\"nodeId\": \"")
+                .append(projectName)
+                .append("@0.0.0")
+                .append("\",\"pkgId\": \"")
+                .append(projectName)
+                .append("@0.0.0")
+                .append("\",\"deps\": [")
+                .append(String.join(",", nodeIds))
+                .append("]},")
+                .append(String.join(",", nodes))
+                .append("]}}}");
+            getLogger().debug("Dependency Graph: " + output.toString());
+            try (CloseableHttpClient client = HttpClients.createDefault()) {
+                HttpPost postRequest = new HttpPost(url);
+                postRequest.addHeader("Authorization", "token " + token);
+                postRequest.addHeader("Content-Type", "application/json");
+                postRequest.setEntity(new StringEntity(output.toString()));
+                CloseableHttpResponse response = client.execute(postRequest);
+                getLogger().info("API call response status: " + response.getStatusLine().getStatusCode());
+                getLogger().debug(EntityUtils.toString(response.getEntity()));
+            } catch (Exception e) {
+                throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e);
+            }
+        }
+    }
+}

client/test/build.gradle

@@ -50,6 +50,7 @@ jarHell.enabled = false
 // TODO: should we have licenses for our test deps?
 tasks.named("dependencyLicenses").configure { enabled = false }
 tasks.named("dependenciesInfo").configure { enabled = false }
+tasks.named("dependenciesGraph").configure { it.enabled = false }
 
 //we aren't releasing this jar
 tasks.named("thirdPartyAudit").configure { enabled = false }

test/framework/build.gradle

@@ -49,6 +49,7 @@ tasks.named('forbiddenApisMain').configure {
 // TODO: should we have licenses for our test deps?
 tasks.named("dependencyLicenses").configure { enabled = false }
 tasks.named("dependenciesInfo").configure { enabled = false }
+tasks.named("dependenciesGraph").configure { enabled = false }
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(

x-pack/plugin/sql/qa/server/build.gradle

@@ -36,7 +36,7 @@ dependencies {
 }
 
 // this is just a test fixture used by other projects and not in production
-['test', 'dependencyLicenses', 'thirdPartyAudit', 'dependenciesInfo'].each {
+['test', 'dependencyLicenses', 'thirdPartyAudit', 'dependenciesInfo', 'dependenciesGraph'].each {
   tasks.named(it).configure {
     enabled = false
   }