Scan stable plugins for named components upon install (#92528)

stable plugins not build with ES's gradle plugin will not have named_components.json file.
To allow these plugins to expose their named components, a scan can be performed upon install.

relates #88980

distribution/tools/plugin-cli/build.gradle

@@ -10,9 +10,18 @@ apply plugin: 'elasticsearch.build'
 
 archivesBaseName = 'elasticsearch-plugin-cli'
 
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:elasticsearch-cli")
+  implementation project(":libs:elasticsearch-plugin-api")
+  implementation project(":libs:elasticsearch-plugin-scanner")
+  implementation 'org.ow2.asm:asm:9.3'
+  implementation 'org.ow2.asm:asm-tree:9.3'
+
   api "org.bouncycastle:bcpg-fips:1.0.4"
   api "org.bouncycastle:bc-fips:1.0.2"
   testImplementation project(":test:framework")

distribution/tools/plugin-cli/licenses/asm-LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

distribution/tools/plugin-cli/licenses/asm-NOTICE.txt

@@ -0,0 +1 @@
+ 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -38,9 +38,12 @@ import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
+import org.elasticsearch.plugin.scanner.ClassReaders;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginsUtils;
+import org.objectweb.asm.ClassReader;
 
 import java.io.BufferedReader;
 import java.io.Closeable;
@@ -82,6 +85,7 @@ import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -197,6 +201,7 @@ public class InstallPluginAction implements Closeable {
     private Environment env;
     private boolean batch;
     private Proxy proxy = null;
+    private NamedComponentScanner scanner = new NamedComponentScanner();
 
     public InstallPluginAction(Terminal terminal, Environment env, boolean batch) {
         this.terminal = terminal;
@@ -208,7 +213,6 @@ public class InstallPluginAction implements Closeable {
         this.proxy = proxy;
     }
 
-    // pkg private for testing
     public void execute(List<InstallablePlugin> plugins) throws Exception {
         if (plugins.isEmpty()) {
             throw new UserException(ExitCodes.USAGE, "at least one plugin id is required");
@@ -867,9 +871,24 @@ public class InstallPluginAction implements Closeable {
         // check for jar hell before any copying
         jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile());
 
+        if (info.isStable() && hasNamedComponentFile(pluginRoot) == false) {
+            generateNameComponentFile(pluginRoot);
+        }
         return info;
     }
 
+    private void generateNameComponentFile(Path pluginRoot) throws IOException {
+        Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream(); // contains plugin-api
+        List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(pluginRoot).stream(), classPath).toList();
+        Map<String, Map<String, String>> namedComponentsMap = scanner.scanForNamedClasses(classReaders);
+        Path outputFile = pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        scanner.writeToFile(namedComponentsMap, outputFile);
+    }
+
+    private boolean hasNamedComponentFile(Path pluginRoot) {
+        return Files.exists(pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME));
+    }
+
     private static final String LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR;
 
     static {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -48,11 +48,14 @@ import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginTestUtil;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -86,6 +89,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -124,6 +128,7 @@ public class InstallPluginActionTests extends ESTestCase {
     private MockTerminal terminal;
     private Tuple<Path, Environment> env;
     private Path pluginDir;
+    private NamedComponentScanner namedComponentScanner;
 
     private final boolean isPosix;
     private final boolean isReal;
@@ -131,6 +136,8 @@ public class InstallPluginActionTests extends ESTestCase {
 
     @SuppressForbidden(reason = "sets java.io.tmpdir")
     public InstallPluginActionTests(FileSystem fs, Function<String, Path> temp) {
+        assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set";
+
         this.temp = temp;
         this.isPosix = fs.supportedFileAttributeViews().contains("posix");
         this.isReal = fs == PathUtils.getDefaultFileSystem();
@@ -152,6 +159,7 @@ public class InstallPluginActionTests extends ESTestCase {
                 // no jarhell check
             }
         };
+
         defaultAction = new InstallPluginAction(terminal, env.v2(), false);
     }
 
@@ -199,7 +207,9 @@ public class InstallPluginActionTests extends ESTestCase {
         return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
     }
 
-    /** Creates a test environment with bin, config and plugins directories. */
+    /**
+     * Creates a test environment with bin, config and plugins directories.
+     */
     static Tuple<Path, Environment> createEnv(Function<String, Path> temp) throws IOException {
         Path home = temp.apply("install-plugin-command-tests");
         Files.createDirectories(home.resolve("bin"));
@@ -216,7 +226,9 @@ public class InstallPluginActionTests extends ESTestCase {
         return temp.apply("pluginDir");
     }
 
-    /** creates a fake jar file with empty class files */
+    /**
+     * creates a fake jar file with empty class files
+     */
     static void writeJar(Path jar, String... classes) throws IOException {
         try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
             for (String clazz : classes) {
@@ -237,13 +249,47 @@ public class InstallPluginActionTests extends ESTestCase {
         return zip;
     }
 
-    /** creates a plugin .zip and returns the url for testing */
+    /**
+     * creates a plugin .zip and returns the url for testing
+     */
     static InstallablePlugin createPluginZip(String name, Path structure, String... additionalProps) throws IOException {
         return createPlugin(name, structure, additionalProps);
     }
 
+    static void writeStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        String[] properties = pluginProperties(name, additionalProps, true);
+        PluginTestUtil.writeStablePluginProperties(structure, properties);
+
+        if (hasNamedComponentFile) {
+            PluginTestUtil.writeNamedComponentsFile(structure, namedComponentsJSON());
+        }
+        Path jar = structure.resolve("plugin.jar");
+
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("a_component")
+            public class A implements ExtensibleInterface{}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("b_component")
+            public class B implements ExtensibleInterface{}
+            """)));
+    }
+
     static void writePlugin(String name, Path structure, String... additionalProps) throws IOException {
-        String[] properties = Stream.concat(
+        String[] properties = pluginProperties(name, additionalProps, false);
+        PluginTestUtil.writePluginProperties(structure, properties);
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    private static String[] pluginProperties(String name, String[] additionalProps, boolean isStable) {
+        return Stream.of(
             Stream.of(
                 "description",
                 "fake desc",
@@ -254,15 +300,12 @@ public class InstallPluginActionTests extends ESTestCase {
                 "elasticsearch.version",
                 Version.CURRENT.toString(),
                 "java.version",
-                System.getProperty("java.specification.version"),
-                "classname",
-                "FakePlugin"
+                System.getProperty("java.specification.version")
+
             ),
+            isStable ? Stream.empty() : Stream.of("classname", "FakePlugin"),
             Arrays.stream(additionalProps)
-        ).toArray(String[]::new);
-        PluginTestUtil.writePluginProperties(structure, properties);
-        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
-        writeJar(structure.resolve("plugin.jar"), className);
+        ).flatMap(Function.identity()).toArray(String[]::new);
     }
 
     static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
@@ -276,6 +319,12 @@ public class InstallPluginActionTests extends ESTestCase {
         Files.write(pluginDir.resolve("plugin-security.policy"), securityPolicyContent.toString().getBytes(StandardCharsets.UTF_8));
     }
 
+    static InstallablePlugin createStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        writeStablePlugin(name, structure, hasNamedComponentFile, additionalProps);
+        return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
+    }
+
     static InstallablePlugin createPlugin(String name, Path structure, String... additionalProps) throws IOException {
         writePlugin(name, structure, additionalProps);
         return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
@@ -310,6 +359,11 @@ public class InstallPluginActionTests extends ESTestCase {
         assertInstallCleaned(environment);
     }
 
+    void assertNamedComponentFile(String name, Path pluginDir, String expectedContent) throws IOException {
+        Path namedComponents = pluginDir.resolve(name).resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        assertThat(Files.readString(namedComponents), equalTo(expectedContent));
+    }
+
     void assertPluginInternal(String name, Path pluginsFile, Path originalPlugin) throws IOException {
         Path got = pluginsFile.resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
@@ -1507,4 +1561,42 @@ public class InstallPluginActionTests extends ESTestCase {
             assertThat(terminal.getErrorOutput(), containsString("[" + id + "] is no longer a plugin"));
         }
     }
+
+    public void testStablePluginWithNamedComponentsFile() throws Exception {
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, true);
+        installPlugins(List.of(stablePluginZip), env.v1());
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testStablePluginWithoutNamedComponentsFile() throws Exception {
+        // named component will have to be generated upon install
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, false);
+
+        installPlugins(List.of(stablePluginZip), env.v1());
+
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    private Map<String, Map<String, String>> namedComponentsMap() {
+        Map<String, Map<String, String>> result = new LinkedHashMap<>();
+        Map<String, String> extensibles = new LinkedHashMap<>();
+        extensibles.put("a_component", "p.A");
+        extensibles.put("b_component", "p.B");
+        result.put("org.elasticsearch.plugins.cli.test_model.ExtensibleInterface", extensibles);
+        return result;
+    }
+
+    private static String namedComponentsJSON() {
+        return """
+            {
+              "org.elasticsearch.plugins.cli.test_model.ExtensibleInterface": {
+                "a_component": "p.A",
+                "b_component": "p.B"
+              }
+            }
+            """.replaceAll("[\n\r\s]", "");
+    }
 }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.cli.test_model;
+
+import org.elasticsearch.plugin.Extensible;
+
+@Extensible
+public interface ExtensibleInterface {}

docs/changelog/92528.yaml

@@ -0,0 +1,5 @@
+pr: 92528
+summary: Scan stable plugins for named components upon install
+area: Infra/CLI
+type: enhancement
+issues: []

libs/plugin-scanner/build.gradle

@@ -19,8 +19,8 @@ dependencies {
   api project(':libs:elasticsearch-plugin-api')
   api project(":libs:elasticsearch-x-content")
 
-  implementation 'org.ow2.asm:asm:9.3'
-  implementation 'org.ow2.asm:asm-tree:9.3'
+  api 'org.ow2.asm:asm:9.3'
+  api 'org.ow2.asm:asm-tree:9.3'
 
   testImplementation "junit:junit:${versions.junit}"
   testImplementation(project(":test:framework")) {

libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java

@@ -40,11 +40,10 @@ public class ClassReaders {
      * This method must be used within a try-with-resources statement or similar
      * control structure.
      */
-    public static List<ClassReader> ofDirWithJars(String path) {
-        if (path == null) {
+    public static List<ClassReader> ofDirWithJars(Path dir) {
+        if (dir == null) {
             return Collections.emptyList();
         }
-        Path dir = Paths.get(path);
         try (var stream = Files.list(dir)) {
             return ofPaths(stream);
         } catch (IOException e) {

libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java

@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 
 public class NamedComponentScanner {
+
     // main method to be used by gradle build plugin
     public static void main(String[] args) throws IOException {
         List<ClassReader> classReaders = ClassReaders.ofClassPath();
@@ -37,8 +38,7 @@ public class NamedComponentScanner {
     }
 
     // scope for testing
-    void writeToFile(Map<String, Map<String, String>> namedComponentsMap, Path outputFile) throws IOException {
-        // String json = OBJECT_MAPPER.writeValueAsString(namedComponentsMap);
+    public void writeToFile(Map<String, Map<String, String>> namedComponentsMap, Path outputFile) throws IOException {
         Files.createDirectories(outputFile.getParent());
 
         try (OutputStream outputStream = Files.newOutputStream(outputFile)) {
@@ -58,12 +58,12 @@ public class NamedComponentScanner {
     }
 
     // returns a Map<String, Map<String,String> - extensible interface -> map{ namedName -> className }
-    public Map<String, Map<String, String>> scanForNamedClasses(List<ClassReader> classReaderStream) {
+    public Map<String, Map<String, String>> scanForNamedClasses(List<ClassReader> classReaders) {
         ClassScanner extensibleClassScanner = new ClassScanner(Type.getDescriptor(Extensible.class), (classname, map) -> {
             map.put(classname, classname);
             return null;
         });
-        extensibleClassScanner.visit(classReaderStream);
+        extensibleClassScanner.visit(classReaders);
 
         ClassScanner namedComponentsScanner = new ClassScanner(
             Type.getDescriptor(NamedComponent.class),
@@ -77,7 +77,7 @@ public class NamedComponentScanner {
             }
         );
 
-        namedComponentsScanner.visit(classReaderStream);
+        namedComponentsScanner.visit(classReaders);
 
         Map<String, Map<String, String>> componentInfo = new HashMap<>();
         for (var e : namedComponentsScanner.getFoundClasses().entrySet()) {

libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java

@@ -122,7 +122,7 @@ public class ClassReadersTests extends ESTestCase {
             public class D {}
             """)));
 
-        List<ClassReader> classReaders = ClassReaders.ofDirWithJars(dirWithJar.toString());
+        List<ClassReader> classReaders = ClassReaders.ofDirWithJars(dirWithJar);
         List<String> collect = classReaders.stream().map(cr -> cr.getClassName()).collect(Collectors.toList());
         org.hamcrest.MatcherAssert.assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B", "p/C", "p/D"));
     }

libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java

@@ -77,7 +77,7 @@ public class NamedComponentScannerTests extends ESTestCase {
             public class B implements ExtensibleInterface{}
             """)));
         List<ClassReader> classReaderStream = Stream.concat(
-            ClassReaders.ofDirWithJars(dirWithJar.toString()).stream(),
+            ClassReaders.ofDirWithJars(dirWithJar).stream(),
             ClassReaders.ofClassPath().stream()
         )// contains plugin-api
             .toList();
@@ -154,13 +154,11 @@ public class NamedComponentScannerTests extends ESTestCase {
         Path jar = dirWithJar.resolve("plugin.jar");
         JarUtils.createJarWithEntries(jar, jarEntries);
 
-        List<ClassReader> classReaderStream = Stream.concat(
-            ClassReaders.ofDirWithJars(dirWithJar.toString()).stream(),
-            ClassReaders.ofClassPath().stream()
-        )// contains plugin-api
+        Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream();
+        List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(dirWithJar).stream(), classPath)// contains plugin-api
             .toList();
 
-        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaderStream);
+        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaders);
 
         org.hamcrest.MatcherAssert.assertThat(
             namedComponents,

server/src/main/java/org/elasticsearch/plugins/PluginDescriptor.java

@@ -41,6 +41,8 @@ public class PluginDescriptor implements Writeable, ToXContentObject {
 
     public static final String INTERNAL_DESCRIPTOR_FILENAME = "plugin-descriptor.properties";
     public static final String STABLE_DESCRIPTOR_FILENAME = "stable-plugin-descriptor.properties";
+    public static final String NAMED_COMPONENTS_FILENAME = "named_components.json";
+
     public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
     private static final Version LICENSED_PLUGINS_SUPPORT = Version.V_7_11_0;

test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java

@@ -60,4 +60,9 @@ public class PluginTestUtil {
             properties.store(out, "");
         }
     }
+
+    public static void writeNamedComponentsFile(Path pluginDir, String namedComponentsJson) throws IOException {
+        Path namedComponentsFile = pluginDir.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        Files.writeString(namedComponentsFile, namedComponentsJson);
+    }
 }