Decentralize plugin security

* Add ability for plugins to declare additional permissions with a custom plugin-security.policy file and corresponding AccessController logic. See the plugin author's guide for more information.
* Add warning messages to users for extra plugin permissions in bin/plugin.
* When bin/plugin is run interactively (stdin is a controlling terminal and -b/--batch not supplied), require user confirmation.
* Improve unit test and IDE support for plugins with additional permissions by exposing plugin's metadata as a maven test resource.

Closes #14108

Squashed commit of the following:

commit cf8ace65a7397aaccd356bf55f95d6fbb8bb571c
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 13:36:05 2015 -0400

    fix new unit test from master merge

commit 9be3c5aa38f2d9ae50f3d54924a30ad9cddeeb65
Merge: 2f168b8 7368231
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:58:31 2015 -0400

    Merge branch 'master' into off_my_back

commit 2f168b8038e32672f01ad0279fb5db77ba902ae8
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:56:04 2015 -0400

    improve plugin author documentation

commit 6e6c2bfda68a418d92733ac22a58eec35508b2d0
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:52:14 2015 -0400

    move security confirmation after 'plugin already installed' check, to prevent user from answering unnecessary questions.

commit 08233a2972554afef2a6a7521990283102e20d92
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 05:36:42 2015 -0400

    Add documentation and pluginmanager support

commit 05dad86c51488ba43ccbd749f0164f3fbd3aee62
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 02:22:24 2015 -0400

    Decentralize plugin permissions (modulo docs and pluginmanager work)

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -29,6 +29,7 @@ import java.security.PermissionCollection;
 import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.security.URIParameter;
+import java.util.Map;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
@@ -41,13 +42,15 @@ final class ESPolicy extends Policy {
     final Policy template;
     final Policy untrusted;
     final PermissionCollection dynamic;
+    final Map<String,PermissionCollection> plugins;
 
-    public ESPolicy(PermissionCollection dynamic) throws Exception {
+    public ESPolicy(PermissionCollection dynamic, Map<String,PermissionCollection> plugins) throws Exception {
         URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI();
         URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI();
         this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri));
         this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri));
         this.dynamic = dynamic;
+        this.plugins = plugins;
     }
 
     @Override @SuppressForbidden(reason = "fast equals check is desired")
@@ -66,6 +69,11 @@ final class ESPolicy extends Policy {
             if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
                 return untrusted.implies(domain, permission);
             }
+            // check for an additional plugin permission
+            PermissionCollection plugin = plugins.get(location.getFile());
+            if (plugin != null && plugin.implies(permission)) {
+                return true;
+            }
         }
 
         // Special handling for broken AWS code which destroys all SSL security

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -21,6 +21,7 @@ package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.*;
 import java.net.URL;
@@ -30,8 +31,11 @@ import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
+import java.security.URIParameter;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
@@ -65,7 +69,7 @@ import java.util.regex.Pattern;
  * when they are so dangerous that general code should not be granted the
  * permission, but there are extenuating circumstances.
  * <p>
- * Groovy scripts are assigned no permissions. This does not provide adequate
+ * Scripts (groovy, javascript, python) are assigned minimal permissions. This does not provide adequate
  * sandboxing, as these scripts still have access to ES classes, and could
  * modify members, etc that would cause bad things to happen later on their
  * behalf (no package protections are yet in place, this would need some
@@ -81,7 +85,7 @@ import java.util.regex.Pattern;
  * <h1>Debugging Security</h1>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
- * JAVA_OPTS="-Djava.security.debug=access:failure" bin/elasticsearch
+ * JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
  * </pre>
  * See <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/troubleshooting-security.html">
  * Troubleshooting Security</a> for information.
@@ -97,11 +101,9 @@ final class Security {
     static void configure(Environment environment) throws Exception {
         // set properties for jar locations
         setCodebaseProperties();
-        // set properties for problematic plugins
-        setPluginCodebaseProperties(environment);
 
-        // enable security policy: union of template and environment-based paths.
-        Policy.setPolicy(new ESPolicy(createPermissions(environment)));
+        // enable security policy: union of template and environment-based paths, and possibly plugin permissions
+        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment)));
 
         // enable security manager
         System.setSecurityManager(new SecurityManager() {
@@ -157,70 +159,39 @@ final class Security {
         }
     }
 
-    // mapping of plugins to plugin class name. see getPluginClass why we need this.
-    // plugin codebase property is always implicit (es.security.plugin.foobar)
-    // note that this is only read once, when policy is parsed.
-    static final Map<String,String> SPECIAL_PLUGINS;
-    static {
-        Map<String,String> m = new HashMap<>();
-        m.put("repository-s3",       "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin");
-        m.put("discovery-ec2",       "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin");
-        m.put("discovery-gce",       "org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin");
-        m.put("lang-expression",     "org.elasticsearch.script.expression.ExpressionPlugin");
-        m.put("lang-groovy",         "org.elasticsearch.script.groovy.GroovyPlugin");
-        m.put("lang-javascript",     "org.elasticsearch.plugin.javascript.JavaScriptPlugin");
-        m.put("lang-python",         "org.elasticsearch.plugin.python.PythonPlugin");
-        SPECIAL_PLUGINS = Collections.unmodifiableMap(m);
-    }
-
-    /**
-     * Returns policy property for plugin, if it has special permissions.
-     * otherwise returns null.
-     */
-    static String getPluginProperty(String pluginName) {
-        if (SPECIAL_PLUGINS.containsKey(pluginName)) {
-            return "es.security.plugin." + pluginName;
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * Returns plugin class name, if it has special permissions.
-     * otherwise returns null.
-     */
-    // this is only here to support the intellij IDE
-    // it sucks to duplicate information, but its worth the tradeoff: sanity
-    // if it gets out of sync, tests will fail.
-    static String getPluginClass(String pluginName) {
-        return SPECIAL_PLUGINS.get(pluginName);
-    }
-
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
      */
     @SuppressForbidden(reason = "proper use of URL")
-    static void setPluginCodebaseProperties(Environment environment) throws IOException {
+    static Map<String,PermissionCollection> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
+        Map<String,PermissionCollection> map = new HashMap<>();
         if (Files.exists(environment.pluginsFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
                 for (Path plugin : stream) {
-                    String prop = getPluginProperty(plugin.getFileName().toString());
-                    if (prop != null) {
-                        if (System.getProperty(prop) != null) {
-                            throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop));
+                    Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY);
+                    if (Files.exists(policyFile)) {
+                        // parse the plugin's policy file into a set of permissions
+                        Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri()));
+                        PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain());
+                        // this method is supported with the specific implementation we use, but just check for safety.
+                        if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
+                            throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
+                        }
+                        // grant the permissions to each jar in the plugin
+                        try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
+                            for (Path jar : jarStream) {
+                                if (map.put(jar.toUri().toURL().getFile(), permissions) != null) {
+                                    // just be paranoid ok?
+                                    throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
+                                }
+                            }
                         }
-                        System.setProperty(prop, plugin.toUri().toURL().toString() + "*");
                     }
                 }
             }
         }
-        for (String plugin : SPECIAL_PLUGINS.keySet()) {
-            String prop = getPluginProperty(plugin);
-            if (System.getProperty(prop) == null) {
-                System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all"
-            }
-        }
+        return Collections.unmodifiableMap(map);
     }
 
     /** returns dynamic Permissions to configured paths */

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -36,6 +36,7 @@ import java.util.Properties;
 public class PluginInfo implements Streamable, ToXContent {
 
     public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
+    public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
     static final class Fields {
         static final XContentBuilderString NAME = new XContentBuilderString("name");

core/src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -100,7 +100,7 @@ public class PluginManager {
         this.timeout = timeout;
     }
 
-    public void downloadAndExtract(String name, Terminal terminal) throws IOException {
+    public void downloadAndExtract(String name, Terminal terminal, boolean batch) throws IOException {
         if (name == null && url == null) {
             throw new IllegalArgumentException("plugin name or url must be supplied with install.");
         }
@@ -124,7 +124,7 @@ public class PluginManager {
         }
 
         Path pluginFile = download(pluginHandle, terminal);
-        extract(pluginHandle, terminal, pluginFile);
+        extract(pluginHandle, terminal, pluginFile, batch);
     }
 
     private Path download(PluginHandle pluginHandle, Terminal terminal) throws IOException {
@@ -207,7 +207,7 @@ public class PluginManager {
         return pluginFile;
     }
 
-    private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile) throws IOException {
+    private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile, boolean batch) throws IOException {
         // unzip plugin to a staging temp dir, named for the plugin
         Path tmp = Files.createTempDirectory(environment.tmpFile(), null);
         Path root = tmp.resolve(pluginHandle.name);
@@ -232,6 +232,13 @@ public class PluginManager {
             throw new IOException("plugin directory " + extractLocation.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + pluginHandle.name + "' command");
         }
 
+        // read optional security policy (extra permissions)
+        // if it exists, confirm or warn the user
+        Path policy = root.resolve(PluginInfo.ES_PLUGIN_POLICY);
+        if (Files.exists(policy)) {
+            PluginSecurity.readPolicy(policy, terminal, environment, batch);
+        }
+
         // install plugin
         FileSystemUtils.copyDirectoryRecursively(root, extractLocation);
         terminal.println("Installed %s into %s", pluginHandle.name, extractLocation.toAbsolutePath());
@@ -335,7 +342,7 @@ public class PluginManager {
         fileAttributeView.setPermissions(permissions);
     }
 
-    private void tryToDeletePath(Terminal terminal, Path ... paths) {
+    static void tryToDeletePath(Terminal terminal, Path ... paths) {
         for (Path path : paths) {
             try {
                 IOUtils.rm(path);

core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java

@@ -180,6 +180,7 @@ public class PluginManagerCliParser extends CliTool {
 
         private static final CliToolConfig.Cmd CMD = cmd(NAME, Install.class)
                 .options(option("t", "timeout").required(false).hasArg(false))
+                .options(option("b", "batch").required(false))
                 .build();
 
         static Command parse(Terminal terminal, CommandLine cli) {
@@ -210,21 +211,28 @@ public class PluginManagerCliParser extends CliTool {
             if (cli.hasOption("v")) {
                 outputMode = OutputMode.VERBOSE;
             }
+            
+            boolean batch = System.console() == null;
+            if (cli.hasOption("b")) {
+                batch = true;
+            }
 
-            return new Install(terminal, name, outputMode, optionalPluginUrl, timeout);
+            return new Install(terminal, name, outputMode, optionalPluginUrl, timeout, batch);
         }
 
         final String name;
         private OutputMode outputMode;
         final URL url;
         final TimeValue timeout;
+        final boolean batch;
 
-        Install(Terminal terminal, String name, OutputMode outputMode, URL url, TimeValue timeout) {
+        Install(Terminal terminal, String name, OutputMode outputMode, URL url, TimeValue timeout, boolean batch) {
             super(terminal);
             this.name = name;
             this.outputMode = outputMode;
             this.url = url;
             this.timeout = timeout;
+            this.batch = batch;
         }
 
         @Override
@@ -235,7 +243,7 @@ public class PluginManagerCliParser extends CliTool {
             } else {
                 terminal.println("-> Installing from " + URLDecoder.decode(url.toString(), "UTF-8") + "...");
             }
-            pluginManager.downloadAndExtract(name, terminal);
+            pluginManager.downloadAndExtract(name, terminal, batch);
             return ExitStatus.OK;
         }
     }

core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java

@@ -0,0 +1,177 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.common.cli.Terminal.Verbosity;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.security.Permission;
+import java.security.PermissionCollection;
+import java.security.Permissions;
+import java.security.Policy;
+import java.security.URIParameter;
+import java.security.UnresolvedPermission;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+class PluginSecurity {
+    
+    /**
+     * Reads plugin policy, prints/confirms exceptions
+     */
+    static void readPolicy(Path file, Terminal terminal, Environment environment, boolean batch) throws IOException {
+        PermissionCollection permissions = parsePermissions(terminal, file, environment.tmpFile());
+        List<Permission> requested = Collections.list(permissions.elements());
+        if (requested.isEmpty()) {
+            terminal.print(Verbosity.VERBOSE, "plugin has a policy file with no additional permissions");
+            return;
+        }
+        
+        // sort permissions in a reasonable order
+        Collections.sort(requested, new Comparator<Permission>() {
+            @Override
+            public int compare(Permission o1, Permission o2) {
+                int cmp = o1.getClass().getName().compareTo(o2.getClass().getName());
+                if (cmp == 0) {
+                    String name1 = o1.getName();
+                    String name2 = o2.getName();
+                    if (name1 == null) {
+                        name1 = "";
+                    }
+                    if (name2 == null) {
+                        name2 = "";
+                    }
+                    cmp = name1.compareTo(name2);
+                    if (cmp == 0) {
+                        String actions1 = o1.getActions();
+                        String actions2 = o2.getActions();
+                        if (actions1 == null) {
+                            actions1 = "";
+                        }
+                        if (actions2 == null) {
+                            actions2 = "";
+                        }
+                        cmp = actions1.compareTo(actions2);
+                    }
+                }
+                return cmp;
+            }
+        });
+        
+        terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+        terminal.println(Verbosity.NORMAL, "@     WARNING: plugin requires additional permissions     @");
+        terminal.println(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
+        // print all permissions:
+        for (Permission permission : requested) {
+            terminal.println(Verbosity.NORMAL, "* %s", formatPermission(permission));
+        }
+        terminal.println(Verbosity.NORMAL, "See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html");
+        terminal.println(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks.");
+        if (!batch) {
+            terminal.println(Verbosity.NORMAL);
+            String text = terminal.readText("Continue with installation? [y/N]");
+            if (!text.equalsIgnoreCase("y")) {
+                throw new RuntimeException("installation aborted by user");
+            }
+        }
+    }
+    
+    /** Format permission type, name, and actions into a string */
+    static String formatPermission(Permission permission) {
+        StringBuilder sb = new StringBuilder();
+        
+        String clazz = null;
+        if (permission instanceof UnresolvedPermission) {
+            clazz = ((UnresolvedPermission) permission).getUnresolvedType();
+        } else {
+            clazz = permission.getClass().getName();
+        }
+        sb.append(clazz);
+        
+        String name = null;
+        if (permission instanceof UnresolvedPermission) {
+            name = ((UnresolvedPermission) permission).getUnresolvedName();
+        } else {
+            name = permission.getName();
+        }
+        if (name != null && name.length() > 0) {
+            sb.append(' ');
+            sb.append(name);
+        }
+        
+        String actions = null;
+        if (permission instanceof UnresolvedPermission) {
+            actions = ((UnresolvedPermission) permission).getUnresolvedActions();
+        } else {
+            actions = permission.getActions();
+        }
+        if (actions != null && actions.length() > 0) {
+            sb.append(' ');
+            sb.append(actions);
+        }
+        return sb.toString();
+    }
+    
+    /**
+     * Parses plugin policy into a set of permissions
+     */
+    static PermissionCollection parsePermissions(Terminal terminal, Path file, Path tmpDir) throws IOException {
+        // create a zero byte file for "comparison"
+        // this is necessary because the default policy impl automatically grants two permissions:
+        // 1. permission to exitVM (which we ignore)
+        // 2. read permission to the code itself (e.g. jar file of the code)
+
+        Path emptyPolicyFile = Files.createTempFile(tmpDir, "empty", "tmp");
+        final Policy emptyPolicy;
+        try {
+            emptyPolicy = Policy.getInstance("JavaPolicy", new URIParameter(emptyPolicyFile.toUri()));
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        PluginManager.tryToDeletePath(terminal, emptyPolicyFile);
+        
+        // parse the plugin's policy file into a set of permissions
+        final Policy policy;
+        try {
+            policy = Policy.getInstance("JavaPolicy", new URIParameter(file.toUri()));
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        PermissionCollection permissions = policy.getPermissions(PluginSecurity.class.getProtectionDomain());
+        // this method is supported with the specific implementation we use, but just check for safety.
+        if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
+            throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
+        }
+        PermissionCollection actualPermissions = new Permissions();
+        for (Permission permission : Collections.list(permissions.elements())) {
+            if (!emptyPolicy.implies(PluginSecurity.class.getProtectionDomain(), permission)) {
+                actualPermissions.add(permission);
+            }
+        }
+        actualPermissions.setReadOnly();
+        return actualPermissions;
+    }
+}

core/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -37,52 +37,6 @@ grant codeBase "${es.security.jar.lucene.core}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-//// Special plugin permissions:
-//// These are dangerous permissions only needed by special plugins that we don't
-//// want to grant in general. Some may be due to problems in third-party library code,
-//// others may just be more obscure integrations.
-
-grant codeBase "${es.security.plugin.repository-s3}" {
-  // needed because of problems in aws-sdk
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-};
-
-grant codeBase "${es.security.plugin.discovery-ec2}" {
-  // needed because of problems in aws-sdk
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-};
-
-grant codeBase "${es.security.plugin.discovery-gce}" {
-  // needed because of problems in discovery-gce
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-};
-
-grant codeBase "${es.security.plugin.lang-expression}" {
-  // needed to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
-};
-
-grant codeBase "${es.security.plugin.lang-groovy}" {
-  // needed to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
-  // needed by groovy engine
-  permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
-  // needed by GroovyScriptEngineService to close its classloader (why?)
-  permission java.lang.RuntimePermission "closeClassLoader";
-  // Allow executing groovy scripts with codesource of /untrusted
-  permission groovy.security.GroovyCodeSourcePermission "/untrusted";
-};
-
-grant codeBase "${es.security.plugin.lang-javascript}" {
-  // needed to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
-};
-
-grant codeBase "${es.security.plugin.lang-python}" {
-  // needed to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
-};
-
 //// test framework permissions.
 //// These are mock objects and test management that we allow test framework libs
 //// to provide on our behalf. But tests themselves cannot do this stuff!

core/src/main/resources/org/elasticsearch/plugins/plugin-install.help

@@ -61,3 +61,5 @@ OPTIONS
     -v,--verbose                 Verbose output
 
     -h,--help                    Shows this message
+    
+    -b,--batch                   Enable batch mode explicitly, automatic confirmation of security permissions

core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -25,15 +25,22 @@ import org.elasticsearch.bootstrap.ESPolicy;
 import org.elasticsearch.bootstrap.Security;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.FilePermission;
+import java.io.InputStream;
+import java.net.URI;
 import java.net.URL;
 import java.nio.file.Path;
+import java.security.Permission;
+import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
-import java.util.Map;
+import java.security.URIParameter;
+import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
+import java.util.Properties;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
 
@@ -117,33 +124,43 @@ public class BootstrapForTesting {
                 final Policy policy;
                 // if its a plugin with special permissions, we use a wrapper policy impl to try
                 // to simulate what happens with a real distribution
-                String artifact = System.getProperty("tests.artifact");
-                // in case we are running from the IDE:
-                if (artifact == null && System.getProperty("tests.maven") == null) {
-                    // look for plugin classname as a resource to determine what project we are.
-                    // while gross, this will work with any IDE.
-                    for (Map.Entry<String,String> kv : Security.SPECIAL_PLUGINS.entrySet()) {
-                        String resource = kv.getValue().replace('.', '/') + ".class";
-                        if (BootstrapForTesting.class.getClassLoader().getResource(resource) != null) {
-                            artifact = kv.getKey();
-                            break;
+                List<URL> pluginPolicies = Collections.list(BootstrapForTesting.class.getClassLoader().getResources(PluginInfo.ES_PLUGIN_POLICY));
+                if (!pluginPolicies.isEmpty()) {
+                    Permissions extra = new Permissions();
+                    for (URL url : pluginPolicies) {
+                        URI uri = url.toURI();
+                        Policy pluginPolicy = Policy.getInstance("JavaPolicy", new URIParameter(uri));
+                        PermissionCollection permissions = pluginPolicy.getPermissions(BootstrapForTesting.class.getProtectionDomain());
+                        // this method is supported with the specific implementation we use, but just check for safety.
+                        if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
+                            throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
+                        }
+                        for (Permission permission : Collections.list(permissions.elements())) {
+                            extra.add(permission);
                         }
                     }
-                }
-                String pluginProp = Security.getPluginProperty(artifact);
-                if (pluginProp != null) {
-                    policy = new MockPluginPolicy(perms, pluginProp);
+                    // TODO: try to get rid of this class now that the world is simpler?
+                    policy = new MockPluginPolicy(perms, extra);
                 } else {
-                    policy = new ESPolicy(perms);
+                    policy = new ESPolicy(perms, Collections.emptyMap());
                 }
                 Policy.setPolicy(policy);
                 System.setSecurityManager(new TestSecurityManager());
                 Security.selfTest();
 
-                if (pluginProp != null) {
-                    // initialize the plugin class, in case it has one-time hacks (unit tests often won't do this)
-                    String clazz = Security.getPluginClass(artifact);
-                    Class.forName(clazz);
+                // guarantee plugin classes are initialized first, in case they have one-time hacks.
+                // this just makes unit testing more realistic
+                for (URL url : Collections.list(BootstrapForTesting.class.getClassLoader().getResources(PluginInfo.ES_PLUGIN_PROPERTIES))) {
+                    Properties properties = new Properties();
+                    try (InputStream stream = url.openStream()) {
+                        properties.load(stream);
+                    }
+                    if (Boolean.parseBoolean(properties.getProperty("jvm"))) {
+                        String clazz = properties.getProperty("classname");
+                        if (clazz != null) {
+                            Class.forName(clazz);
+                        }
+                    }
                 }
             } catch (Exception e) {
                 throw new RuntimeException("unable to install test security manager", e);

core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java

@@ -32,6 +32,7 @@ import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
 import java.security.cert.Certificate;
+import java.util.Collections;
 
 /** 
  * Tests for ESPolicy
@@ -54,7 +55,7 @@ public class ESPolicyTests extends ESTestCase {
         Permission all = new AllPermission();
         PermissionCollection allCollection = all.newPermissionCollection();
         allCollection.add(all);
-        ESPolicy policy = new ESPolicy(allCollection);
+        ESPolicy policy = new ESPolicy(allCollection, Collections.emptyMap());
         // restrict ourselves to NoPermission
         PermissionCollection noPermissions = new Permissions();
         assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read")));
@@ -68,7 +69,7 @@ public class ESPolicyTests extends ESTestCase {
     public void testNullLocation() throws Exception {
         assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         PermissionCollection noPermissions = new Permissions();
-        ESPolicy policy = new ESPolicy(noPermissions);
+        ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap());
         assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[])null), noPermissions), new FilePermission("foo", "read")));
     }
 

core/src/test/java/org/elasticsearch/bootstrap/MockPluginPolicy.java

@@ -29,7 +29,6 @@ import java.net.URL;
 import java.security.CodeSource;
 import java.security.Permission;
 import java.security.PermissionCollection;
-import java.security.Permissions;
 import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.security.cert.Certificate;
@@ -58,33 +57,11 @@ final class MockPluginPolicy extends Policy {
      * adding the extra plugin permissions from {@code insecurePluginProp} to
      * all code except test classes.
      */
-    MockPluginPolicy(Permissions permissions, String insecurePluginProp) throws Exception {
+    MockPluginPolicy(PermissionCollection standard, PermissionCollection extra) throws Exception {
         // the hack begins!
 
-        // parse whole policy file, with and without the substitution, compute the delta
-        standardPolicy = new ESPolicy(permissions);
-
-        URL bogus = new URL("file:/bogus"); // its "any old codebase" this time: generic permissions
-        PermissionCollection smallPermissions = standardPolicy.template.getPermissions(new CodeSource(bogus, (Certificate[])null)); 
-        Set<Permission> small = new HashSet<>(Collections.list(smallPermissions.elements()));
-
-        // set the URL for the property substitution, this time it will also have special permissions
-        System.setProperty(insecurePluginProp, bogus.toString());
-        ESPolicy biggerPolicy = new ESPolicy(permissions);
-        System.clearProperty(insecurePluginProp);
-        PermissionCollection bigPermissions = biggerPolicy.template.getPermissions(new CodeSource(bogus, (Certificate[])null));
-        Set<Permission> big = new HashSet<>(Collections.list(bigPermissions.elements()));
-
-        // compute delta to remove all the generic permissions
-        // we want equals() vs implies() for this check, in case we need 
-        // to pass along any UnresolvedPermission to the plugin
-        big.removeAll(small);
-
-        // build collection of the special permissions for easy checking
-        extraPermissions = new Permissions();
-        for (Permission p : big) {
-            extraPermissions.add(p);
-        }
+        this.standardPolicy = new ESPolicy(standard, Collections.emptyMap());
+        this.extraPermissions = extra;
 
         excludedSources = new HashSet<CodeSource>();
         // exclude some obvious places
@@ -101,7 +78,7 @@ final class MockPluginPolicy extends Policy {
         // scripts
         excludedSources.add(new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[])null));
 
-        Loggers.getLogger(getClass()).debug("Apply permissions [{}] excluding codebases [{}]", extraPermissions, excludedSources);
+        Loggers.getLogger(getClass()).debug("Apply extra permissions [{}] excluding codebases [{}]", extraPermissions, excludedSources);
     }
 
     @Override

core/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java

@@ -84,7 +84,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
             Files.setPosixFilePermissions(binPath, PosixFilePermissions.fromString("---------"));
 
             PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
 
             fail("Expected IOException but did not happen");
         } catch (IOException e) {
@@ -115,7 +115,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
             Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("---------"));
 
             PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
 
             fail("Expected IOException but did not happen, terminal output was " + terminal.getTerminalOutput());
         } catch (IOException e) {
@@ -148,7 +148,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
             Files.setPosixFilePermissions(binPath, PosixFilePermissions.fromString("---------"));
 
             PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
         } finally {
             Files.setPosixFilePermissions(binPath, PosixFilePermissions.fromString("rwxrwxrwx"));
             Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("rwxrwxrwx"));
@@ -168,7 +168,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
             Files.setPosixFilePermissions(environment.pluginsFile(), PosixFilePermissions.fromString("---------"));
             PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
             try {
-                pluginManager.downloadAndExtract(pluginName, terminal);
+                pluginManager.downloadAndExtract(pluginName, terminal, true);
                 fail("Expected IOException due to read-only plugins/ directory");
             } catch (IOException e) {
                 assertFileNotExists(environment.binFile().resolve(pluginName));
@@ -200,7 +200,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
         try {
             Files.setPosixFilePermissions(backupConfigFile, PosixFilePermissions.fromString("---------"));
 
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
 
             if (pluginContainsExecutables) {
                 assertDirectoryExists(environment.binFile().resolve(pluginName));
@@ -227,7 +227,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
         PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
 
         try {
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
             fail("Expected plugin installation to fail, but didnt");
         } catch (IOException e) {
             assertFileExists(environment.configFile().resolve(pluginName));
@@ -246,7 +246,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
         PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
 
         try {
-            pluginManager.downloadAndExtract(pluginName, terminal);
+            pluginManager.downloadAndExtract(pluginName, terminal, true);
             fail("Expected plugin installation to fail, but didnt");
         } catch (IOException e) {
             assertFileExists(environment.binFile().resolve(pluginName));
@@ -259,7 +259,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
         assumeTrue("File system does not support permissions, skipping", supportsPermissions);
         URL pluginUrl = createPlugin(false, true);
         PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
-        pluginManager.downloadAndExtract(pluginName, terminal);
+        pluginManager.downloadAndExtract(pluginName, terminal, true);
         PosixFileAttributes parentFileAttributes = Files.getFileAttributeView(environment.configFile(), PosixFileAttributeView.class).readAttributes();
         Path configPath = environment.configFile().resolve(pluginName);
         PosixFileAttributes pluginConfigDirAttributes = Files.getFileAttributeView(configPath, PosixFileAttributeView.class).readAttributes();
@@ -288,7 +288,7 @@ public class PluginManagerPermissionTests extends ESTestCase {
         assumeTrue("File system does not support permissions, skipping", supportsPermissions);
         URL pluginUrl = createPlugin(true, false);
         PluginManager pluginManager = new PluginManager(environment, pluginUrl, PluginManager.OutputMode.VERBOSE, TimeValue.timeValueSeconds(10));
-        pluginManager.downloadAndExtract(pluginName, terminal);
+        pluginManager.downloadAndExtract(pluginName, terminal, true);
         PosixFileAttributes parentFileAttributes = Files.getFileAttributeView(environment.binFile(), PosixFileAttributeView.class).readAttributes();
         Path binPath = environment.binFile().resolve(pluginName);
         PosixFileAttributes pluginBinDirAttributes = Files.getFileAttributeView(binPath, PosixFileAttributeView.class).readAttributes();

core/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.file.Path;
+import java.security.Permission;
+import java.security.PermissionCollection;
+import java.security.Permissions;
+import java.util.Collections;
+import java.util.List;
+
+/** Tests plugin manager security check */
+public class PluginSecurityTests extends ESTestCase {
+    
+    /** Test that we can parse the set of permissions correctly for a simple policy */
+    public void testParsePermissions() throws Exception {
+        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        Path scratch = createTempDir();
+        Path testFile = this.getDataPath("security/simple-plugin-security.policy");
+        Permissions expected = new Permissions();
+        expected.add(new RuntimePermission("queuePrintJob"));
+        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        assertEquals(expected, actual);
+    }
+    
+    /** Test that we can parse the set of permissions correctly for a complex policy */
+    public void testParseTwoPermissions() throws Exception {
+        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        Path scratch = createTempDir();
+        Path testFile = this.getDataPath("security/complex-plugin-security.policy");
+        Permissions expected = new Permissions();
+        expected.add(new RuntimePermission("getClassLoader"));
+        expected.add(new RuntimePermission("closeClassLoader"));
+        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        assertEquals(expected, actual);
+    }
+    
+    /** Test that we can format some simple permissions properly */
+    public void testFormatSimplePermission() throws Exception {
+        assertEquals("java.lang.RuntimePermission queuePrintJob", PluginSecurity.formatPermission(new RuntimePermission("queuePrintJob")));
+    }
+    
+    /** Test that we can format an unresolved permission properly */
+    public void testFormatUnresolvedPermission() throws Exception {
+        assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
+        Path scratch = createTempDir();
+        Path testFile = this.getDataPath("security/unresolved-plugin-security.policy");
+        PermissionCollection actual = PluginSecurity.parsePermissions(Terminal.DEFAULT, testFile, scratch);
+        List<Permission> permissions = Collections.list(actual.elements());
+        assertEquals(1, permissions.size());
+        assertEquals("org.fake.FakePermission fakeName", PluginSecurity.formatPermission(permissions.get(0)));
+    }
+    
+    /** no guaranteed equals on these classes, we assert they contain the same set */
+    private void assertEquals(PermissionCollection expected, PermissionCollection actual) {
+        assertEquals(asSet(Collections.list(expected.elements())), asSet(Collections.list(actual.elements())));
+    }
+}

core/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy

@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to cause problems
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "closeClassLoader";
+};

core/src/test/resources/org/elasticsearch/plugins/security/simple-plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to waste paper
+  permission java.lang.RuntimePermission "queuePrintJob";
+};

core/src/test/resources/org/elasticsearch/plugins/security/unresolved-plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // an unresolved permission
+  permission org.fake.FakePermission "fakeName";
+};

dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml

@@ -5,6 +5,16 @@
         <format>zip</format>
     </formats>
     <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/src/main/plugin-metadata</directory>
+            <includes>
+              <include>plugin-security.policy</include>
+            </includes>
+            <outputDirectory></outputDirectory>
+            <filtered>false</filtered>
+        </fileSet>
+    </fileSets>
     <files>
         <file>
             <source>${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties</source>

docs/plugins/authors.asciidoc

@@ -119,3 +119,40 @@ You may also load your plugin within the test framework for integration tests.
 Read more in {ref}/integration-tests.html#changing-node-configuration[Changing Node Configuration].
 
 
+[float]
+=== Java Security permissions
+
+Some plugins may need additional security permissions. A plugin can include
+the optional `plugin-security.policy` file containing `grant` statements for 
+additional permissions. Any additional permissions will be displayed to the user 
+with a large warning, and they will have to confirm them when installing the 
+plugin interactively. So if possible, it is best to avoid requesting any
+spurious permissions!
+
+If you are using the elasticsearch Maven build system, place this file in
+`src/main/plugin-metadata` and it will be applied during unit tests as well.
+
+Keep in mind that the Java security model is stack-based, and the additional
+permissions will only be granted to the jars in your plugin, so you will have
+write proper security code around operations requiring elevated privileges.
+It is recommended to add a check to prevent unprivileged code (such as scripts)
+from gaining escalated permissions. For example:
+
+[source,java]
+--------------------------------------------------
+// ES permission you should check before doPrivileged() blocks
+import org.elasticsearch.SpecialPermission;
+
+SecurityManager sm = System.getSecurityManager();
+if (sm != null) {
+  // unprivileged code such as scripts do not have SpecialPermission
+  sm.checkPermission(new SpecialPermission());
+}
+AccessController.doPrivileged(
+  // sensitive operation
+);
+--------------------------------------------------
+
+See http://www.oracle.com/technetwork/java/seccodeguide-139067.html[Secure Coding Guidelines for Java SE]
+for more information.
+

plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed because of problems in aws-sdk
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};

plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed because of problems in gce
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};

plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to generate runtime classes
+  permission java.lang.RuntimePermission "createClassLoader";
+};

plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to generate runtime classes
+  permission java.lang.RuntimePermission "createClassLoader";
+  // needed by groovy engine
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
+  // needed by GroovyScriptEngineService to close its classloader (why?)
+  permission java.lang.RuntimePermission "closeClassLoader";
+  // Allow executing groovy scripts with codesource of /untrusted
+  permission groovy.security.GroovyCodeSourcePermission "/untrusted";
+};

plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to generate runtime classes
+  permission java.lang.RuntimePermission "createClassLoader";
+};

plugins/lang-python/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed to generate runtime classes
+  permission java.lang.RuntimePermission "createClassLoader";
+};

plugins/pom.xml

@@ -246,9 +246,41 @@
                 <directory>${elasticsearch.tools.directory}/shared-test-resources</directory>
                 <filtering>false</filtering>
             </testResource>
+            <!-- plugin metadata as a test resource -->
+            <testResource>
+                <directory>${basedir}/target/metadata-test-resources</directory>
+                <filtering>false</filtering>
+            </testResource>
         </testResources>
 
         <plugins>
+           <!-- we don't have a proper .zip plugin structure for tests, but we need the metadata as test resource -->
+           <plugin>
+              <artifactId>maven-resources-plugin</artifactId>
+              <executions>
+                  <execution>
+                      <id>copy-resources</id>
+                      <!-- process-resources makes more sense, but is not done by e.g. mvn eclipse:eclipse! -->
+                      <phase>generate-resources</phase>
+                      <goals>
+                          <goal>copy-resources</goal>
+                      </goals>
+                      <configuration>
+                          <outputDirectory>${basedir}/target/metadata-test-resources</outputDirectory>
+                          <resources>
+                              <resource>
+                                  <directory>src/main/plugin-metadata</directory>
+                                  <filtering>false</filtering>
+                              </resource>
+                              <resource>
+                                  <directory>${elasticsearch.tools.directory}/plugin-metadata</directory>
+                                  <filtering>true</filtering>
+                              </resource>
+                          </resources>
+                      </configuration>
+                  </execution>
+              </executions>
+           </plugin>
            <!-- integration tests -->
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>

plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // needed because of problems in aws-sdk
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};