Add precommit task for detecting split packages (#73784)

Modularization of the JDK has been ongoing for several years. Recently
in Java 16 the JDK began enforcing module boundaries by default. While
Elasticsearch does not yet use the module system directly, there are
some side effects even for those projects not modularized (eg #73517).
Before we can even begin to think about how to modularize, we must
Prepare The Way by enforcing packages only exist in a single jar file,
since the module system does not allow packages to coexist in multiple
modules.

This commit adds a precommit check to the build which detects split
packages. The expectation is that we will add the existing split
packages to the ignore list so that any new classes will not exacerbate
the problem, and the work to cleanup these split packages can be
parallelized.

relates #73525

benchmarks/build.gradle

@@ -8,7 +8,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 apply plugin: 'application'
 mainClassName = 'org.openjdk.jmh.Main'
 
@@ -61,22 +61,6 @@ tasks.named("run").configure {
   dependsOn "copyExpression", "copyPainless"
 }
 
-// classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-disableTasks('forbiddenApisMain')
-
-// No licenses for our benchmark deps (we don't ship benchmarks)
-tasks.named("dependencyLicenses").configure { enabled = false }
-tasks.named("dependenciesInfo").configure {  enabled = false }
-tasks.named("dependenciesGraph").configure {  enabled = false }
-
-
-tasks.named("thirdPartyAudit").configure {
-  ignoreViolations(
-          // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)
-          'org.openjdk.jmh.util.Utils'
-  )
-}
-
 spotless {
   java {
     // IDEs can sometimes run annotation processors that leave files in

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java

@@ -11,6 +11,8 @@ package org.elasticsearch.gradle.internal;
 import org.codehaus.groovy.runtime.DefaultGroovyMethods;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
+import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
+import org.elasticsearch.gradle.internal.precommit.SplitPackagesAuditPrecommitPlugin;
 import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.Plugin;
@@ -42,6 +44,7 @@ public class BuildPlugin implements Plugin<Project> {
         project.getPluginManager().apply(DependenciesGraphPlugin.class);
 
         InternalPrecommitTasks.create(project, true);
+        project.getPluginManager().apply(SplitPackagesAuditPrecommitPlugin.class);
     }
 
     public static void configureLicenseAndNotice(final Project project) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskProvider;
+
+public class SplitPackagesAuditPrecommitPlugin extends PrecommitPlugin {
+    public static final String TASK_NAME = "splitPackagesAudit";
+
+    @Override
+    public TaskProvider<? extends Task> createTask(Project project) {
+        TaskProvider<SplitPackagesAuditTask> task = project.getTasks().register(TASK_NAME, SplitPackagesAuditTask.class);
+        task.configure(t -> {
+            t.setClasspath(project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME));
+            SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            t.getSrcDirs().set(project.provider(() -> mainSourceSet.getAllSource().getSrcDirs()));
+        });
+        return task;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java

@@ -0,0 +1,328 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.SetProperty;
+import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.workers.WorkAction;
+import org.gradle.workers.WorkParameters;
+import org.gradle.workers.WorkerExecutor;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * Checks for split packages with dependencies. These are not allowed in a future modularized world.
+ */
+public class SplitPackagesAuditTask extends DefaultTask {
+
+    private static final Logger LOGGER = Logging.getLogger(SplitPackagesAuditTask.class);
+
+    private final WorkerExecutor workerExecutor;
+    private FileCollection classpath;
+    private final SetProperty<File> srcDirs;
+    private final SetProperty<String> ignoreClasses;
+    private final RegularFileProperty markerFile;
+
+    @Inject
+    public SplitPackagesAuditTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory) {
+        this.workerExecutor = workerExecutor;
+        this.srcDirs = objectFactory.setProperty(File.class);
+        this.ignoreClasses = objectFactory.setProperty(String.class);
+        this.markerFile = objectFactory.fileProperty();
+
+        this.markerFile.set(getProject().getLayout().getBuildDirectory().file("markers/" + this.getName() + ".marker"));
+    }
+
+    @TaskAction
+    public void auditSplitPackages() {
+        workerExecutor.noIsolation().submit(SplitPackagesAuditAction.class, params -> {
+            params.getProjectPath().set(getProject().getPath());
+            params.getProjectBuildDirs().set(getProjectBuildDirs());
+            params.getClasspath().from(classpath);
+            params.getSrcDirs().set(srcDirs);
+            params.getIgnoreClasses().set(ignoreClasses);
+            params.getMarkerFile().set(markerFile);
+        });
+    }
+
+    private Map<File, String> getProjectBuildDirs() {
+        // while this is done in every project, it should be cheap to calculate
+        Map<File, String> buildDirs = new HashMap<>();
+        for (Project project : getProject().getRootProject().getAllprojects()) {
+            buildDirs.put(project.getBuildDir(), project.getPath());
+        }
+        return buildDirs;
+    }
+
+    @CompileClasspath
+    public FileCollection getClasspath() {
+        return classpath.filter(File::exists);
+    }
+
+    public void setClasspath(FileCollection classpath) {
+        this.classpath = classpath;
+    }
+
+    @InputFiles
+    @SkipWhenEmpty
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public SetProperty<File> getSrcDirs() {
+        return srcDirs;
+    }
+
+    @Input
+    public SetProperty<String> getIgnoreClasses() {
+        return ignoreClasses;
+    }
+
+    /**
+     * Add classes that exist in split packages but should be ignored.
+     */
+    public void ignoreClasses(String... classes) {
+        for (String classname : classes) {
+            ignoreClasses.add(classname);
+        }
+    }
+
+    @OutputFile
+    public RegularFileProperty getMarkerFile() {
+        return markerFile;
+    }
+
+    public abstract static class SplitPackagesAuditAction implements WorkAction<Parameters> {
+        @Override
+        public void execute() {
+            final Parameters parameters = getParameters();
+            final String projectPath = parameters.getProjectPath().get();
+
+            // First determine all the packages that exist in the dependencies. There might be
+            // split packages across the dependencies, which is "ok", in that we don't care
+            // about it for the purpose of this project, that split will be detected in
+            // the other project
+            Map<String, List<File>> dependencyPackages = getDependencyPackages();
+
+            // Next read each of the source directories and find if we define any package directories
+            // that match those in our dependencies.
+            Map<String, Set<String>> splitPackages = findSplitPackages(dependencyPackages.keySet());
+
+            // Then filter out any known split packages/classes that we want to ignore.
+            filterSplitPackages(splitPackages);
+
+            // Finally, print out (and fail) if we have any split packages
+            for (var entry : splitPackages.entrySet()) {
+                String packageName = entry.getKey();
+                List<File> deps = dependencyPackages.get(packageName);
+                List<String> msg = new ArrayList<>();
+                msg.add("Project " + projectPath + " defines classes in package " + packageName + " exposed by dependencies");
+                msg.add("  Dependencies:");
+                deps.forEach(f -> msg.add("    " + formatDependency(f)));
+                msg.add("  Classes:");
+                entry.getValue().forEach(c -> msg.add("    '" + c + "',"));
+                LOGGER.error(String.join(System.lineSeparator(), msg));
+            }
+            if (splitPackages.isEmpty() == false) {
+                throw new GradleException("Verification failed: Split packages found! See errors above for details.\n" +
+                    "DO NOT ADD THESE SPLIT PACKAGES TO THE IGNORE LIST! Choose a new package name for the classes added.");
+            }
+
+            try {
+                Files.write(parameters.getMarkerFile().getAsFile().get().toPath(), new byte[] {}, StandardOpenOption.CREATE);
+            } catch (IOException e) {
+                throw new RuntimeException("Failed to create marker file", e);
+            }
+        }
+
+        private Map<String, List<File>> getDependencyPackages() {
+            Map<String, List<File>> packages = new HashMap<>();
+            for (File classpathElement : getParameters().getClasspath().getFiles()) {
+                for (String packageName : readPackages(classpathElement)) {
+                    packages.computeIfAbsent(packageName, k -> new ArrayList<>()).add(classpathElement);
+                }
+            }
+            if (LOGGER.isInfoEnabled()) {
+                List<String> msg = new ArrayList<>();
+                msg.add("Packages from dependencies:");
+                packages.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(e -> msg.add("  -" + e.getKey() + " -> " + e.getValue()));
+                LOGGER.info(String.join(System.lineSeparator(), msg));
+            }
+            return packages;
+        }
+
+        private Map<String, Set<String>> findSplitPackages(Set<String> dependencyPackages) {
+            Map<String, Set<String>> splitPackages = new HashMap<>();
+            for (File srcDir : getParameters().getSrcDirs().get()) {
+                try {
+                    walkJavaFiles(srcDir.toPath(), ".java", path -> {
+                        String packageName = getPackageName(path);
+                        String className = path.subpath(path.getNameCount() - 1, path.getNameCount()).toString();
+                        className = className.substring(0, className.length() - ".java".length());
+                        LOGGER.info("Inspecting " + path + System.lineSeparator()
+                            + "  package: " + packageName + System.lineSeparator()
+                            + "  class: " + className);
+                        if (dependencyPackages.contains(packageName)) {
+                            splitPackages.computeIfAbsent(packageName, k -> new TreeSet<>()).add(packageName + "." + className);
+                        }
+                    });
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+            if (LOGGER.isInfoEnabled()) {
+                List<String> msg = new ArrayList<>();
+                msg.add("Split packages:");
+                splitPackages.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(e -> msg.add("  -" + e.getKey() + " -> " + e.getValue()));
+                LOGGER.info(String.join(System.lineSeparator(), msg));
+            }
+            return splitPackages;
+        }
+
+        private void filterSplitPackages(Map<String, Set<String>> splitPackages) {
+            String lastPackageName = null;
+            Set<String> currentClasses = null;
+            boolean filterErrorsFound = false;
+            for (String fqcn : getParameters().getIgnoreClasses().get().stream().sorted().collect(Collectors.toList())) {
+                int lastDot = fqcn.lastIndexOf('.');
+                if (lastDot == -1) {
+                    LOGGER.error("Missing package in classname in split package ignores: " + fqcn);
+                    filterErrorsFound = true;
+                    continue;
+                }
+                String packageName = fqcn.substring(0, lastDot);
+                String className = fqcn.substring(lastDot + 1);
+                LOGGER.info("IGNORING package: " + packageName + ", class: " + className);
+                if (packageName.equals(lastPackageName) == false) {
+                    currentClasses = splitPackages.get(packageName);
+                    lastPackageName = packageName;
+                }
+
+                if (currentClasses == null) {
+                    LOGGER.error("Package is not split: " + fqcn);
+                    filterErrorsFound = true;
+                } else {
+                    if (className.equals("*")) {
+                        currentClasses.clear();
+                    } else if (currentClasses.remove(fqcn) == false) {
+                        LOGGER.error("Class does not exist: " + fqcn);
+                        filterErrorsFound = true;
+                    }
+                    // cleanup if we have ignored the last class in a package
+                    if (currentClasses.isEmpty()) {
+                        splitPackages.remove(packageName);
+                    }
+                }
+            }
+            if (filterErrorsFound) {
+                throw new GradleException("Unnecessary split package ignores found");
+            }
+        }
+
+        // TODO: want to read packages the same for src dirs and jars, but src dirs we also want the files in the src package dir
+        private static Set<String> readPackages(File classpathElement) {
+            Set<String> packages = new HashSet<>();
+            Consumer<Path> addClassPackage = p -> packages.add(getPackageName(p));
+
+            try {
+                if (classpathElement.isDirectory()) {
+                    walkJavaFiles(classpathElement.toPath(), ".class", addClassPackage);
+                } else if (classpathElement.getName().endsWith(".jar")) {
+                    try (FileSystem jar = FileSystems.newFileSystem(classpathElement.toPath(), Map.of())) {
+                        for (Path root : jar.getRootDirectories()) {
+                            walkJavaFiles(root, ".class", addClassPackage);
+                        }
+                    }
+                } else {
+                    throw new GradleException("Unsupported classpath element: " + classpathElement);
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+
+            return packages;
+        }
+
+        private static void walkJavaFiles(Path root, String suffix, Consumer<Path> classConsumer) throws IOException {
+            if (Files.exists(root) == false) {
+                return;
+            }
+            Files.walk(root)
+                .filter(p -> p.toString().endsWith(suffix))
+                .map(root::relativize)
+                .filter(p -> p.getNameCount() > 1) // module-info or other things without a package can be skipped
+                .filter(p -> p.toString().startsWith("META-INF") == false)
+                .forEach(classConsumer);
+        }
+
+        private static String getPackageName(Path path) {
+            List<String> subpackages = new ArrayList<>();
+            for (int i = 0; i < path.getNameCount() - 1; ++i) {
+                subpackages.add(path.getName(i).toString());
+            }
+            return String.join(".", subpackages);
+        }
+
+        private String formatDependency(File dependencyFile) {
+            if (dependencyFile.isDirectory()) {
+                while (dependencyFile.getName().equals("build") == false) {
+                    dependencyFile = dependencyFile.getParentFile();
+                }
+                String projectName = getParameters().getProjectBuildDirs().get().get(dependencyFile);
+                if (projectName == null) {
+                    throw new IllegalStateException("Build directory unknown to gradle: " + dependencyFile);
+                }
+                return "project " + projectName;
+            }
+            return dependencyFile.getName(); // just the jar filename
+        }
+    }
+
+    interface Parameters extends WorkParameters {
+        Property<String> getProjectPath();
+        MapProperty<File, String> getProjectBuildDirs();
+        ConfigurableFileCollection getClasspath();
+        SetProperty<File> getSrcDirs();
+        SetProperty<String> getIgnoreClasses();
+        RegularFileProperty getMarkerFile();
+    }
+}

client/rest-high-level/build.gradle

@@ -124,3 +124,8 @@ testClusters.all {
   setting 'xpack.searchable.snapshot.shared_cache.size', '1mb'
   setting 'xpack.searchable.snapshot.shared_cache.region_size', '16kb'
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // the client package should be owned by the client, but server has some classes there too
+  ignoreClasses 'org.elasticsearch.client.*'
+}

distribution/tools/keystore-cli/build.gradle

@@ -15,3 +15,17 @@ dependencies {
   testImplementation "com.google.jimfs:jimfs:${versions.jimfs}"
   testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}"
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // common.settings is owned by server, this should be keystore specifici
+  ignoreClasses 'org.elasticsearch.common.settings.AddFileKeyStoreCommand',
+    'org.elasticsearch.common.settings.AddStringKeyStoreCommand',
+    'org.elasticsearch.common.settings.BaseKeyStoreCommand',
+    'org.elasticsearch.common.settings.ChangeKeyStorePasswordCommand',
+    'org.elasticsearch.common.settings.CreateKeyStoreCommand',
+    'org.elasticsearch.common.settings.HasPasswordKeyStoreCommand',
+    'org.elasticsearch.common.settings.KeyStoreCli',
+    'org.elasticsearch.common.settings.ListKeyStoreCommand',
+    'org.elasticsearch.common.settings.RemoveSettingKeyStoreCommand',
+    'org.elasticsearch.common.settings.UpgradeKeyStoreCommand'
+}

distribution/tools/plugin-cli/build.gradle

@@ -57,3 +57,12 @@ tasks.named("thirdPartyAudit").configure {
           'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator$2'
   )
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // o.e.plugins is owned by server, these shouldb be renamed to plugincli
+  ignoreClasses 'org.elasticsearch.plugins.InstallPluginCommand',
+    'org.elasticsearch.plugins.ListPluginsCommand',
+    'org.elasticsearch.plugins.PluginCli',
+    'org.elasticsearch.plugins.ProgressInputStream',
+    'org.elasticsearch.plugins.RemovePluginCommand'
+}

modules/aggs-matrix-stats/build.gradle

@@ -18,3 +18,12 @@ restResources {
     include '_common', 'indices', 'cluster', 'index', 'search', 'nodes'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // search should be owned by server, this should be renamed to aggsmatrix
+  ignoreClasses 'org.elasticsearch.search.aggregations.support.ArrayValuesSource',
+    'org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregationBuilder',
+    'org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregatorFactory',
+    'org.elasticsearch.search.aggregations.support.ArrayValuesSourceParser',
+    'org.elasticsearch.search.aggregations.MatrixStatsAggregationBuilders'
+}

modules/mapper-extras/build.gradle

@@ -19,3 +19,19 @@ restResources {
     include '_common', 'cluster', 'field_caps', 'nodes', 'indices', 'index', 'search', 'get'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // index is owned by server, move these to mapper specific packages
+  ignoreClasses 'org.elasticsearch.index.query.RankFeatureQueryBuilder',
+    'org.elasticsearch.index.query.RankFeatureQueryBuilders',
+    'org.elasticsearch.index.query.SourceConfirmedTextQuery',
+    'org.elasticsearch.index.query.SourceIntervalsSource',
+    'org.elasticsearch.index.mapper.MapperExtrasPlugin',
+    'org.elasticsearch.index.mapper.MatchOnlyTextFieldMapper',
+    'org.elasticsearch.index.mapper.RankFeatureFieldMapper',
+    'org.elasticsearch.index.mapper.RankFeatureMetaFieldMapper',
+    'org.elasticsearch.index.mapper.RankFeaturesFieldMapper',
+    'org.elasticsearch.index.mapper.ScaledFloatFieldMapper',
+    'org.elasticsearch.index.mapper.SearchAsYouTypeFieldMapper',
+    'org.elasticsearch.index.mapper.TokenCountFieldMapper'
+}

modules/reindex/build.gradle

@@ -175,3 +175,29 @@ tasks.named("yamlRestCompatTest").configure {
     'update_by_query/20_validation/update_by_query without source gives useful error message'
   ].join(',')
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // index is owned by server, move these to reindex specific
+  ignoreClasses 'org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction',
+    'org.elasticsearch.index.reindex.AbstractBaseReindexRestHandler',
+    'org.elasticsearch.index.reindex.AbstractBulkByQueryRestHandler',
+    'org.elasticsearch.index.reindex.AsyncDeleteByQueryAction',
+    'org.elasticsearch.index.reindex.BulkByScrollParallelizationHelper',
+    'org.elasticsearch.index.reindex.BulkIndexByScrollResponseContentListener',
+    'org.elasticsearch.index.reindex.ReindexPlugin',
+    'org.elasticsearch.index.reindex.ReindexSslConfig',
+    'org.elasticsearch.index.reindex.ReindexValidator',
+    'org.elasticsearch.index.reindex.Reindexer',
+    'org.elasticsearch.index.reindex.RestDeleteByQueryAction',
+    'org.elasticsearch.index.reindex.RestReindexAction',
+    'org.elasticsearch.index.reindex.RestRethrottleAction',
+    'org.elasticsearch.index.reindex.RestUpdateByQueryAction',
+    'org.elasticsearch.index.reindex.RethrottleAction',
+    'org.elasticsearch.index.reindex.RethrottleRequest',
+    'org.elasticsearch.index.reindex.RethrottleRequestBuilder',
+    'org.elasticsearch.index.reindex.TransportDeleteByQueryAction',
+    'org.elasticsearch.index.reindex.TransportReindexAction',
+    'org.elasticsearch.index.reindex.TransportRethrottleAction',
+    'org.elasticsearch.index.reindex.TransportUpdateByQueryAction',
+    'org.elasticsearch.index.reindex.package-info'
+}

modules/transport-netty4/build.gradle

@@ -218,3 +218,14 @@ tasks.named("thirdPartyAudit").configure {
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator'
   )
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // transport is owned by server, these should be renamed to netty specific
+  ignoreClasses 'org.elasticsearch.transport.CopyBytesServerSocketChannel',
+    'org.elasticsearch.transport.CopyBytesSocketChannel',
+    'org.elasticsearch.transport.Netty4NioSocketChannel',
+    'org.elasticsearch.transport.Netty4Plugin',
+    'org.elasticsearch.transport.NettyAllocator',
+    'org.elasticsearch.transport.NettyByteBufSizer',
+    'org.elasticsearch.transport.SharedGroupFactory'
+}

plugins/analysis-icu/build.gradle

@@ -42,3 +42,17 @@ tasks.named("yamlRestCompatTest").configure {
     'analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter'
   ].join(',')
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // o.e.i.analysis is owned by server. Move these to an icu package
+  ignoreClasses 'org.elasticsearch.index.analysis.ICUCollationKeyFilter',
+    'org.elasticsearch.index.analysis.IcuAnalyzerProvider',
+    'org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory',
+    'org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory',
+    'org.elasticsearch.index.analysis.IcuNormalizerCharFilterFactory',
+    'org.elasticsearch.index.analysis.IcuNormalizerTokenFilterFactory',
+    'org.elasticsearch.index.analysis.IcuTokenizerFactory',
+    'org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory',
+    'org.elasticsearch.index.analysis.IndexableBinaryStringTools',
+    'org.elasticsearch.index.mapper.ICUCollationKeywordFieldMapper'
+}

plugins/analysis-kuromoji/build.gradle

@@ -26,3 +26,15 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
 
+tasks.named('splitPackagesAudit').configure {
+    // o.e.i.analysis is owned by server, these should be moved to a kuromoji package
+  ignoreClasses 'org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiAnalyzerProvider',
+    'org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiIterationMarkCharFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiKatakanaStemmerFactory',
+    'org.elasticsearch.index.analysis.KuromojiNumberFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory',
+    'org.elasticsearch.index.analysis.KuromojiTokenizerFactory'
+}

plugins/analysis-nori/build.gradle

@@ -26,3 +26,12 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // Lucene packages should be owned by Lucene!
+  ignoreClasses 'org.elasticsearch.index.analysis.NoriAnalyzerProvider',
+    'org.elasticsearch.index.analysis.NoriNumberFilterFactory',
+    'org.elasticsearch.index.analysis.NoriPartOfSpeechStopFilterFactory',
+    'org.elasticsearch.index.analysis.NoriReadingFormFilterFactory',
+    'org.elasticsearch.index.analysis.NoriTokenizerFactory'
+}

plugins/analysis-phonetic/build.gradle

@@ -27,3 +27,8 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // Lucene packages should be owned by Lucene!
+  ignoreClasses 'org.elasticsearch.index.analysis.PhoneticTokenFilterFactory'
+}

plugins/analysis-smartcn/build.gradle

@@ -27,3 +27,10 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
 
+tasks.named('splitPackagesAudit').configure {
+    // Lucene packages should be owned by Lucene!
+  ignoreClasses 'org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider',
+    'org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory',
+    'org.elasticsearch.index.analysis.SmartChineseStopTokenFilterFactory',
+    'org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory' 
+}

plugins/analysis-ukrainian/build.gradle

@@ -37,3 +37,9 @@ tasks.named("thirdPartyAudit").configure {
           'morfologik.stemming.polish.PolishStemmer'
   )
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // Lucene packages should be owned by Lucene!
+  ignoreClasses 'org.elasticsearch.index.analysis.UkrainianAnalyzerProvider',
+    'org.apache.lucene.analysis.uk.XUkrainianMorfologikAnalyzer'
+}

plugins/mapper-annotated-text/build.gradle

@@ -19,3 +19,9 @@ restResources {
     include '_common', 'indices', 'index', 'search'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // TODO: rename this package to be annotated text specific
+  ignoreClasses 'org.elasticsearch.search.fetch.subphase.highlight.AnnotatedPassageFormatter',
+    'org.elasticsearch.search.fetch.subphase.highlight.AnnotatedTextHighlighter'
+}

plugins/store-smb/build.gradle

@@ -17,3 +17,8 @@ restResources {
     include '_common', 'cluster', 'nodes', 'index', 'indices', 'get'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // TODO: rename this package to be smb specific
+  ignoreClasses 'org.elasticsearch.index.store.SmbDirectoryWrapper'
+}

qa/die-with-dignity/build.gradle

@@ -29,3 +29,8 @@ tasks.named("test").configure {
   enabled = false
 }
 
+tasks.named('splitPackagesAudit').configure {
+  // these should be moved to an actual package, not the root package
+  ignoreClasses 'org.elasticsearch.DieWithDignityPlugin',
+    'org.elasticsearch.RestDieWithDignityAction'
+}

server/build.gradle

@@ -285,3 +285,51 @@ tasks.named("licenseHeaders").configure {
     excludes << 'org/apache/lucene/search/RegExp87*'
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
 }
+
+tasks.named('splitPackagesAudit').configure {
+    // Lucene packages should be owned by Lucene!
+  ignoreClasses 'org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter',
+    'org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute',
+    'org.apache.lucene.analysis.miscellaneous.DisableGraphAttributeImpl',
+    'org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter',
+    'org.apache.lucene.analysis.miscellaneous.DuplicateSequenceAttribute',
+    'org.apache.lucene.analysis.miscellaneous.DuplicateSequenceAttributeImpl',
+    'org.apache.lucene.document.BinaryRange',
+    'org.apache.lucene.queryparser.classic.XQueryParser',
+    'org.apache.lucene.queries.BinaryDocValuesRangeQuery',
+    'org.apache.lucene.queries.BlendedTermQuery',
+    'org.apache.lucene.queries.MinDocQuery',
+    'org.apache.lucene.queries.SearchAfterSortedDocQuery',
+    'org.apache.lucene.queries.SpanMatchNoDocsQuery',
+    'org.apache.lucene.search.grouping.CollapseTopFieldDocs',
+    'org.apache.lucene.search.grouping.CollapsingDocValuesSource',
+    'org.apache.lucene.search.grouping.CollapsingTopDocsCollector',
+    'org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner',
+    'org.apache.lucene.search.uhighlight.CustomFieldHighlighter',
+    'org.apache.lucene.search.uhighlight.CustomPassageFormatter',
+    'org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter',
+    'org.apache.lucene.search.uhighlight.Snippet',
+    'org.apache.lucene.util.CombinedBitSet',
+    'org.apache.lucene.search.vectorhighlight.CustomFieldQuery',
+
+    // These are tricky because Lucene itself splits the index package,
+    // but this should be fixed in Lucene 9
+    'org.apache.lucene.index.LazySoftDeletesDirectoryReaderWrapper',
+    'org.apache.lucene.index.OneMergeHelper',
+    'org.apache.lucene.index.ShuffleForcedMergePolicy',
+
+    // Joda should own its own packages! This should be a simple move.
+    'org.joda.time.format.StrictISODateTimeFormat',
+
+    // xcontent should not have common, all it's common packages should be renamed to xcontent
+    'org.elasticsearch.common.xcontent.*',
+    'org.elasticsearch.common.xcontent.support.XContentMapValues',
+
+    // cli is owned by the libs/cli, so these should be moved to o.e.server.cli
+    'org.elasticsearch.cli.CommandLoggingConfigurator',
+    'org.elasticsearch.cli.EnvironmentAwareCommand',
+    'org.elasticsearch.cli.KeyStoreAwareCommand',
+    'org.elasticsearch.cli.LoggingAwareCommand',
+    'org.elasticsearch.cli.LoggingAwareMultiCommand'
+
+}

test/external-modules/delayed-aggs/build.gradle

@@ -16,3 +16,9 @@ restResources {
     include '_common', 'indices', 'index', 'cluster', 'search'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // aggs is owned by server, these should be moved to delayedaggs
+  ignoreClasses 'org.elasticsearch.search.aggregations.DelayedShardAggregationBuilder',
+    'org.elasticsearch.search.aggregations.DelayedShardAggregationPlugin'
+}

test/external-modules/error-query/build.gradle

@@ -16,3 +16,10 @@ restResources {
     include '_common', 'indices', 'index', 'cluster', 'search'
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // search.query is owned by server, these should be moved to errorquery
+  ignoreClasses 'org.elasticsearch.search.query.ErrorQueryBuilder',
+    'org.elasticsearch.search.query.ErrorQueryPlugin',
+    'org.elasticsearch.search.query.IndexError'
+}

test/framework/build.gradle

@@ -108,3 +108,8 @@ tasks.register("integTest", Test) {
 tasks.register("verifyVersions") {
   dependsOn "test"
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // for now we always run tests with the classpath, so we are ok with split packages for tests 
+  onlyIf { false }
+}

x-pack/plugin/ccr/build.gradle

@@ -42,3 +42,8 @@ tasks.named("testingConventions").configure {
     }
   }
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // xpack core has xpackinfo classes for ccr under this package
+  ignoreClasses 'org.elasticsearch.xpack.ccr.*'
+}

x-pack/plugin/core/build.gradle

@@ -144,3 +144,17 @@ testClusters.all {
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // common should be owned by server, this should be renamed
+  ignoreClasses 'org.elasticsearch.common.network.InetAddressHelper',
+
+    // engine is owned by server, these should be renamed
+    'org.elasticsearch.index.engine.FrozenEngine',
+    'org.elasticsearch.index.engine.RewriteCachingDirectoryReader',
+
+    // snapshots are part of server, these should be renamed
+    'org.elasticsearch.snapshots.SeqIdGeneratingFilterReader',
+    'org.elasticsearch.snapshots.SourceOnlySnapshot',
+    'org.elasticsearch.snapshots.SourceOnlySnapshotRepository'
+}

x-pack/plugin/ql/test-fixtures/build.gradle

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 
 dependencies {
   api project(xpackModule('ql'))

x-pack/plugin/search-business-rules/build.gradle

@@ -15,3 +15,9 @@ dependencies {
   testImplementation project(":test:framework")
 }
 
+tasks.named('splitPackagesAudit').configure {
+  // Lucene classes should be owned by Lucene!
+  ignoreClasses  'org.apache.lucene.search.CappedScoreQuery',
+    'org.apache.lucene.search.CappedScoreWeight',
+    'org.apache.lucene.search.CappedScorer'
+}

x-pack/plugin/searchable-snapshots/build.gradle

@@ -22,3 +22,8 @@ tasks.named("internalClusterTest").configure {
   // Reduce the buffer size per cache-fetch thread to keep direct memory usage under control.
   systemProperty 'es.searchable.snapshot.shared_cache.write_buffer.size', '256k'
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // xpack core needs to not split packages, the xpack info classes need to be moved
+  ignoreClasses 'org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots'
+}

x-pack/plugin/voting-only-node/build.gradle

@@ -11,3 +11,9 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
 }
+
+tasks.named('splitPackagesAudit').configure {
+  // o.e.cluster is owned by server, rename these
+  ignoreClasses 'org.elasticsearch.cluster.coordination.VotingOnlyNodeFeatureSet',
+    'org.elasticsearch.cluster.coordination.VotingOnlyNodePlugin'
+}