Remove all usages of Metadata customs removal methods (#129043)

This removes all non-test usage of
```
Metadata.Builder.removeProjectCustom(String)
Metadata.Builder.removeProjectCustomIf(BiPredicate)
```

And replaces it with appropriate calls to the equivalent method on
`ProjectMetadata.Builder`

In most cases this _does not_ make the code project aware, but does
reduce the number of deprecated methods in use.

server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.IndexReshardingMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -45,6 +46,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
@@ -706,12 +708,15 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
 
             @Override
             public Metadata.Builder remove(Metadata.Builder builder, String name) {
+                @FixForMultiProject
+                final var projectBuilder = builder.getProject(ProjectId.DEFAULT);
                 if (IndexGraveyard.TYPE.equals(name)) {
                     // there must always be at least an empty graveyard
-                    return builder.indexGraveyard(IndexGraveyard.builder().build());
+                    projectBuilder.indexGraveyard(IndexGraveyard.builder().build());
                 } else {
-                    return builder.removeProjectCustom(name);
+                    projectBuilder.removeCustom(name);
                 }
+                return builder;
             }
 
             @Override

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java

@@ -91,13 +91,13 @@ public class CustomMetadataContextIT extends AbstractSnapshotIntegTestCase {
         assertThat(getSnapshot("test-repo", "test-snapshot").state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("update custom persistent metadata");
-        updateClusterState(currentState -> currentState.copyAndUpdateMetadata(metadataBuilder -> {
+        updateClusterState(currentState -> currentState.copyAndUpdateProject(currentState.metadata().getProject().id(), builder -> {
             if (isSnapshotMetadataSet == false || randomBoolean()) {
-                metadataBuilder.putCustom(SnapshotMetadata.TYPE, new SnapshotMetadata("after_snapshot_s"));
+                builder.putCustom(SnapshotMetadata.TYPE, new SnapshotMetadata("after_snapshot_s"));
             } else {
-                metadataBuilder.removeProjectCustom(SnapshotMetadata.TYPE);
+                builder.removeCustom(SnapshotMetadata.TYPE);
             }
-            metadataBuilder.putCustom(ApiMetadata.TYPE, new ApiMetadata("after_snapshot_ns"));
+            builder.putCustom(ApiMetadata.TYPE, new ApiMetadata("after_snapshot_ns"));
         }));
 
         logger.info("restore snapshot");

server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java

@@ -16,6 +16,8 @@ import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.Tuple;
@@ -66,6 +68,8 @@ public class RemoveCustomsCommand extends ElasticsearchNodeCommand {
             "project scoped custom metadata names: " + oldClusterState.metadata().getProject().customs().keySet()
         );
         final Metadata.Builder metadataBuilder = Metadata.builder(oldClusterState.metadata());
+        @FixForMultiProject
+        final ProjectMetadata.Builder projectBuilder = metadataBuilder.getProject(ProjectId.DEFAULT);
         for (String customToRemove : customsToRemove) {
             @FixForMultiProject
             boolean matched = false;
@@ -82,7 +86,7 @@ public class RemoveCustomsCommand extends ElasticsearchNodeCommand {
             }
             for (String customKey : oldClusterState.metadata().getProject().customs().keySet()) {
                 if (Regex.simpleMatch(customToRemove, customKey)) {
-                    metadataBuilder.removeProjectCustom(customKey);
+                    projectBuilder.removeCustom(customKey);
                     if (matched == false) {
                         terminal.println("The following customs will be removed:");
                     }

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -1743,18 +1743,6 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
             return this;
         }
 
-        @Deprecated(forRemoval = true)
-        public Builder removeProjectCustom(String type) {
-            getSingleProject().removeCustom(type);
-            return this;
-        }
-
-        @Deprecated(forRemoval = true)
-        public Builder removeProjectCustomIf(BiPredicate<String, ? super ProjectCustom> p) {
-            getSingleProject().removeCustomIf(p);
-            return this;
-        }
-
         @Deprecated(forRemoval = true)
         public Builder projectCustoms(Map<String, ProjectCustom> projectCustoms) {
             projectCustoms.forEach((key, value) -> Objects.requireNonNull(value, key));

server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java

@@ -681,10 +681,12 @@ public class ClusterChangedEventTests extends ESTestCase {
         final ClusterState.Builder builder = ClusterState.builder(previousState);
         builder.stateUUID(UUIDs.randomBase64UUID());
         Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata());
+        ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(previousState.metadata().projects().values().iterator().next());
         metadataBuilder.removeCustomIf((ignore, custom) -> custom instanceof TestClusterCustomMetadata);
-        metadataBuilder.removeProjectCustomIf((ignore, custom) -> custom instanceof TestProjectCustomMetadata);
+        projectBuilder.removeCustomIf((ignore, custom) -> custom instanceof TestProjectCustomMetadata);
         clusterCustoms.forEach(clusterCustom -> metadataBuilder.putCustom(clusterCustom.getWriteableName(), clusterCustom));
-        projectCustoms.forEach(projectCustom -> metadataBuilder.putCustom(projectCustom.getWriteableName(), projectCustom));
+        projectCustoms.forEach(projectCustom -> projectBuilder.putCustom(projectCustom.getWriteableName(), projectCustom));
+        metadataBuilder.put(projectBuilder);
         builder.metadata(metadataBuilder);
         return builder.build();
     }

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

@@ -1391,20 +1391,6 @@ public class MetadataTests extends ESTestCase {
         assertThat(metadata.custom("custom2"), sameInstance(custom2));
     }
 
-    public void testBuilderRemoveProjectCustomIf() {
-        var custom1 = new TestProjectCustomMetadata();
-        var custom2 = new TestProjectCustomMetadata();
-        var builder = Metadata.builder();
-        builder.putCustom("custom1", custom1);
-        builder.putCustom("custom2", custom2);
-
-        builder.removeProjectCustomIf((key, value) -> Objects.equals(key, "custom1"));
-
-        var metadata = builder.build();
-        assertThat(metadata.getProject().custom("custom1"), nullValue());
-        assertThat(metadata.getProject().custom("custom2"), sameInstance(custom2));
-    }
-
     public void testBuilderRejectsDataStreamThatConflictsWithIndex() {
         final String dataStreamName = "my-data-stream";
         IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();

server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java

@@ -20,6 +20,7 @@ import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -933,7 +934,10 @@ public class PersistentTasksClusterServiceTests extends ESTestCase {
                 if (scope == PersistentTasksExecutor.Scope.CLUSTER) {
                     metadata = Metadata.builder(clusterState.metadata()).removeCustom(ClusterPersistentTasksCustomMetadata.TYPE);
                 } else {
-                    metadata = Metadata.builder(clusterState.metadata()).removeProjectCustom(PersistentTasksCustomMetadata.TYPE);
+                    metadata = Metadata.builder(clusterState.metadata())
+                        .put(
+                            ProjectMetadata.builder(clusterState.metadata().getProject()).removeCustom(PersistentTasksCustomMetadata.TYPE)
+                        );
                 }
                 return builder.metadata(metadata).build();
             }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java

@@ -31,7 +31,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
@@ -865,10 +865,10 @@ public abstract class CcrIntegTestCase extends ESTestCase {
             public ClusterState execute(ClusterState currentState) throws Exception {
                 AutoFollowMetadata empty = new AutoFollowMetadata(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
                 ClusterState.Builder newState = ClusterState.builder(currentState);
-                newState.metadata(
-                    Metadata.builder(currentState.getMetadata())
+                newState.putProjectMetadata(
+                    ProjectMetadata.builder(currentState.metadata().getProject())
                         .putCustom(AutoFollowMetadata.TYPE, empty)
-                        .removeProjectCustom(PersistentTasksCustomMetadata.TYPE)
+                        .removeCustom(PersistentTasksCustomMetadata.TYPE)
                         .build()
                 );
                 return newState.build();

x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java

@@ -29,7 +29,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -863,9 +862,10 @@ public class SystemIndexMigrator extends AllocatedPersistentTask {
         submitUnbatchedTask(clusterService, "clear migration results", new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
-                if (currentState.metadata().getProject().custom(FeatureMigrationResults.TYPE) != null) {
+                final var project = currentState.metadata().getProject();
+                if (project.custom(FeatureMigrationResults.TYPE) != null) {
                     return ClusterState.builder(currentState)
-                        .metadata(Metadata.builder(currentState.metadata()).removeProjectCustom(FeatureMigrationResults.TYPE))
+                        .putProjectMetadata(ProjectMetadata.builder(project).removeCustom(FeatureMigrationResults.TYPE))
                         .build();
                 }
                 return currentState;