Revert #135052 #133630 (#135341)

* Revert "Move individual stats fields to AsyncRefresh (#135052)"

This reverts commit 2b0153bdbb6d490defb918deadf95fafd7a2f515.

* Revert "Track shardStarted events for simulation in DesiredBalanceComputer (#133630)"

This reverts commit f248596de8d03208fec0497cb39eb8a755170a4d.

docs/changelog/133630.yaml

@@ -1,5 +0,0 @@
-pr: 133630
-summary: Track `shardStarted` events for simulation in `DesiredBalanceComputer`
-area: Allocation
-type: enhancement
-issues: []

server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -20,15 +20,12 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -68,7 +65,6 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
     final Map<ShardId, Double> shardWriteLoads;
     // max heap size per node ID
     final Map<String, ByteSizeValue> maxHeapSizePerNode;
-    private final Map<ShardId, Set<String>> shardToNodeIds;
 
     protected ClusterInfo() {
         this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
@@ -110,7 +106,6 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
         this.nodeUsageStatsForThreadPools = Map.copyOf(nodeUsageStatsForThreadPools);
         this.shardWriteLoads = Map.copyOf(shardWriteLoads);
         this.maxHeapSizePerNode = Map.copyOf(maxHeapSizePerNode);
-        this.shardToNodeIds = computeShardToNodeIds(dataPath);
     }
 
     public ClusterInfo(StreamInput in) throws IOException {
@@ -140,23 +135,6 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
         } else {
             this.maxHeapSizePerNode = Map.of();
         }
-        this.shardToNodeIds = computeShardToNodeIds(dataPath);
-    }
-
-    private static Map<ShardId, Set<String>> computeShardToNodeIds(Map<NodeAndShard, String> dataPath) {
-        if (dataPath.isEmpty()) {
-            return Map.of();
-        }
-        final var shardToNodeIds = new HashMap<ShardId, Set<String>>();
-        for (NodeAndShard nodeAndShard : dataPath.keySet()) {
-            shardToNodeIds.computeIfAbsent(nodeAndShard.shardId, ignore -> new HashSet<>()).add(nodeAndShard.nodeId);
-        }
-        Maps.transformValues(shardToNodeIds, nodeIds -> Collections.unmodifiableSet(nodeIds));
-        return shardToNodeIds;
-    }
-
-    public Set<String> getNodeIdsForShard(ShardId shardId) {
-        return shardToNodeIds.getOrDefault(shardId, Set.of());
     }
 
     @Override
@@ -365,16 +343,6 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
         return this.maxHeapSizePerNode;
     }
 
-    /**
-     * Return true if the shard has moved since the time ClusterInfo was created.
-     */
-    public boolean hasShardMoved(ShardRouting shardRouting) {
-        // We use dataPath to find out whether a shard is allocated on a node.
-        // TODO: DataPath is sent with disk usages but thread pool usage is sent separately so that local shard allocation
-        // may change between the two calls.
-        return getDataPath(shardRouting) == null;
-    }
-
     /**
      * Method that incorporates the ShardId for the shard into a string that
      * includes a 'p' or 'r' depending on whether the shard is a primary.

server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java

@@ -9,16 +9,12 @@
 
 package org.elasticsearch.cluster;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterInfo.NodeAndShard;
 import org.elasticsearch.cluster.routing.ShardMovementWriteLoadSimulator;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CopyOnFirstWriteMap;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.HashMap;
@@ -29,12 +25,9 @@ import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting;
 import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize;
 import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.shouldReserveSpaceForInitializingShard;
 import static org.elasticsearch.cluster.routing.ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE;
-import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED;
 
 public class ClusterInfoSimulator {
 
-    private static final Logger logger = LogManager.getLogger(ClusterInfoSimulator.class);
-
     private final RoutingAllocation allocation;
 
     private final Map<String, DiskUsage> leastAvailableSpaceUsage;
@@ -102,7 +95,7 @@ public class ClusterInfoSimulator {
      * Balance is later recalculated with a refreshed cluster info containing actual shards placement.
      */
     public void simulateShardStarted(ShardRouting shard) {
-        assert shard.initializing() : "expected an initializing shard, but got: " + shard;
+        assert shard.initializing();
 
         var project = allocation.metadata().projectFor(shard.index());
         var size = getExpectedShardSize(
@@ -129,36 +122,6 @@ public class ClusterInfoSimulator {
         shardMovementWriteLoadSimulator.simulateShardStarted(shard);
     }
 
-    /**
-     * This method simulates starting an already started shard with an optional {@code sourceNodeId} in case of a relocation.
-     * @param startedShard The shard to simulate. Must be started already.
-     * @param sourceNodeId The source node ID if the shard started as a result of relocation. {@code null} otherwise.
-     */
-    public void simulateAlreadyStartedShard(ShardRouting startedShard, @Nullable String sourceNodeId) {
-        assert startedShard.started() : "expected an already started shard, but got: " + startedShard;
-        if (logger.isDebugEnabled()) {
-            logger.debug(
-                "simulated started shard {} on node [{}] as a {}",
-                startedShard.shardId(),
-                startedShard.currentNodeId(),
-                sourceNodeId != null ? "relocating shard from node [" + sourceNodeId + "]" : "new shard"
-            );
-        }
-        final long expectedShardSize = startedShard.getExpectedShardSize();
-        if (sourceNodeId != null) {
-            final var relocatingShard = startedShard.moveToUnassigned(new UnassignedInfo(REINITIALIZED, "simulation"))
-                .initialize(sourceNodeId, null, expectedShardSize)
-                .moveToStarted(expectedShardSize)
-                .relocate(startedShard.currentNodeId(), expectedShardSize)
-                .getTargetRelocatingShard();
-            simulateShardStarted(relocatingShard);
-        } else {
-            final var initializingShard = startedShard.moveToUnassigned(new UnassignedInfo(REINITIALIZED, "simulation"))
-                .initialize(startedShard.currentNodeId(), null, expectedShardSize);
-            simulateShardStarted(initializingShard);
-        }
-    }
-
     private void modifyDiskUsage(String nodeId, long freeDelta) {
         if (freeDelta == 0) {
             return;

server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java

@@ -100,6 +100,13 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
     private volatile TimeValue updateFrequency;
     private volatile TimeValue fetchTimeout;
 
+    private volatile Map<String, DiskUsage> leastAvailableSpaceUsages;
+    private volatile Map<String, DiskUsage> mostAvailableSpaceUsages;
+    private volatile Map<String, ByteSizeValue> maxHeapPerNode;
+    private volatile Map<String, Long> estimatedHeapUsagePerNode;
+    private volatile Map<String, NodeUsageStatsForThreadPools> nodeThreadPoolUsageStatsPerNode;
+    private volatile IndicesStatsSummary indicesStatsSummary;
+
     private final ThreadPool threadPool;
     private final Client client;
     private final Supplier<ClusterState> clusterStateSupplier;
@@ -113,8 +120,6 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
     private AsyncRefresh currentRefresh;
     private RefreshScheduler refreshScheduler;
 
-    private volatile ClusterInfo currentClusterInfo = ClusterInfo.EMPTY;
-
     @SuppressWarnings("this-escape")
     public InternalClusterInfoService(
         Settings settings,
@@ -124,6 +129,12 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
         EstimatedHeapUsageCollector estimatedHeapUsageCollector,
         NodeUsageStatsForThreadPoolsCollector nodeUsageStatsForThreadPoolsCollector
     ) {
+        this.leastAvailableSpaceUsages = Map.of();
+        this.mostAvailableSpaceUsages = Map.of();
+        this.maxHeapPerNode = Map.of();
+        this.estimatedHeapUsagePerNode = Map.of();
+        this.nodeThreadPoolUsageStatsPerNode = Map.of();
+        this.indicesStatsSummary = IndicesStatsSummary.EMPTY;
         this.threadPool = threadPool;
         this.client = client;
         this.estimatedHeapUsageCollector = estimatedHeapUsageCollector;
@@ -197,13 +208,6 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
 
     private class AsyncRefresh {
 
-        private volatile Map<String, DiskUsage> leastAvailableSpaceUsages;
-        private volatile Map<String, DiskUsage> mostAvailableSpaceUsages;
-        private volatile Map<String, ByteSizeValue> maxHeapPerNode;
-        private volatile Map<String, Long> estimatedHeapUsagePerNode;
-        private volatile Map<String, NodeUsageStatsForThreadPools> nodeThreadPoolUsageStatsPerNode;
-        private volatile IndicesStatsSummary indicesStatsSummary;
-
         private final List<ActionListener<ClusterInfo>> thisRefreshListeners;
         private final RefCountingRunnable fetchRefs = new RefCountingRunnable(this::callListeners);
 
@@ -449,7 +453,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
         private void callListeners() {
             try {
                 logger.trace("stats all received, computing cluster info and notifying listeners");
-                final ClusterInfo clusterInfo = updateAndGetCurrentClusterInfo();
+                final ClusterInfo clusterInfo = getClusterInfo();
                 boolean anyListeners = false;
                 for (final Consumer<ClusterInfo> listener : listeners) {
                     anyListeners = true;
@@ -469,32 +473,6 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
                 onRefreshComplete(this);
             }
         }
-
-        private ClusterInfo updateAndGetCurrentClusterInfo() {
-            final IndicesStatsSummary indicesStatsSummary = this.indicesStatsSummary; // single volatile read
-            final Map<String, EstimatedHeapUsage> estimatedHeapUsages = new HashMap<>();
-            final var currentMaxHeapPerNode = this.maxHeapPerNode; // Make sure we use a consistent view
-            currentMaxHeapPerNode.forEach((nodeId, maxHeapSize) -> {
-                final Long estimatedHeapUsage = estimatedHeapUsagePerNode.get(nodeId);
-                if (estimatedHeapUsage != null) {
-                    estimatedHeapUsages.put(nodeId, new EstimatedHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage));
-                }
-            });
-            final var newClusterInfo = new ClusterInfo(
-                leastAvailableSpaceUsages,
-                mostAvailableSpaceUsages,
-                indicesStatsSummary.shardSizes,
-                indicesStatsSummary.shardDataSetSizes,
-                indicesStatsSummary.dataPath,
-                indicesStatsSummary.reservedSpace,
-                estimatedHeapUsages,
-                nodeThreadPoolUsageStatsPerNode,
-                indicesStatsSummary.shardWriteLoads(),
-                currentMaxHeapPerNode
-            );
-            currentClusterInfo = newClusterInfo;
-            return newClusterInfo;
-        }
     }
 
     private void onRefreshComplete(AsyncRefresh completedRefresh) {
@@ -559,7 +537,27 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
 
     @Override
     public ClusterInfo getClusterInfo() {
-        return currentClusterInfo;
+        final IndicesStatsSummary indicesStatsSummary = this.indicesStatsSummary; // single volatile read
+        final Map<String, EstimatedHeapUsage> estimatedHeapUsages = new HashMap<>();
+        final var currentMaxHeapPerNode = this.maxHeapPerNode; // Make sure we use a consistent view
+        currentMaxHeapPerNode.forEach((nodeId, maxHeapSize) -> {
+            final Long estimatedHeapUsage = estimatedHeapUsagePerNode.get(nodeId);
+            if (estimatedHeapUsage != null) {
+                estimatedHeapUsages.put(nodeId, new EstimatedHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage));
+            }
+        });
+        return new ClusterInfo(
+            leastAvailableSpaceUsages,
+            mostAvailableSpaceUsages,
+            indicesStatsSummary.shardSizes,
+            indicesStatsSummary.shardDataSetSizes,
+            indicesStatsSummary.dataPath,
+            indicesStatsSummary.reservedSpace,
+            estimatedHeapUsages,
+            nodeThreadPoolUsageStatsPerNode,
+            indicesStatsSummary.shardWriteLoads(),
+            currentMaxHeapPerNode
+        );
     }
 
     // allow tests to adjust the node stats on receipt

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java

@@ -12,11 +12,7 @@ package org.elasticsearch.cluster.routing.allocation.allocator;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoSimulator;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -144,8 +140,6 @@ public class DesiredBalanceComputer {
             return new DesiredBalance(desiredBalanceInput.index(), Map.of(), Map.of(), finishReason);
         }
 
-        maybeSimulateAlreadyStartedShards(desiredBalanceInput.routingAllocation().clusterInfo(), routingNodes, clusterInfoSimulator);
-
         // we assume that all ongoing recoveries will complete
         for (final var routingNode : routingNodes) {
             for (final var shardRouting : routingNode) {
@@ -489,73 +483,6 @@ public class DesiredBalanceComputer {
         return new DesiredBalance(lastConvergedIndex, assignments, routingNodes.getBalanceWeightStatsPerNode(), finishReason);
     }
 
-    /**
-     * For shards started after initial polling of the ClusterInfo but before the next polling, we need to
-     * account for their impacts by simulating the events, either relocation or new shard start. This is done
-     * by comparing the current RoutingNodes against the shard allocation information from the ClusterInfo to
-     * find out the shard allocation changes. Note this approach is approximate in some edge cases:
-     * <ol>
-     * <li> If a shard is relocated twice from node A to B to C. It is considered as relocating from A to C directly
-     * for simulation purpose.</li>
-     * <li>If a shard has 2 replicas and they both relocate, replica 1 from A to X and replica 2 from B to Y. The
-     * simulation may see them as relocations A->X and B->Y. But it may also see them as A->Y and B->X. </li>
-     * </ol>
-     * In both cases, it should not really matter for simulation to account for resource changes.
-     */
-    static void maybeSimulateAlreadyStartedShards(
-        ClusterInfo clusterInfo,
-        RoutingNodes routingNodes,
-        ClusterInfoSimulator clusterInfoSimulator
-    ) {
-        // Find all shards that are started in RoutingNodes but have no data on corresponding node in ClusterInfo
-        final var startedShards = new ArrayList<ShardRouting>();
-        for (var routingNode : routingNodes) {
-            for (var shardRouting : routingNode.started()) {
-                if (clusterInfo.hasShardMoved(shardRouting)) {
-                    startedShards.add(shardRouting);
-                }
-            }
-        }
-        if (startedShards.isEmpty()) {
-            return;
-        }
-        logger.debug(
-            "Found [{}] started shards not accounted in ClusterInfo. The first one is {}",
-            startedShards.size(),
-            startedShards.getFirst()
-        );
-
-        // For started shards, attempt to find its source node. If found, it is a relocation, otherwise it is a new shard.
-        // The same shard on the same source node cannot be relocated twice to different nodes. So we exclude it once used.
-        final Map<ShardId, Set<String>> alreadySeenSourceNodes = new HashMap<>();
-        for (var startedShard : startedShards) {
-            // The source node is found by checking whether the ClusterInfo has a node hosting a shard with the same ShardId
-            // and has compatible node role. If multiple nodes are found, simply pick the first one.
-            final var sourceNodeId = clusterInfo.getNodeIdsForShard(startedShard.shardId())
-                .stream()
-                // Do not use the same source node twice for the same shard
-                .filter(nodeId -> alreadySeenSourceNodes.getOrDefault(startedShard.shardId(), Set.of()).contains(nodeId) == false)
-                .map(routingNodes::node)
-                // The source node must not currently host the shard
-                .filter(routingNode -> routingNode != null && routingNode.getByShardId(startedShard.shardId()) == null)
-                .map(RoutingNode::node)
-                // The source node must have compatible node roles
-                .filter(node -> node != null && switch (startedShard.role()) {
-                    case DEFAULT -> node.canContainData();
-                    case INDEX_ONLY -> node.getRoles().contains(DiscoveryNodeRole.INDEX_ROLE);
-                    case SEARCH_ONLY -> node.getRoles().contains(DiscoveryNodeRole.SEARCH_ROLE);
-                })
-                .map(DiscoveryNode::getId)
-                .findFirst()
-                .orElse(null);
-
-            if (sourceNodeId != null) {
-                alreadySeenSourceNodes.computeIfAbsent(startedShard.shardId(), k -> new HashSet<>()).add(sourceNodeId);
-            }
-            clusterInfoSimulator.simulateAlreadyStartedShard(startedShard, sourceNodeId);
-        }
-    }
-
     private void maybeLogAllocationExplainForUnassigned(
         DesiredBalance.ComputationFinishReason finishReason,
         RoutingNodes routingNodes,

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterInfoSimulatorTests.java

@@ -26,7 +26,6 @@ import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -41,8 +40,6 @@ import org.elasticsearch.snapshots.InternalSnapshotsInfoService;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
 
 import java.util.HashMap;
 import java.util.List;
@@ -56,16 +53,11 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
 import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder;
-import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED;
 import static org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS;
 import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE;
 import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING;
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
 
 public class ClusterInfoSimulatorTests extends ESAllocationTestCase {
 
@@ -639,48 +631,6 @@ public class ClusterInfoSimulatorTests extends ESAllocationTestCase {
         );
     }
 
-    public void testSimulateAlreadyStartedShard() {
-        final var clusterInfoSimulator = mock(ClusterInfoSimulator.class);
-        doCallRealMethod().when(clusterInfoSimulator).simulateAlreadyStartedShard(any(), any());
-
-        final ShardRouting startedShard = ShardRouting.newUnassigned(
-            new ShardId(randomIdentifier(), randomUUID(), between(0, 5)),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(REINITIALIZED, "simulation"),
-            ShardRouting.Role.DEFAULT
-        ).initialize(randomIdentifier(), null, randomLongBetween(100, 999)).moveToStarted(randomLongBetween(100, 999));
-
-        // New shard without relocation
-        {
-            final ArgumentCaptor<ShardRouting> shardCaptor = ArgumentCaptor.forClass(ShardRouting.class);
-            clusterInfoSimulator.simulateAlreadyStartedShard(startedShard, null);
-            verify(clusterInfoSimulator).simulateShardStarted(shardCaptor.capture());
-            final var captureShard = shardCaptor.getValue();
-            assertTrue(captureShard.initializing());
-            assertNull(captureShard.relocatingNodeId());
-            assertThat(captureShard.shardId(), equalTo(startedShard.shardId()));
-            assertThat(captureShard.currentNodeId(), equalTo(startedShard.currentNodeId()));
-            assertThat(captureShard.getExpectedShardSize(), equalTo(startedShard.getExpectedShardSize()));
-        }
-
-        // Relocation
-        {
-            Mockito.clearInvocations(clusterInfoSimulator);
-            final ArgumentCaptor<ShardRouting> shardCaptor = ArgumentCaptor.forClass(ShardRouting.class);
-            final String sourceNodeId = randomIdentifier();
-            clusterInfoSimulator.simulateAlreadyStartedShard(startedShard, sourceNodeId);
-            verify(clusterInfoSimulator).simulateShardStarted(shardCaptor.capture());
-            final var captureShard = shardCaptor.getValue();
-            assertTrue(captureShard.initializing());
-            assertThat(captureShard.relocatingNodeId(), equalTo(sourceNodeId));
-            assertThat(captureShard.shardId(), equalTo(startedShard.shardId()));
-            assertThat(captureShard.currentNodeId(), equalTo(startedShard.currentNodeId()));
-            assertThat(captureShard.getExpectedShardSize(), equalTo(startedShard.getExpectedShardSize()));
-        }
-
-    }
-
     private static void addIndex(Metadata.Builder metadataBuilder, RoutingTable.Builder routingTableBuilder, ShardRouting shardRouting) {
         var name = shardRouting.getIndexName();
         metadataBuilder.put(IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 0)));

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java

@@ -14,7 +14,6 @@ import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfo.NodeAndPath;
 import org.elasticsearch.cluster.ClusterInfo.NodeAndShard;
 import org.elasticsearch.cluster.ClusterInfo.ReservedSpace;
-import org.elasticsearch.cluster.ClusterInfoSimulator;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
@@ -24,7 +23,6 @@ import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
-import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -56,7 +54,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.repositories.IndexId;
@@ -67,7 +64,6 @@ import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.mockito.Mockito;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -84,7 +80,6 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toMap;
 import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting;
@@ -104,10 +99,6 @@ import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 public class DesiredBalanceComputerTests extends ESAllocationTestCase {
 
@@ -827,19 +818,19 @@ public class DesiredBalanceComputerTests extends ESAllocationTestCase {
             .withNode("node-1", 1000, 100)
             .withNode("node-2", 1000, 1000)
             // node-0 & node-1
-            .withShard(findShard(clusterState, "index-0", true), 500)
-            .withShard(findShard(clusterState, "index-0", false), 500)
+            .withShard(findShardId(clusterState, "index-0"), true, 500)
+            .withShard(findShardId(clusterState, "index-0"), false, 500)
             // node-0
-            .withShard(findShard(clusterState, "index-1", true), 400)
+            .withShard(findShardId(clusterState, "index-1"), true, 400)
             // node-1
-            .withShard(findShard(clusterState, "index-2", true), 50)
-            .withShard(findShard(clusterState, "index-3", true), 50)
-            .withShard(findShard(clusterState, "index-4", true), 50)
-            .withShard(findShard(clusterState, "index-5", true), 50)
-            .withShard(findShard(clusterState, "index-6", true), 50)
-            .withShard(findShard(clusterState, "index-7", true), 50)
-            .withShard(findShard(clusterState, "index-8", true), 50)
-            .withShard(findShard(clusterState, "index-9", true), 50)
+            .withShard(findShardId(clusterState, "index-2"), true, 50)
+            .withShard(findShardId(clusterState, "index-3"), true, 50)
+            .withShard(findShardId(clusterState, "index-4"), true, 50)
+            .withShard(findShardId(clusterState, "index-5"), true, 50)
+            .withShard(findShardId(clusterState, "index-6"), true, 50)
+            .withShard(findShardId(clusterState, "index-7"), true, 50)
+            .withShard(findShardId(clusterState, "index-8"), true, 50)
+            .withShard(findShardId(clusterState, "index-9"), true, 50)
             .build();
 
         var settings = Settings.builder()
@@ -1189,7 +1180,6 @@ public class DesiredBalanceComputerTests extends ESAllocationTestCase {
         private final Map<String, DiskUsage> diskUsage = new HashMap<>();
         private final Map<String, Long> shardSizes = new HashMap<>();
         private final Map<NodeAndPath, ReservedSpace> reservedSpace = new HashMap<>();
-        private final Map<NodeAndShard, String> dataPath = new HashMap<>();
 
         public ClusterInfoTestBuilder withNode(String nodeId, long totalBytes, long freeBytes) {
             diskUsage.put(nodeId, new DiskUsage(nodeId, nodeId, "/path", totalBytes, freeBytes));
@@ -1209,14 +1199,6 @@ public class DesiredBalanceComputerTests extends ESAllocationTestCase {
             return this;
         }
 
-        public ClusterInfoTestBuilder withShard(ShardRouting shard, long size) {
-            shardSizes.put(shardIdentifierFromRouting(shard), size);
-            if (shard.unassigned() == false) {
-                dataPath.put(NodeAndShard.from(shard), "/data/path");
-            }
-            return this;
-        }
-
         public ClusterInfoTestBuilder withReservedSpace(String nodeId, long size, ShardId... shardIds) {
             reservedSpace.put(new NodeAndPath(nodeId, "/path"), new ReservedSpace(size, Set.of(shardIds)));
             return this;
@@ -1228,7 +1210,6 @@ public class DesiredBalanceComputerTests extends ESAllocationTestCase {
                 .mostAvailableSpaceUsage(diskUsage)
                 .shardSizes(shardSizes)
                 .reservedSpace(reservedSpace)
-                .dataPath(dataPath)
                 .build();
         }
     }
@@ -1598,130 +1579,6 @@ public class DesiredBalanceComputerTests extends ESAllocationTestCase {
         }
     }
 
-    public void testMaybeSimulateAlreadyStartedShards() {
-        final var clusterInfoSimulator = mock(ClusterInfoSimulator.class);
-        final var routingChangesObserver = mock(RoutingChangesObserver.class);
-
-        final ClusterState initialState = createInitialClusterState(3, 3, 1);
-        final RoutingNodes routingNodes = initialState.mutableRoutingNodes();
-
-        final IndexRoutingTable indexRoutingTable = initialState.routingTable(ProjectId.DEFAULT).index(TEST_INDEX);
-
-        final List<ShardRouting> existingStartedShards = new ArrayList<>();
-
-        // Shard 0 - primary started
-        final String shard0PrimaryNodeId = randomFrom(initialState.nodes().getDataNodes().values()).getId();
-        existingStartedShards.add(
-            routingNodes.startShard(
-                routingNodes.initializeShard(
-                    indexRoutingTable.shard(0).primaryShard(),
-                    shard0PrimaryNodeId,
-                    null,
-                    randomLongBetween(100, 999),
-                    routingChangesObserver
-                ),
-                routingChangesObserver,
-                randomLongBetween(100, 999)
-            )
-        );
-        if (randomBoolean()) {
-            existingStartedShards.add(
-                routingNodes.startShard(
-                    routingNodes.initializeShard(
-                        indexRoutingTable.shard(0).replicaShards().getFirst(),
-                        randomValueOtherThan(shard0PrimaryNodeId, () -> randomFrom(initialState.nodes().getDataNodes().values()).getId()),
-                        null,
-                        randomLongBetween(100, 999),
-                        routingChangesObserver
-                    ),
-                    routingChangesObserver,
-                    randomLongBetween(100, 999)
-                )
-            );
-        }
-
-        // Shard 1 - initializing primary or replica
-        final String shard1PrimaryNodeId = randomFrom(initialState.nodes().getDataNodes().values()).getId();
-        ShardRouting initializingShard = routingNodes.initializeShard(
-            indexRoutingTable.shard(1).primaryShard(),
-            shard1PrimaryNodeId,
-            null,
-            randomLongBetween(100, 999),
-            routingChangesObserver
-        );
-        if (randomBoolean()) {
-            existingStartedShards.add(routingNodes.startShard(initializingShard, routingChangesObserver, randomLongBetween(100, 999)));
-            initializingShard = routingNodes.initializeShard(
-                indexRoutingTable.shard(1).replicaShards().getFirst(),
-                randomValueOtherThan(shard1PrimaryNodeId, () -> randomFrom(initialState.nodes().getDataNodes().values()).getId()),
-                null,
-                randomLongBetween(100, 999),
-                routingChangesObserver
-            );
-        }
-
-        // Shard 2 - Relocating primary
-        final String shard2PrimaryNodeId = randomFrom(initialState.nodes().getDataNodes().values()).getId();
-        final Tuple<ShardRouting, ShardRouting> relocationTuple = routingNodes.relocateShard(
-            routingNodes.startShard(
-                routingNodes.initializeShard(
-                    indexRoutingTable.shard(2).primaryShard(),
-                    shard2PrimaryNodeId,
-                    null,
-                    randomLongBetween(100, 999),
-                    routingChangesObserver
-                ),
-                routingChangesObserver,
-                randomLongBetween(100, 999)
-            ),
-            randomValueOtherThan(shard2PrimaryNodeId, () -> randomFrom(initialState.nodes().getDataNodes().values()).getId()),
-            randomLongBetween(100, 999),
-            "test",
-            routingChangesObserver
-        );
-        existingStartedShards.add(relocationTuple.v1());
-
-        final ClusterInfo clusterInfo = ClusterInfo.builder()
-            .dataPath(
-                existingStartedShards.stream()
-                    .collect(Collectors.toUnmodifiableMap(NodeAndShard::from, ignore -> "/data/" + randomIdentifier()))
-            )
-            .build();
-
-        // No extra simulation calls since there is no new shard or relocated shard that are not in ClusterInfo
-        {
-            DesiredBalanceComputer.maybeSimulateAlreadyStartedShards(clusterInfo, routingNodes, clusterInfoSimulator);
-            verifyNoInteractions(clusterInfoSimulator);
-        }
-
-        // Start the initializing shard and it should be identified and simulated
-        final var startedShard = routingNodes.startShard(initializingShard, routingChangesObserver, randomLongBetween(100, 999));
-        {
-            DesiredBalanceComputer.maybeSimulateAlreadyStartedShards(clusterInfo, routingNodes, clusterInfoSimulator);
-            verify(clusterInfoSimulator).simulateAlreadyStartedShard(startedShard, null);
-            verifyNoMoreInteractions(clusterInfoSimulator);
-        }
-
-        // Also start the relocating shard and both should be identified and simulated
-        {
-            Mockito.clearInvocations(clusterInfoSimulator);
-            final var startedRelocatingShard = routingNodes.startShard(
-                relocationTuple.v2(),
-                routingChangesObserver,
-                randomLongBetween(100, 999)
-            );
-            DesiredBalanceComputer.maybeSimulateAlreadyStartedShards(clusterInfo, routingNodes, clusterInfoSimulator);
-            verify(clusterInfoSimulator).simulateAlreadyStartedShard(startedShard, null);
-            verify(clusterInfoSimulator).simulateAlreadyStartedShard(startedRelocatingShard, relocationTuple.v1().currentNodeId());
-            verifyNoMoreInteractions(clusterInfoSimulator);
-        }
-    }
-
-    private static ShardRouting findShard(ClusterState clusterState, String name, boolean primary) {
-        final var indexShardRoutingTable = clusterState.getRoutingTable().index(name).shard(0);
-        return primary ? indexShardRoutingTable.primaryShard() : indexShardRoutingTable.replicaShards().getFirst();
-    }
-
     private static ShardId findShardId(ClusterState clusterState, String name) {
         return clusterState.getRoutingTable().index(name).shard(0).shardId();
     }

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java

@@ -19,23 +19,17 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.cluster.NodeUsageStatsForThreadPools;
-import org.elasticsearch.cluster.NodeUsageStatsForThreadPools.ThreadPoolUsageStats;
 import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
-import org.elasticsearch.cluster.metadata.ProjectId;
-import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata.Type;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -62,7 +56,6 @@ import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collections;
 import java.util.List;
@@ -783,262 +776,6 @@ public class DesiredBalanceShardsAllocatorTests extends ESAllocationTestCase {
         }
     }
 
-    public void testSimulateStartedShardsWithinASingleClusterInfoPolling() {
-        final var firstNode = newNode("node-1");
-        final var secondNode = newNode("node-2");
-        final var thirdNode = newNode("node-3");
-        final DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder().add(firstNode).add(secondNode).add(thirdNode);
-        discoveryNodesBuilder.localNodeId(firstNode.getId()).masterNodeId(firstNode.getId());
-
-        final ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(ProjectId.DEFAULT);
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
-        final var indexMetadata = IndexMetadata.builder("test-index").settings(indexSettings(IndexVersion.current(), 2, 0)).build();
-        final var shardId0 = new ShardId(indexMetadata.getIndex(), 0);
-        final ShardRouting shardRouting0 = ShardRoutingHelper.moveToStarted(
-            ShardRoutingHelper.initialize(
-                ShardRouting.newUnassigned(
-                    shardId0,
-                    true,
-                    RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "new index"),
-                    ShardRouting.Role.DEFAULT
-                ),
-                firstNode.getId()
-            )
-        );
-
-        final ShardId shardId1 = new ShardId(indexMetadata.getIndex(), 1);
-        final ShardRouting shardRouting1 = ShardRoutingHelper.initialize(
-            ShardRouting.newUnassigned(
-                shardId1,
-                true,
-                RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "new index"),
-                ShardRouting.Role.DEFAULT
-            ),
-            thirdNode.getId()
-        );
-        routingTableBuilder.add(IndexRoutingTable.builder(shardId0.getIndex()).addShard(shardRouting0).addShard(shardRouting1).build());
-        projectBuilder.put(
-            IndexMetadata.builder(indexMetadata).putInSyncAllocationIds(0, Set.of(shardRouting0.allocationId().getId())).build(),
-            false
-        );
-
-        var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(discoveryNodesBuilder)
-            .metadata(Metadata.builder().put(projectBuilder))
-            .putRoutingTable(ProjectId.DEFAULT, routingTableBuilder.build())
-            .build();
-
-        var threadPool = new TestThreadPool(getTestName());
-        var clusterService = ClusterServiceUtils.createClusterService(clusterState, threadPool);
-
-        final var relocated = new AtomicBoolean(false);
-        final AtomicReference<ClusterInfo> clusterInfoUsedByAllocator = new AtomicReference<>();
-        var delegateAllocator = new ShardsAllocator() {
-            @Override
-            public void allocate(RoutingAllocation allocation) {
-                allocation.routingNodes().setBalanceWeightStatsPerNode(Map.of());
-                clusterInfoUsedByAllocator.set(allocation.clusterInfo());
-                if (relocated.compareAndSet(false, true)) {
-                    logger.info("--> relocating shard [{}]", shardId0);
-                    final ShardRouting shardRouting = allocation.routingTable(ProjectId.DEFAULT).shardRoutingTable(shardId0).primaryShard();
-                    allocation.routingNodes().relocateShard(shardRouting, secondNode.getId(), 100, "test", allocation.changes());
-                }
-            }
-
-            @Override
-            public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
-                throw new AssertionError("only used for allocation explain");
-            }
-        };
-
-        final var allocationServiceRef = new AtomicReference<AllocationService>();
-        var clusterSettings = createBuiltInClusterSettings();
-        var desiredBalanceComputer = new DesiredBalanceComputer(clusterSettings, threadPool, delegateAllocator, TEST_ONLY_EXPLAINER);
-        var desiredBalanceShardsAllocator = new DesiredBalanceShardsAllocator(
-            delegateAllocator,
-            threadPool,
-            clusterService,
-            desiredBalanceComputer,
-            (reconcilerClusterState, rerouteStrategy) -> allocationServiceRef.get()
-                .executeWithRoutingAllocation(reconcilerClusterState, "reconcile-desired-balance", rerouteStrategy),
-            EMPTY_NODE_ALLOCATION_STATS,
-            DesiredBalanceMetrics.NOOP
-        ) {
-            @Override
-            protected void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) {
-                logger.info("--> reconcile called");
-                super.reconcile(desiredBalance, allocation);
-            }
-        };
-
-        // A fixed ClusterInfo from one polling cycle with hotspot on first node
-        final var zeroUsageStats = new ThreadPoolUsageStats(10, 0.0f, 0);
-        final Map<String, NodeUsageStatsForThreadPools> initialThreadPoolStats = Map.of(
-            firstNode.getId(),
-            new NodeUsageStatsForThreadPools(firstNode.getId(), Map.of(ThreadPool.Names.WRITE, new ThreadPoolUsageStats(10, 10.0f, 30))),
-            secondNode.getId(),
-            new NodeUsageStatsForThreadPools(secondNode.getId(), Map.of(ThreadPool.Names.WRITE, zeroUsageStats)),
-            thirdNode.getId(),
-            new NodeUsageStatsForThreadPools(secondNode.getId(), Map.of(ThreadPool.Names.WRITE, zeroUsageStats))
-        );
-
-        final var clusterInfoRef = new AtomicReference<>(
-            ClusterInfo.builder()
-                .dataPath(Map.of(ClusterInfo.NodeAndShard.from(shardRouting0), "/data/path0"))
-                .shardWriteLoads(Map.of(shardId0, 10.0d, shardId1, 10.0d))
-                .nodeUsageStatsForThreadPools(initialThreadPoolStats)
-                .build()
-        );
-
-        var service = new AllocationService(
-            new AllocationDeciders(List.of()),
-            createGatewayAllocator(),
-            desiredBalanceShardsAllocator,
-            clusterInfoRef::get,
-            () -> SnapshotShardSizeInfo.EMPTY,
-            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
-        );
-        allocationServiceRef.set(service);
-
-        try {
-            // 1. Initial reroute should produce a desired balance which moves the shard from first node to second node.
-            rerouteAndWait(service, clusterState, "first-reroute");
-            final DesiredBalance desiredBalance = desiredBalanceShardsAllocator.getDesiredBalance();
-            assertNotNull(desiredBalance.assignments().get(shardId0));
-
-            // When compute ends, the last ClusterInfo it uses is an updated one different from the initial one because the simulation
-            // starts all initializing shards on their target nodes. Thread pool stats are updated to reflect the shard movements.
-            final ClusterInfo updatedClusterInfo = clusterInfoUsedByAllocator.get();
-            assertThat(updatedClusterInfo, not(equalTo(clusterInfoRef.get())));
-
-            // First node has reduced utilization and zero latency since shard0 has moved away
-            final var firstNodeUpdatedStats = updatedClusterInfo.getNodeUsageStatsForThreadPools()
-                .get(firstNode.getId())
-                .threadPoolUsageStatsMap()
-                .get(ThreadPool.Names.WRITE);
-            assertThat(
-                firstNodeUpdatedStats.averageThreadPoolUtilization(),
-                equalTo(
-                    initialThreadPoolStats.get(firstNode.getId())
-                        .threadPoolUsageStatsMap()
-                        .get(ThreadPool.Names.WRITE)
-                        .averageThreadPoolUtilization() - 1.0f
-                )
-            );
-            assertThat(firstNodeUpdatedStats.maxThreadPoolQueueLatencyMillis(), equalTo(0L));
-
-            // Second node has increased utilization since shard0 moved onto it. Latency does not change for incoming shards
-            final var secondNodeUpdatedStats = updatedClusterInfo.getNodeUsageStatsForThreadPools()
-                .get(secondNode.getId())
-                .threadPoolUsageStatsMap()
-                .get(ThreadPool.Names.WRITE);
-            assertThat(secondNodeUpdatedStats.averageThreadPoolUtilization(), equalTo(1.0f));
-            assertThat(secondNodeUpdatedStats.maxThreadPoolQueueLatencyMillis(), equalTo(0L));
-
-            // Third node has increased utilization since shard1 started on it
-            final var thirdNodeUpdatedStats = updatedClusterInfo.getNodeUsageStatsForThreadPools()
-                .get(thirdNode.getId())
-                .threadPoolUsageStatsMap()
-                .get(ThreadPool.Names.WRITE);
-            assertThat(thirdNodeUpdatedStats.averageThreadPoolUtilization(), equalTo(1.0f));
-
-            // 2. Reroute again and the simulated ClusterInfo remains the same due to no new change
-            rerouteAndWait(service, clusterService.state(), "reroute-with-desired-shard-movements");
-            assertThat(clusterInfoUsedByAllocator.get(), equalTo(updatedClusterInfo));
-
-            // 3. Wait until reconciliation is completed for the moved shard. This means shard0 is now relocating in the
-            // actual cluster state, i.e. not just in the desired balance assignments.
-            ClusterServiceUtils.addTemporaryStateListener(
-                clusterService,
-                state -> state.routingTable(ProjectId.DEFAULT).shardRoutingTable(shardId0).primaryShard().relocating()
-            );
-            // Reroute again and the ClusterInfo simulation result remains unchanged and correctly accounts for all shard movements
-            // (reconciled or not)
-            rerouteAndWait(service, clusterService.state(), "reroute-with-reconciled-shard-movement");
-            assertThat(clusterInfoUsedByAllocator.get(), equalTo(updatedClusterInfo));
-
-            // 4. Actually start the relocating shard0 on the target node-2, i.e. action to honour the reconciliation result
-            ClusterServiceUtils.setState(
-                clusterService,
-                service.applyStartedShards(
-                    clusterService.state(),
-                    List.of(
-                        clusterService.state()
-                            .routingTable(ProjectId.DEFAULT)
-                            .shardRoutingTable(shardId0)
-                            .primaryShard()
-                            .getTargetRelocatingShard()
-                    )
-                )
-            );
-            assertThat(
-                clusterService.state().routingTable(ProjectId.DEFAULT).shardRoutingTable(shardId0).primaryShard().currentNodeId(),
-                equalTo(secondNode.getId())
-            );
-            // The actually started shard0 is still accounted for when simulating ClusterInfo
-            rerouteAndWait(service, clusterService.state(), "reroute-with-actual-relocating-shard-started-event");
-            assertThat(clusterInfoUsedByAllocator.get(), equalTo(updatedClusterInfo));
-
-            // 5. Also start the initializing shard1 on the target node-3
-            ClusterServiceUtils.setState(
-                clusterService,
-                service.applyStartedShards(
-                    clusterService.state(),
-                    List.of(clusterService.state().routingTable(ProjectId.DEFAULT).shardRoutingTable(shardId1).primaryShard())
-                )
-            );
-            assertThat(
-                clusterService.state().routingTable(ProjectId.DEFAULT).shardRoutingTable(shardId1).primaryShard().currentNodeId(),
-                equalTo(thirdNode.getId())
-            );
-            // The actually started shard1 is accounted for when simulating ClusterInfo
-            rerouteAndWait(service, clusterService.state(), "reroute-with-actual-initializing-shard-started-event");
-            assertThat(clusterInfoUsedByAllocator.get(), equalTo(updatedClusterInfo));
-
-            // 6. A new ClusterInfo is polled
-            clusterInfoRef.set(
-                ClusterInfo.builder()
-                    .dataPath(
-                        Map.of(
-                            new ClusterInfo.NodeAndShard(secondNode.getId(), shardId0),
-                            "/data/path0",
-                            new ClusterInfo.NodeAndShard(thirdNode.getId(), shardId1),
-                            "/data/path1"
-                        )
-                    )
-                    .shardWriteLoads(Map.of(shardId0, 10.0d, shardId1, 1.0d))
-                    .nodeUsageStatsForThreadPools(
-                        Map.of(
-                            firstNode.getId(),
-                            new NodeUsageStatsForThreadPools(
-                                firstNode.getId(),
-                                Map.of(ThreadPool.Names.WRITE, new ThreadPoolUsageStats(10, 5.0f, 30))
-                            ),
-                            secondNode.getId(),
-                            new NodeUsageStatsForThreadPools(
-                                secondNode.getId(),
-                                Map.of(ThreadPool.Names.WRITE, new ThreadPoolUsageStats(10, 5.0f, 30))
-                            ),
-                            thirdNode.getId(),
-                            new NodeUsageStatsForThreadPools(
-                                secondNode.getId(),
-                                Map.of(ThreadPool.Names.WRITE, new ThreadPoolUsageStats(10, 1.0f, 5))
-                            )
-                        )
-                    )
-                    .build()
-            );
-            // ClusterInfo is updated from the new polling and no adjustment is applied onto the new ClusterInfo due to no movement since
-            rerouteAndWait(service, clusterService.state(), "reroute-after-new-cluster-info-polled");
-            assertThat(clusterInfoUsedByAllocator.get(), equalTo(clusterInfoRef.get()));
-        } finally {
-            clusterService.close();
-            terminate(threadPool);
-        }
-    }
-
     public void testResetDesiredBalanceOnNoLongerMaster() {
 
         var node1 = newNode(LOCAL_NODE_ID);