Make some Allocation Decider Code a Little More JIT Aware (#62275)

When investigating our code cache usage for another issue I ran into this. This PR just fixes a few spots and there's many more. The current way we compute the decisions often creates much larger than necessary methods because the compiler has no efficient way of optimizing away things like using CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey() as an explain parameter that do allocations (but whose results are thrown away immediately if debug is off).

As a result e.g. the max retry allocation decider's canAllocate compiles into an 18kb method before and into a 2kb method after this change (at C1 L3). I think if we're a little more mindful of the JIT here we can get some measurable speedups out of the allocation deciders logic. Plus, this kind of change saves quite a few in allocations in isolation as well which is always nice on a hot CS thread I suppose.

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java

@@ -50,9 +50,9 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canRebalance(shardRouting, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -72,14 +72,14 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canAllocate(shardRouting, node, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Can not allocate [{}] on node [{}] due to [{}]",
                         shardRouting, node.node(), allocationDecider.getClass().getSimpleName());
                 }
                 // short circuit only if debugging is not enabled
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -102,13 +102,13 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canRemain(shardRouting, node, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Shard [{}] can not remain on node [{}] due to [{}]",
                         shardRouting, node.nodeId(), allocationDecider.getClass().getSimpleName());
                 }
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -125,9 +125,9 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canAllocate(indexMetadata, node, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -144,9 +144,9 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.shouldAutoExpandToNode(indexMetadata, node, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -163,9 +163,9 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canAllocate(shardRouting, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -182,9 +182,9 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider allocationDecider : allocations) {
             Decision decision = allocationDecider.canRebalance(allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }
@@ -206,13 +206,13 @@ public class AllocationDeciders extends AllocationDecider {
         for (AllocationDecider decider : allocations) {
             Decision decision = decider.canForceAllocatePrimary(shardRouting, node, allocation);
             // short track if a NO is returned.
-            if (decision == Decision.NO) {
+            if (decision.type() == Decision.Type.NO) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Shard [{}] can not be forcefully allocated to node [{}] due to [{}].",
                         shardRouting.shardId(), node.nodeId(), decider.getClass().getSimpleName());
                 }
                 if (!allocation.debugDecision()) {
-                    return decision;
+                    return Decision.NO;
                 } else {
                     ret.add(decision);
                 }

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java

@@ -125,22 +125,25 @@ public class AwarenessAllocationDecider extends AllocationDecider {
         return underCapacity(shardRouting, node, allocation, false);
     }
 
+    private static final Decision YES_NOT_ENABLED = Decision.single(Decision.Type.YES, NAME,
+            "allocation awareness is not enabled, set cluster setting ["
+                    + CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey() + "] to enable it");
+
+    private static final Decision YES_ALL_MET =
+            Decision.single(Decision.Type.YES, NAME, "node meets all awareness attribute requirements");
+
     private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
         if (awarenessAttributes.isEmpty()) {
-            return allocation.decision(Decision.YES, NAME,
-                "allocation awareness is not enabled, set cluster setting [%s] to enable it",
-                CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey());
+            return YES_NOT_ENABLED;
         }
 
+        final boolean debug = allocation.debugDecision();
         IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
         int shardCount = indexMetadata.getNumberOfReplicas() + 1; // 1 for primary
         for (String awarenessAttribute : awarenessAttributes) {
             // the node the shard exists on must be associated with an awareness attribute
             if (node.node().getAttributes().containsKey(awarenessAttribute) == false) {
-                return allocation.decision(Decision.NO, NAME,
-                    "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]",
-                    awarenessAttribute, CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
-                    allocation.debugDecision() ? Strings.collectionToCommaDelimitedString(awarenessAttributes) : null);
+                return debug ? debugNoMissingAttribute(awarenessAttribute, awarenessAttributes) : Decision.NO;
             }
 
             // build attr_value -> nodes map
@@ -185,18 +188,31 @@ public class AwarenessAllocationDecider extends AllocationDecider {
             final int currentNodeCount = shardPerAttribute.get(node.node().getAttributes().get(awarenessAttribute));
             final int maximumNodeCount = (shardCount + numberOfAttributes - 1) / numberOfAttributes; // ceil(shardCount/numberOfAttributes)
             if (currentNodeCount > maximumNodeCount) {
-                return allocation.decision(Decision.NO, NAME,
-                        "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
-                        "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
-                        "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
-                        awarenessAttribute,
-                        shardCount,
-                        numberOfAttributes,
-                        currentNodeCount,
-                        maximumNodeCount);
+                return debug ? debugNoTooManyCopies(shardCount, awarenessAttribute, numberOfAttributes, currentNodeCount, maximumNodeCount)
+                        : Decision.NO;
             }
         }
 
-        return allocation.decision(Decision.YES, NAME, "node meets all awareness attribute requirements");
+        return YES_ALL_MET;
+    }
+
+    private static Decision debugNoTooManyCopies(int shardCount, String awarenessAttribute, int numberOfAttributes, int currentNodeCount,
+                                                 int maximumNodeCount) {
+        return Decision.single(Decision.Type.NO, NAME,
+                "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
+                        "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
+                        "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
+                awarenessAttribute,
+                shardCount,
+                numberOfAttributes,
+                currentNodeCount,
+                maximumNodeCount);
+    }
+
+    private static Decision debugNoMissingAttribute(String awarenessAttribute, List<String> awarenessAttributes) {
+        return Decision.single(Decision.Type.NO, NAME,
+                "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]", awarenessAttribute,
+                CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
+                Strings.collectionToCommaDelimitedString(awarenessAttributes));
     }
 }

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java

@@ -23,6 +23,7 @@ import java.util.Locale;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -109,40 +110,55 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider {
         return canRebalance(allocation);
     }
 
+    private static final Decision YES_ALL_PRIMARIES_ACTIVE = Decision.single(Decision.Type.YES, NAME, "all primary shards are active");
+
+    private static final Decision YES_ALL_SHARDS_ACTIVE = Decision.single(Decision.Type.YES, NAME, "all shards are active");
+
+    private static final Decision NO_UNASSIGNED_PRIMARIES = Decision.single(Decision.Type.NO, NAME,
+            "the cluster has unassigned primary shards and cluster setting ["
+                    + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE + "] is set to [" + ClusterRebalanceType.INDICES_PRIMARIES_ACTIVE + "]");
+
+    private static final Decision NO_INACTIVE_PRIMARIES = Decision.single(Decision.Type.NO, NAME,
+            "the cluster has inactive primary shards and cluster setting [" + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE +
+                    "] is set to [" + ClusterRebalanceType.INDICES_PRIMARIES_ACTIVE + "]");
+
+    private static final Decision NO_UNASSIGNED_SHARDS = Decision.single(Decision.Type.NO, NAME,
+            "the cluster has unassigned shards and cluster setting [" + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE +
+                    "] is set to [" + ClusterRebalanceType.INDICES_ALL_ACTIVE + "]");
+
+    private static final Decision NO_INACTIVE_SHARDS = Decision.single(Decision.Type.NO, NAME,
+            "the cluster has inactive shards and cluster setting [" + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE +
+                    "] is set to [" + ClusterRebalanceType.INDICES_ALL_ACTIVE + "]");
+
+    @SuppressWarnings("fallthrough")
     @Override
     public Decision canRebalance(RoutingAllocation allocation) {
-        if (type == ClusterRebalanceType.INDICES_PRIMARIES_ACTIVE) {
-            // check if there are unassigned primaries.
-            if ( allocation.routingNodes().hasUnassignedPrimaries() ) {
-                return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned primary shards and cluster setting [%s] is set to [%s]",
-                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
-            }
-            // check if there are initializing primaries that don't have a relocatingNodeId entry.
-            if ( allocation.routingNodes().hasInactivePrimaries() ) {
-                return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive primary shards and cluster setting [%s] is set to [%s]",
-                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
-            }
-
-            return allocation.decision(Decision.YES, NAME, "all primary shards are active");
-        }
-        if (type == ClusterRebalanceType.INDICES_ALL_ACTIVE) {
-            // check if there are unassigned shards.
-            if (allocation.routingNodes().hasUnassignedShards() ) {
-                return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned shards and cluster setting [%s] is set to [%s]",
-                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
-            }
-            // in case all indices are assigned, are there initializing shards which
-            // are not relocating?
-            if ( allocation.routingNodes().hasInactiveShards() ) {
-                return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive shards and cluster setting [%s] is set to [%s]",
-                        CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
-            }
+        final RoutingNodes routingNodes = allocation.routingNodes();
+        switch (type) {
+            case INDICES_PRIMARIES_ACTIVE:
+                // check if there are unassigned primaries.
+                if (routingNodes.hasUnassignedPrimaries()) {
+                    return NO_UNASSIGNED_PRIMARIES;
+                }
+                // check if there are initializing primaries that don't have a relocatingNodeId entry.
+                if (routingNodes.hasInactivePrimaries()) {
+                    return NO_INACTIVE_PRIMARIES;
+                }
+                return YES_ALL_PRIMARIES_ACTIVE;
+            case INDICES_ALL_ACTIVE:
+                // check if there are unassigned shards.
+                if (routingNodes.hasUnassignedShards()) {
+                    return NO_UNASSIGNED_SHARDS;
+                }
+                // in case all indices are assigned, are there initializing shards which
+                // are not relocating?
+                if (routingNodes.hasInactiveShards()) {
+                    return NO_INACTIVE_SHARDS;
+                }
+                // fall-through
+            default:
+                // all shards active from above or type == Type.ALWAYS
+                return YES_ALL_SHARDS_ACTIVE;
         }
-        // type == Type.ALWAYS
-        return allocation.decision(Decision.YES, NAME, "all shards are active");
     }
 }

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java

@@ -71,11 +71,20 @@ public abstract class Decision implements ToXContent, Writeable {
             }
             return result;
         } else {
-            Single result = new Single();
-            result.type = Type.readFrom(in);
-            result.label = in.readOptionalString();
-            result.explanationString = in.readOptionalString();
-            return result;
+            final Type type = Type.readFrom(in);
+            final String label = in.readOptionalString();
+            final String explanation = in.readOptionalString();
+            if (label == null && explanation == null) {
+                switch (type) {
+                    case YES:
+                        return YES;
+                    case THROTTLE:
+                        return THROTTLE;
+                    case NO:
+                        return NO;
+                }
+            }
+            return new Single(type, label, explanation);
         }
     }
 
@@ -153,21 +162,15 @@ public abstract class Decision implements ToXContent, Writeable {
      * Simple class representing a single decision
      */
     public static class Single extends Decision implements ToXContentObject {
-        private Type type;
-        private String label;
-        private String explanation;
-        private String explanationString;
-        private Object[] explanationParams;
-
-        public Single() {
-
-        }
+        private final Type type;
+        private final String label;
+        private final String explanationString;
 
         /**
          * Creates a new {@link Single} decision of a given type
          * @param type {@link Type} of the decision
          */
-        public Single(Type type) {
+        private Single(Type type) {
             this(type, null, null, (Object[]) null);
         }
 
@@ -181,8 +184,11 @@ public abstract class Decision implements ToXContent, Writeable {
         public Single(Type type, @Nullable String label, @Nullable String explanation, @Nullable Object... explanationParams) {
             this.type = type;
             this.label = label;
-            this.explanation = explanation;
-            this.explanationParams = explanationParams;
+            if (explanationParams != null && explanationParams.length > 0) {
+                this.explanationString = String.format(Locale.ROOT, explanation, explanationParams);
+            } else {
+                this.explanationString = explanation;
+            }
         }
 
         @Override
@@ -207,9 +213,6 @@ public abstract class Decision implements ToXContent, Writeable {
         @Override
         @Nullable
         public String getExplanation() {
-            if (explanationString == null && explanation != null) {
-                explanationString = String.format(Locale.ROOT, explanation, explanationParams);
-            }
             return this.explanationString;
         }
 
@@ -226,22 +229,22 @@ public abstract class Decision implements ToXContent, Writeable {
             Decision.Single s = (Decision.Single) object;
             return this.type == s.type &&
                        Objects.equals(label, s.label) &&
-                       Objects.equals(getExplanation(), s.getExplanation());
+                       Objects.equals(explanationString, s.explanationString);
         }
 
         @Override
         public int hashCode() {
             int result = type.hashCode();
             result = 31 * result + (label == null ? 0 : label.hashCode());
-            String explanationStr = getExplanation();
+            String explanationStr = explanationString;
             result = 31 * result + (explanationStr == null ? 0 : explanationStr.hashCode());
             return result;
         }
 
         @Override
         public String toString() {
-            if (explanationString != null || explanation != null) {
-                return type + "(" + getExplanation() + ")";
+            if (explanationString != null) {
+                return type + "(" + explanationString + ")";
             }
             return type + "()";
         }
@@ -251,8 +254,7 @@ public abstract class Decision implements ToXContent, Writeable {
             builder.startObject();
             builder.field("decider", label);
             builder.field("decision", type);
-            String explanation = getExplanation();
-            builder.field("explanation", explanation != null ? explanation : "none");
+            builder.field("explanation", explanationString != null ? explanationString : "none");
             builder.endObject();
             return builder;
         }
@@ -264,7 +266,7 @@ public abstract class Decision implements ToXContent, Writeable {
             out.writeOptionalString(label);
             // Flatten explanation on serialization, so that explanationParams
             // do not need to be serialized
-            out.writeOptionalString(getExplanation());
+            out.writeOptionalString(explanationString);
         }
     }
 

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java

@@ -41,25 +41,35 @@ public class MaxRetryAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "max_retry";
 
+    private static final Decision YES_NO_FAILURES = Decision.single(Decision.Type.YES, NAME, "shard has no previous failures");
+
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
         final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
-        final Decision decision;
-        if (unassignedInfo != null && unassignedInfo.getNumFailedAllocations() > 0) {
-            final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
-            final int maxRetry = SETTING_ALLOCATION_MAX_RETRY.get(indexMetadata.getSettings());
-            if (unassignedInfo.getNumFailedAllocations() >= maxRetry) {
-                decision = allocation.decision(Decision.NO, NAME, "shard has exceeded the maximum number of retries [%d] on " +
-                    "failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [%s]",
+        final int numFailedAllocations = unassignedInfo == null ? 0 : unassignedInfo.getNumFailedAllocations();
+        if (numFailedAllocations > 0) {
+            return decisionWithFailures(shardRouting, allocation, unassignedInfo, numFailedAllocations);
+        }
+        return YES_NO_FAILURES;
+    }
+
+    private static Decision decisionWithFailures(ShardRouting shardRouting, RoutingAllocation allocation, UnassignedInfo unassignedInfo,
+                                                 int numFailedAllocations) {
+        final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
+        final int maxRetry = SETTING_ALLOCATION_MAX_RETRY.get(indexMetadata.getSettings());
+        final Decision res = numFailedAllocations >= maxRetry ? Decision.NO : Decision.YES;
+        return allocation.debugDecision() ? debugDecision(res, unassignedInfo, numFailedAllocations, maxRetry) : res;
+    }
+
+    private static Decision debugDecision(Decision decision, UnassignedInfo unassignedInfo, int numFailedAllocations, int maxRetry) {
+        if (decision.type() == Decision.Type.YES) {
+            return Decision.single(Decision.Type.NO, NAME, "shard has exceeded the maximum number of retries [%d] on " +
+                            "failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [%s]",
                     maxRetry, unassignedInfo.toString());
-            } else {
-                decision = allocation.decision(Decision.YES, NAME, "shard has failed allocating [%d] times but [%d] retries are allowed",
-                    unassignedInfo.getNumFailedAllocations(), maxRetry);
-            }
         } else {
-            decision = allocation.decision(Decision.YES, NAME, "shard has no previous failures");
+            return Decision.single(Decision.Type.YES, NAME, "shard has failed allocating [%d] times but [%d] retries are allowed",
+                    numFailedAllocations, maxRetry);
         }
-        return decision;
     }
 
     @Override

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java

@@ -66,6 +66,9 @@ public class SameShardAllocationDecider extends AllocationDecider {
         this.sameHost = sameHost;
     }
 
+    private static final Decision YES_NONE_HOLD_COPY =
+            Decision.single(Decision.Type.YES, NAME, "none of the nodes on this host hold a copy of this shard");
+
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting.shardId());
@@ -94,18 +97,24 @@ public class SameShardAllocationDecider extends AllocationDecider {
                 if (checkNodeOnSameHostAddress || checkNodeOnSameHostName) {
                     for (ShardRouting assignedShard : assignedShards) {
                         if (checkNode.nodeId().equals(assignedShard.currentNodeId())) {
-                            String hostType = checkNodeOnSameHostAddress ? "address" : "name";
-                            String host = checkNodeOnSameHostAddress ? node.node().getHostAddress() : node.node().getHostName();
-                            return allocation.decision(Decision.NO, NAME,
-                                "a copy of this shard is already allocated to host %s [%s], on node [%s], and [%s] is [true] which " +
-                                    "forbids more than one node on this host from holding a copy of this shard",
-                                hostType, host, node.nodeId(), CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey());
+                            return allocation.debugDecision() ?
+                                    debugNoAlreadyAllocatedToHost(node, allocation, checkNodeOnSameHostAddress) : Decision.NO;
                         }
                     }
                 }
             }
         }
-        return allocation.decision(Decision.YES, NAME, "none of the nodes on this host hold a copy of this shard");
+        return YES_NONE_HOLD_COPY;
+    }
+
+    private static Decision debugNoAlreadyAllocatedToHost(RoutingNode node, RoutingAllocation allocation,
+                                                          boolean checkNodeOnSameHostAddress) {
+        String hostType = checkNodeOnSameHostAddress ? "address" : "name";
+        String host = checkNodeOnSameHostAddress ? node.node().getHostAddress() : node.node().getHostName();
+        return allocation.decision(Decision.NO, NAME,
+            "a copy of this shard is already allocated to host %s [%s], on node [%s], and [%s] is [true] which " +
+                "forbids more than one node on this host from holding a copy of this shard",
+            hostType, host, node.nodeId(), CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey());
     }
 
     @Override
@@ -115,21 +124,26 @@ public class SameShardAllocationDecider extends AllocationDecider {
         return decideSameNode(shardRouting, node, allocation, assignedShards);
     }
 
+    private static final Decision YES_NO_COPY = Decision.single(Decision.Type.YES, NAME, "this node does not hold a copy of this shard");
+
     private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
                                     Iterable<ShardRouting> assignedShards) {
+        boolean debug = allocation.debugDecision();
         for (ShardRouting assignedShard : assignedShards) {
             if (node.nodeId().equals(assignedShard.currentNodeId())) {
-                if (assignedShard.isSameAllocation(shardRouting)) {
-                    return allocation.decision(Decision.NO, NAME,
-                        "this shard is already allocated to this node [%s]",
-                        shardRouting.toString());
-                } else {
-                    return allocation.decision(Decision.NO, NAME,
-                        "a copy of this shard is already allocated to this node [%s]",
-                        assignedShard.toString());
-                }
+                return debug ? debugNo(shardRouting, assignedShard) : Decision.NO;
             }
         }
-        return allocation.decision(Decision.YES, NAME, "this node does not hold a copy of this shard");
+        return YES_NO_COPY;
+    }
+
+    private static Decision debugNo(ShardRouting shardRouting, ShardRouting assignedShard) {
+        final String explanation;
+        if (assignedShard.isSameAllocation(shardRouting)) {
+            explanation = "this shard is already allocated to this node [" + shardRouting.toString() + "]";
+        } else {
+            explanation = "a copy of this shard is already allocated to this node [" + assignedShard.toString() + "]";
+        }
+        return Decision.single(Decision.Type.NO, NAME, explanation);
     }
 }

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -2684,13 +2684,15 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             || currentRouting.primary() != newRouting.primary()
             || currentRouting.allocationId().equals(newRouting.allocationId()) == false) {
             assert currentRouting == null || currentRouting.isSameAllocation(newRouting);
-            final String writeReason;
-            if (currentRouting == null) {
-                writeReason = "initial state with allocation id [" + newRouting.allocationId() + "]";
-            } else {
-                writeReason = "routing changed from " + currentRouting + " to " + newRouting;
+            if (logger.isTraceEnabled()) {
+                final String writeReason;
+                if (currentRouting == null) {
+                    writeReason = "initial state with allocation id [" + newRouting.allocationId() + "]";
+                } else {
+                    writeReason = "routing changed from " + currentRouting + " to " + newRouting;
+                }
+                logger.trace("{} writing shard state, reason [{}]", shardId, writeReason);
             }
-            logger.trace("{} writing shard state, reason [{}]", shardId, writeReason);
             final ShardStateMetadata newShardStateMetadata =
                     new ShardStateMetadata(newRouting.primary(), indexSettings.getUUID(), newRouting.allocationId());
             ShardStateMetadata.FORMAT.writeAndCleanup(newShardStateMetadata, shardPath.getShardStatePath());

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java

@@ -117,4 +117,130 @@ public class AllocationDecidersTests extends ESTestCase {
         Decision.Multi multi = (Decision.Multi) decision;
         assertThat(multi.getDecisions(), matcher);
     }
+
+    public void testEarlyTermination() {
+        final Decision decisionOne = randomFrom(Decision.NO, Decision.single(Decision.Type.NO, "label1", "explanation"));
+        final Decision decisionTwo = randomFrom(Decision.NO, Decision.single(Decision.Type.NO, "label2", "explanation"));
+        final AllocationDeciders allocationDeciders = new AllocationDeciders(List.of(
+                new AllocationDecider() {
+                    @Override
+                    public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canRebalance(RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+
+                    @Override
+                    public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decisionOne;
+                    }
+                }, new AllocationDecider() {
+
+                    @Override
+                    public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canRebalance(RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    @Override
+                    public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                        return decision(allocation);
+                    }
+
+                    private Decision decision(RoutingAllocation allocation) {
+                        if (allocation.debugDecision() == false) {
+                            throw new AssertionError("Should not be called");
+                        }
+                        return decisionTwo;
+                    }
+                }));
+
+        // no debug should just short-circuit to no, no matter what kind of no type return the first decider returns
+        final ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId("test", "testUUID", 0), true,
+                RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"));
+        final RoutingNode routingNode = new RoutingNode("testNode", null);
+        final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build();
+        final IndexMetadata indexMetadata =
+                IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
+
+        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders,
+                clusterState.getRoutingNodes(), clusterState, null, null,0L);
+        assertSame(Decision.NO, allocationDeciders.canAllocate(shardRouting, routingNode, allocation));
+        assertSame(Decision.NO, allocationDeciders.canRebalance(shardRouting, allocation));
+        assertSame(Decision.NO, allocationDeciders.canRemain(shardRouting, routingNode, allocation));
+        assertSame(Decision.NO, allocationDeciders.canAllocate(shardRouting, allocation));
+        assertSame(Decision.NO, allocationDeciders.canAllocate(indexMetadata, routingNode, allocation));
+        assertSame(Decision.NO, allocationDeciders.shouldAutoExpandToNode(indexMetadata, null, allocation));
+        assertSame(Decision.NO, allocationDeciders.canRebalance(allocation));
+        assertSame(Decision.NO, allocationDeciders.canForceAllocatePrimary(shardRouting, routingNode, allocation));
+
+        // debug decision should contain both individual decisions in a multi-decision
+        allocation.debugDecision(true);
+        final Decision expectedDebugDecision = new Decision.Multi().add(decisionOne).add(decisionTwo);
+        assertEquals(expectedDebugDecision, allocationDeciders.canAllocate(shardRouting, routingNode, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canRebalance(shardRouting, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canRemain(shardRouting, routingNode, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canAllocate(shardRouting, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canAllocate(indexMetadata, routingNode, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.shouldAutoExpandToNode(indexMetadata, null, allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canRebalance(allocation));
+        assertEquals(expectedDebugDecision, allocationDeciders.canForceAllocatePrimary(shardRouting, routingNode, allocation));
+    }
 }