[9.1] Set default processor allocation for test clusters (#133204) (#133212)

* Set default processor allocation for test clusters (#133204)

Setting the number of processors available for node to adequately size
thread pools.
Related to #130612

* Lowering the allocations during update to fit within available processors

---------

Co-authored-by: Jonathan Buttner <jonathan.buttner@elastic.co>

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -34,12 +34,13 @@ public class InternalTestClustersPlugin implements Plugin<Project> {
                 || buildParams.getBwcVersions().unreleasedInfo(version) == null
         );
 
-        if (shouldConfigureTestClustersWithOneProcessor()) {
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", "1"));
-        }
+        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+            .getExtensions()
+            .getByName(TestClustersPlugin.EXTENSION_NAME);
+        // Limit the number of allocated processors for all nodes to 2 in the cluster by default.
+        // This is to ensure that the tests run consistently across different environments.
+        String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
+        testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", processorCount));
     }
 
     private boolean shouldConfigureTestClustersWithOneProcessor() {

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java

@@ -42,6 +42,10 @@ public class DefaultSettingsProvider implements SettingsProvider {
             }
         }
 
+        // Limit the number of allocated processors for all nodes in the cluster by default.
+        // This is to ensure that the tests run consistently across different environments.
+        settings.put("node.processors", "2");
+
         // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
         settings.put("cluster.routing.allocation.disk.watermark.low", "1b");
         settings.put("cluster.routing.allocation.disk.watermark.high", "1b");

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java

@@ -51,7 +51,7 @@ public class CreateFromDeploymentIT extends InferenceBaseRestTest {
         var results = infer(inferenceId, List.of("washing machine"));
         assertNotNull(results.get("sparse_embedding"));
 
-        var updatedNumAllocations = randomIntBetween(1, 10);
+        var updatedNumAllocations = randomIntBetween(1, 2);
         var updatedEndpointConfig = updateEndpoint(inferenceId, updatedEndpointConfig(updatedNumAllocations), TaskType.SPARSE_EMBEDDING);
         assertThat(
             updatedEndpointConfig.get("service_settings"),
@@ -128,7 +128,7 @@ public class CreateFromDeploymentIT extends InferenceBaseRestTest {
         var results = infer(inferenceId, List.of("washing machine"));
         assertNotNull(results.get("sparse_embedding"));
 
-        var updatedNumAllocations = randomIntBetween(1, 10);
+        var updatedNumAllocations = randomIntBetween(1, 2);
         var updatedEndpointConfig = updateEndpoint(inferenceId, updatedEndpointConfig(updatedNumAllocations), TaskType.SPARSE_EMBEDDING);
         assertThat(
             updatedEndpointConfig.get("service_settings"),