Increase time to wait for green in rolling upgrade tests (#26781)

This commit increases the amount of time to wait for green to accound for unassigned shards that
have been delayed. The default delay is 60s, so we need to wait longer than that. Previously, the
wait would timeout at 30s due to the rest client and the default for the cluster health api.

Closes #26742

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -74,10 +74,11 @@ public class RecoveryIT extends ESRestTestCase {
     @Override
     protected Settings restClientSettings() {
         return Settings.builder().put(super.restClientSettings())
-            // increase the timeout so that we can actually see the result of failed cluster health
-            // calls that have a default timeout of 30s
-            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "40s")
-            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "40s")
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s")
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
             .build();
     }
 
@@ -89,6 +90,8 @@ public class RecoveryIT extends ESRestTestCase {
         Map<String, String> params = new HashMap<>();
         params.put("wait_for_status", "green");
         params.put("wait_for_no_relocating_shards", "true");
+        params.put("timeout", "70s");
+        params.put("level", "shards");
         assertOK(client().performRequest("GET", "_cluster/health", params));
     }
 

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -52,10 +52,11 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     @Override
     protected Settings restClientSettings() {
         return Settings.builder().put(super.restClientSettings())
-            // increase the timeout so that we can actually see the result of failed cluster health
-            // calls that have a default timeout of 30s
-            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "40s")
-            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "40s")
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s")
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
             .build();
     }
 }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -4,6 +4,9 @@
      cluster.health:
         wait_for_status: green
         wait_for_nodes: 2
+        # wait for long enough that we give delayed unassigned shards to stop being delayed
+        timeout: 70s
+        level: shards
 
  - do:
      search: