Dump recovery if fail to get doc count with preference (#40168)

With this change, we will dump the recovery state if we fail to get doc
count for a given index with a preference in rolling upgrade tests. We
should have more information to look into why the provided preference is
not valid. I also unmuted `testRelocationWithConcurrentIndexing` in this
change.

Relates #34950

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Request;
@@ -170,14 +171,27 @@ public class RecoveryIT extends AbstractRollingTestCase {
     }
 
     private void assertCount(final String index, final String preference, final int expectedCount) throws IOException {
-        final Request request = new Request("GET", index + "/_count");
-        request.addParameter("preference", preference);
-        final Response response = client().performRequest(request);
-        final int actualCount = Integer.parseInt(ObjectPath.createFromResponse(response).evaluate("count").toString());
-        assertThat("preference [" + preference + "]", actualCount, equalTo(expectedCount));
+        final int actualDocs;
+        try {
+            final Request request = new Request("GET", index + "/_count");
+            if (preference != null) {
+                request.addParameter("preference", preference);
+            }
+            final Response response = client().performRequest(request);
+            actualDocs = Integer.parseInt(ObjectPath.createFromResponse(response).evaluate("count").toString());
+        } catch (ResponseException e) {
+            try {
+                final Response recoveryStateResponse = client().performRequest(new Request("GET", index + "/_recovery"));
+                fail("failed to get doc count for index [" + index + "] with preference [" + preference + "]" + " response [" + e + "]"
+                    + " recovery [" + EntityUtils.toString(recoveryStateResponse.getEntity()) + "]");
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw e;
+        }
+        assertThat("preference [" + preference + "]", actualDocs, equalTo(expectedCount));
     }
 
-
     private String getNodeId(Predicate<Version> versionPredicate) throws IOException {
         Response response = client().performRequest(new Request("GET", "_nodes"));
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
@@ -191,7 +205,6 @@ public class RecoveryIT extends AbstractRollingTestCase {
         return null;
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34950")
     public void testRelocationWithConcurrentIndexing() throws Exception {
         final String index = "relocation_with_concurrent_indexing";
         switch (CLUSTER_TYPE) {