Reproducible randoms in SearchWhileCreatingIndexIT

This commit removes non-reproducible randomness from
SearchWhileCreatingIndexIT. The cause of the non-reproducible randomness
is the use of a random draws for the shard preference inside of a
non-deterministic while loop. Because the inner while loop executed a
non-deterministic number of times, the draws for the next iteration of
the outer loop would be impacted by this making the random draws
non-reproducible. The solution is to move the random draws outside of
the while loop (just make a single draw for the prefernce and increment
with a counter), and remove the outer loop iteration instead using an
annotation to get the desired repetitions.

Closes #16208

core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java

@@ -29,7 +29,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
-
 /**
  * This test basically verifies that search with a single shard active (cause we indexed to it) and other
  * shards possibly not active at all (cause they haven't allocated) will still work.
@@ -58,39 +57,44 @@ public class SearchWhileCreatingIndexIT extends ESIntegTestCase {
         int shardsNo = numberOfReplicas + 1;
         int neededNodes = shardsNo <= 2 ? 1 : shardsNo / 2 + 1;
         internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(neededNodes, shardsNo));
-        for (int i = 0; i < 20; i++) {
-            logger.info("running iteration {}", i);
-            if (createIndex) {
-                createIndex("test");
-            }
-            client().prepareIndex("test", "type1", randomAsciiOfLength(5)).setSource("field", "test").execute().actionGet();
-            RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().actionGet();
-            assertThat(refreshResponse.getSuccessfulShards(), greaterThanOrEqualTo(1)); // at least one shard should be successful when refreshing
 
-            // we want to make sure that while recovery happens, and a replica gets recovered, its properly refreshed
-            ClusterHealthStatus status = ClusterHealthStatus.RED;
-            while (status != ClusterHealthStatus.GREEN) {
-                // first, verify that search on the primary search works
-                SearchResponse searchResponse = client().prepareSearch("test").setPreference("_primary").setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
-                assertHitCount(searchResponse, 1);
-                // now, let it go to primary or replica, though in a randomized re-creatable manner
-                String preference = randomAsciiOfLength(5);
-                Client client = client();
-                searchResponse = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
-                if (searchResponse.getHits().getTotalHits() != 1) {
-                    refresh();
-                    SearchResponse searchResponseAfterRefresh = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
-                    logger.info("hits count mismatch on any shard search failed, post explicit refresh hits are {}", searchResponseAfterRefresh.getHits().getTotalHits());
-                    ensureGreen();
-                    SearchResponse searchResponseAfterGreen = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
-                    logger.info("hits count mismatch on any shard search failed, post explicit wait for green hits are {}", searchResponseAfterGreen.getHits().getTotalHits());
-                    assertHitCount(searchResponse, 1);
-                }
+        String id = randomAsciiOfLength(5);
+        // we will go the primary or the replica, but in a
+        // randomized re-creatable manner
+        int counter = 0;
+        String preference = randomAsciiOfLength(5);
+
+        logger.info("running iteration for id {}, preference {}", id, preference);
+
+        if (createIndex) {
+            createIndex("test");
+        }
+        client().prepareIndex("test", "type1", id).setSource("field", "test").execute().actionGet();
+        RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().actionGet();
+        assertThat(refreshResponse.getSuccessfulShards(), greaterThanOrEqualTo(1)); // at least one shard should be successful when refreshing
+
+        logger.info("using preference {}", preference);
+        // we want to make sure that while recovery happens, and a replica gets recovered, its properly refreshed
+        ClusterHealthStatus status = ClusterHealthStatus.RED;
+        while (status != ClusterHealthStatus.GREEN) {
+            // first, verify that search on the primary search works
+            SearchResponse searchResponse = client().prepareSearch("test").setPreference("_primary").setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
+            assertHitCount(searchResponse, 1);
+            Client client = client();
+            searchResponse = client.prepareSearch("test").setPreference(preference + Integer.toString(counter++)).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
+            if (searchResponse.getHits().getTotalHits() != 1) {
+                refresh();
+                SearchResponse searchResponseAfterRefresh = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
+                logger.info("hits count mismatch on any shard search failed, post explicit refresh hits are {}", searchResponseAfterRefresh.getHits().getTotalHits());
+                ensureGreen();
+                SearchResponse searchResponseAfterGreen = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
+                logger.info("hits count mismatch on any shard search failed, post explicit wait for green hits are {}", searchResponseAfterGreen.getHits().getTotalHits());
                 assertHitCount(searchResponse, 1);
-                status = client().admin().cluster().prepareHealth("test").get().getStatus();
-                internalCluster().ensureAtLeastNumDataNodes(numberOfReplicas + 1);
             }
-            cluster().wipeIndices("test");
+            assertHitCount(searchResponse, 1);
+            status = client().admin().cluster().prepareHealth("test").get().getStatus();
+            internalCluster().ensureAtLeastNumDataNodes(numberOfReplicas + 1);
         }
+        cluster().wipeIndices("test");
     }
 }