Add more logging when failing watch history entry fails. (#50931)

Relates to #30777

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -305,39 +305,76 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
 
     private ObjectPath getWatchHistoryEntry(String watchId, String state) throws Exception {
         final AtomicReference<ObjectPath> objectPathReference = new AtomicReference<>();
-        assertBusy(() -> {
-            client().performRequest(new Request("POST", "/.watcher-history-*/_refresh"));
-
-            try (XContentBuilder builder = jsonBuilder()) {
-                builder.startObject();
-                builder.startObject("query").startObject("bool").startArray("must");
-                builder.startObject().startObject("term").startObject("watch_id").field("value", watchId).endObject().endObject()
+        try {
+            assertBusy(() -> {
+                client().performRequest(new Request("POST", "/.watcher-history-*/_refresh"));
+
+                try (XContentBuilder builder = jsonBuilder()) {
+                    builder.startObject();
+                    builder.startObject("query").startObject("bool").startArray("must");
+                    builder.startObject().startObject("term").startObject("watch_id").field("value", watchId).endObject().endObject()
                         .endObject();
-                if (Strings.isNullOrEmpty(state) == false) {
-                    builder.startObject().startObject("term").startObject("state").field("value", state).endObject().endObject()
+                    if (Strings.isNullOrEmpty(state) == false) {
+                        builder.startObject().startObject("term").startObject("state").field("value", state).endObject().endObject()
                             .endObject();
-                }
-                builder.endArray().endObject().endObject();
-                builder.startArray("sort").startObject().startObject("trigger_event.triggered_time").field("order", "desc").endObject()
+                    }
+                    builder.endArray().endObject().endObject();
+                    builder.startArray("sort").startObject().startObject("trigger_event.triggered_time").field("order", "desc").endObject()
                         .endObject().endArray();
-                builder.endObject();
-
-                Request searchRequest = new Request("POST", "/.watcher-history-*/_search");
-                searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
-                searchRequest.setJsonEntity(Strings.toString(builder));
-                Response response = client().performRequest(searchRequest);
-                ObjectPath objectPath = ObjectPath.createFromResponse(response);
-                int totalHits = objectPath.evaluate("hits.total");
-                assertThat(totalHits, is(greaterThanOrEqualTo(1)));
-                String watchid = objectPath.evaluate("hits.hits.0._source.watch_id");
-                assertThat(watchid, is(watchId));
-                objectPathReference.set(objectPath);
-            } catch (ResponseException e) {
-                final String err = "Failed to perform search of watcher history";
-                logger.info(err, e);
-                fail(err);
+                    builder.endObject();
+
+                    Request searchRequest = new Request("POST", "/.watcher-history-*/_search");
+                    searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
+                    searchRequest.setJsonEntity(Strings.toString(builder));
+                    Response response = client().performRequest(searchRequest);
+                    ObjectPath objectPath = ObjectPath.createFromResponse(response);
+                    int totalHits = objectPath.evaluate("hits.total");
+                    assertThat(totalHits, is(greaterThanOrEqualTo(1)));
+                    String watchid = objectPath.evaluate("hits.hits.0._source.watch_id");
+                    assertThat(watchid, is(watchId));
+                    objectPathReference.set(objectPath);
+                } catch (ResponseException e) {
+                    final String err = "Failed to perform search of watcher history";
+                    logger.info(err, e);
+                    fail(err);
+                }
+            });
+        } catch (AssertionError ae) {
+            {
+                Request request = new Request("GET", "/_watcher/stats");
+                request.addParameter("metric", "_all");
+                request.addParameter("pretty", "true");
+                try {
+                    Response response = client().performRequest(request);
+                    logger.info("watcher_stats: {}", EntityUtils.toString(response.getEntity()));
+                } catch (IOException e) {
+                    logger.error("error while fetching watcher_stats", e);
+                }
             }
-        });
+            {
+                Request request = new Request("GET", "/_cluster/state");
+                request.addParameter("pretty", "true");
+                try {
+                    Response response = client().performRequest(request);
+                    logger.info("cluster_state: {}", EntityUtils.toString(response.getEntity()));
+                } catch (IOException e) {
+                    logger.error("error while fetching cluster_state", e);
+                }
+            }
+            {
+                Request request = new Request("GET", "/.watcher-history-*/_search");
+                request.addParameter("size", "100");
+                request.addParameter("sort", "trigger_event.triggered_time:desc");
+                request.addParameter("pretty", "true");
+                try {
+                    Response response = client().performRequest(request);
+                    logger.info("watcher_history_snippets: {}", EntityUtils.toString(response.getEntity()));
+                } catch (IOException e) {
+                    logger.error("error while fetching watcher_history_snippets", e);
+                }
+            }
+            throw ae;
+        }
         return objectPathReference.get();
     }
 }