Delay warning about missing x-pack (#54265)

Currently, when monitoring is enabled in a freshly-installed cluster,
the non-master nodes log a warning message indicating that master may
not have x-pack installed. The message is often printed even when the
master does have x-pack installed but takes some time to setup the local
exporter for monitoring. This commit adds the local exporter setting
`wait_master.timeout` which defaults to 30 seconds. The setting
configures the time that the non-master nodes should wait for master to
setup monitoring. After the time elapses, they log a message to the user
about possible missing x-pack installation on master.

The logging of this warning was moved from `resolveBulk()` to
`openBulk()` since `resolveBulk()` is called only on cluster updates and
the message might not be logged until a new cluster update occurs.

Closes #40898

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/reference/settings/monitoring-settings.asciidoc

@@ -153,6 +153,11 @@ Whether to create cluster alerts for this cluster. The default value is `true`.
 To use this feature, {watcher} must be enabled.  If you have a basic license,
 cluster alerts are not displayed.
 
+`wait_master.timeout`::
+
+(<<time-units,time value>>) Time to wait for the master node to setup `local` exporter for monitoring.
+After that, the non-master nodes will warn the user for possible missing X-Pack configuration. Defaults to `30s`.
+
 [float]
 [[http-exporter-settings]]
 ==== HTTP Exporter Settings

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java

@@ -254,6 +254,7 @@ public class Exporters extends AbstractLifecycleComponent {
         List<Setting.AffixSetting<?>> settings = new ArrayList<>();
         settings.addAll(Exporter.getSettings());
         settings.addAll(HttpExporter.getSettings());
+        settings.addAll(LocalExporter.getSettings());
         return settings;
     }
 

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -28,10 +28,13 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
@@ -86,6 +89,16 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     public static final String TYPE = "local";
 
+    /**
+     * Time to wait for the master node to setup local exporter for monitoring.
+     * After that, the non-master nodes will warn the user for possible missing configuration.
+     */
+    public static final Setting.AffixSetting<TimeValue> WAIT_MASTER_TIMEOUT_SETTING = Setting.affixKeySetting(
+        "xpack.monitoring.exporters.",
+        "wait_master.timeout",
+        (key) -> Setting.timeSetting(key, TimeValue.timeValueSeconds(30), Property.Dynamic, Property.NodeScope)
+    );
+
     private final Client client;
     private final ClusterService clusterService;
     private final XPackLicenseState licenseState;
@@ -96,8 +109,10 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final AtomicBoolean installingSomething = new AtomicBoolean(false);
-    private final AtomicBoolean waitedForSetup = new AtomicBoolean(false);
     private final AtomicBoolean watcherSetup = new AtomicBoolean(false);
+    private final AtomicBoolean stateInitialized = new AtomicBoolean(false);
+
+    private long stateInitializedTime;
 
     public LocalExporter(Exporter.Config config, Client client, CleanerService cleanerService) {
         super(config);
@@ -116,6 +131,13 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
+        // Save the time right after the cluster state is initialized/recovered
+        // to use it later for LocalExporter#WAIT_MASTER_TIMEOUT_SETTING
+        if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false) {
+            if (stateInitialized.getAndSet(true) == false) {
+                stateInitializedTime = client.threadPool().relativeTimeInMillis();
+            }
+        }
         if (state.get() == State.INITIALIZED) {
             resolveBulk(event.state(), true);
         }
@@ -144,6 +166,17 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
     @Override
     public void openBulk(final ActionListener<ExportBulk> listener) {
         if (state.get() != State.RUNNING) {
+            // wait for some time before informing the user for possible missing x-pack configuration on master
+            final TimeValue masterTimeout = WAIT_MASTER_TIMEOUT_SETTING.getConcreteSettingForNamespace(config.name())
+                .get(config.settings());
+            TimeValue timeElapsed = TimeValue.timeValueMillis(client.threadPool().relativeTimeInMillis() - stateInitializedTime);
+            if (timeElapsed.compareTo(masterTimeout) > 0) {
+                logger.info(
+                    "waiting for elected master node [{}] to setup local exporter [{}] (does it have x-pack installed?)",
+                    clusterService.state().nodes().getMasterNode(),
+                    config.name()
+                );
+            }
             listener.onResponse(null);
         } else {
             try {
@@ -179,14 +212,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
         // elected master node needs to setup templates; non-master nodes need to wait for it to be setup
         if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
             setup = setupIfElectedMaster(clusterState, templates, clusterStateChange);
-        } else if (setupIfNotElectedMaster(clusterState, templates.keySet()) == false) {
-            // the first pass will be false so that we don't bother users if the master took one-go to setup
-            if (waitedForSetup.getAndSet(true)) {
-                logger.info("waiting for elected master node [{}] to setup local exporter [{}] (does it have x-pack installed?)",
-                            clusterService.state().nodes().getMasterNode(), config.name());
-            }
-
-            setup = false;
+        } else {
+            setup = setupIfNotElectedMaster(clusterState, templates.keySet());
         }
 
         // any failure/delay to setup the local exporter stops it until the next pass (10s by default)
@@ -651,4 +678,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     }
 
+    public static List<Setting.AffixSetting<?>> getSettings() {
+        return List.of(WAIT_MASTER_TIMEOUT_SETTING);
+    }
+
 }