[Transform] Do not log error on node restart when the transform is already failed. (#106171)

docs/changelog/106171.yaml

@@ -0,0 +1,6 @@
+pr: 106171
+summary: Do not log error on node restart when the transform is already failed
+area: Transform
+type: enhancement
+issues:
+ - 106168

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java

@@ -103,15 +103,15 @@ public abstract class AbstractAuditor<T extends AbstractAuditMessage> {
     }
 
     public void info(String resourceId, String message) {
-        indexDoc(messageFactory.newMessage(resourceId, message, Level.INFO, new Date(), nodeName));
+        audit(Level.INFO, resourceId, message);
     }
 
     public void warning(String resourceId, String message) {
-        indexDoc(messageFactory.newMessage(resourceId, message, Level.WARNING, new Date(), nodeName));
+        audit(Level.WARNING, resourceId, message);
     }
 
     public void error(String resourceId, String message) {
-        indexDoc(messageFactory.newMessage(resourceId, message, Level.ERROR, new Date(), nodeName));
+        audit(Level.ERROR, resourceId, message);
     }
 
     private static void onIndexResponse(DocWriteResponse response) {

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/CannotStartFailedTransformException.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.transforms;
+
+import org.elasticsearch.ElasticsearchException;
+
+class CannotStartFailedTransformException extends ElasticsearchException {
+    CannotStartFailedTransformException(String msg) {
+        super(msg);
+    }
+}

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.transform.transforms;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
@@ -62,6 +63,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.common.notifications.Level.ERROR;
+import static org.elasticsearch.xpack.core.common.notifications.Level.INFO;
 import static org.elasticsearch.xpack.transform.transforms.TransformNodes.nodeCanRunThisTransform;
 
 public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<TransformTaskParams> {
@@ -203,11 +206,17 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
         ActionListener<StartTransformAction.Response> startTaskListener = ActionListener.wrap(
             response -> logger.info("[{}] successfully completed and scheduled task in node operation", transformId),
             failure -> {
-                auditor.error(
+                // If the transform is failed then there is no need to log an error on every node restart as the error had already been
+                // logged when the transform first failed.
+                boolean logErrorAsInfo = failure instanceof CannotStartFailedTransformException;
+                auditor.audit(
+                    logErrorAsInfo ? INFO : ERROR,
                     transformId,
-                    "Failed to start transform. " + "Please stop and attempt to start again. Failure: " + failure.getMessage()
+                    "Failed to start transform. Please stop and attempt to start again. Failure: " + failure.getMessage()
                 );
-                logger.error("Failed to start task [" + transformId + "] in node operation", failure);
+                logger.atLevel(logErrorAsInfo ? Level.INFO : Level.ERROR)
+                    .withThrowable(failure)
+                    .log("[{}] Failed to start task in node operation", transformId);
             }
         );
 

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java

@@ -240,9 +240,8 @@ public class TransformTask extends AllocatedPersistentTask implements TransformS
         logger.debug("[{}] start called with state [{}].", getTransformId(), getState());
         if (context.getTaskState() == TransformTaskState.FAILED) {
             listener.onFailure(
-                new ElasticsearchStatusException(
-                    TransformMessages.getMessage(CANNOT_START_FAILED_TRANSFORM, getTransformId(), context.getStateReason()),
-                    RestStatus.CONFLICT
+                new CannotStartFailedTransformException(
+                    TransformMessages.getMessage(CANNOT_START_FAILED_TRANSFORM, getTransformId(), context.getStateReason())
                 )
             );
             return;