[ML] Improve log message for model snapshot update (#134541) (#134905)

Since the 9.0 upgrade, we have introduced a solution by setting large .ml-anomalies to read only during the upgrade assistant. However, there is an edge case that will cause the obsolete model snapshot to fail to upgrade.

Step to reproduce:

setup a 7.17 cluster
create and start an anomaly detection job
upgrade to 8.18 or 8.19
go to upgrade assistant
change .ml-anomalies-shared to read only
then try upgrade obsolete model snapshot
Observed:

upgrade model snapshot will fail.
This PR extends the warning message explaining the steps to resolve the issue.

(cherry picked from commit c47522ca31a88a38dbb9d75cead41ae596a573ce)

# Conflicts:
#	server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
#	server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt

server/src/main/java/org/elasticsearch/common/ReferenceDocs.java

@@ -84,6 +84,7 @@ public enum ReferenceDocs {
     ALLOCATION_EXPLAIN_MAX_RETRY,
     SECURE_SETTINGS,
     CLUSTER_SHARD_LIMIT,
+    DELETE_INDEX_BLOCK,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 

server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt

@@ -46,3 +46,4 @@ JDK_LOCALE_DIFFERENCES                                          mapping-date-for
 ALLOCATION_EXPLAIN_MAX_RETRY                                    cluster-allocation-explain.html#maximum-number-of-retries-exceeded
 SECURE_SETTINGS                                                 secure-settings.html
 CLUSTER_SHARD_LIMIT                                             misc-cluster-settings.html#cluster-shard-limit
+DELETE_INDEX_BLOCK                                              api/doc/elasticsearch/operation/operation-indices-remove-block

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java

@@ -17,6 +17,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -196,15 +197,21 @@ public final class JobModelSnapshotUpgrader {
                     );
                 }
             }, e -> {
-                logger.warn(
-                    () -> format(
+                logger.warn(() -> {
+                    String baseMessage = format(
                         "[%s] [%s] failed to delete old snapshot [%s] result document",
                         jobId,
                         snapshotId,
                         ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()
-                    ),
-                    e
-                );
+                    );
+                    if (e instanceof org.elasticsearch.cluster.block.ClusterBlockException) {
+                        return baseMessage
+                            + ". Remove the write block from the results index to delete the model snapshot. See "
+                            + ReferenceDocs.DELETE_INDEX_BLOCK
+                            + " for details.";
+                    }
+                    return baseMessage;
+                }, e);
             }), () -> runAfter.accept(null)));
     }