[Inference API] Check pipelines on delete inference endpoint (#109123)

* renaming

* check for pipeline references

* add force and dry run options to request

* add force delete and dry run options to Actions

* Revert dependency on ml module

* duplicate InferenceProcessorInfoExtractor to inference module

* Ignore missing IngestMetadata during attempted endpoint delete

* Update docs/changelog/109123.yaml

* rename RestDeleteInference Action

* fix dry run

* Add integration test and misc improvements

* Update 109123.yaml

* merge fixes

* Improvements from PR review

* Moving shared pieces to core (#109183)

* replace `model` in exception message with `inference endpoint`

* test fix

---------

Co-authored-by: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>

docs/changelog/109123.yaml

@@ -0,0 +1,5 @@
+pr: 109123
+summary: "[Inference API] Check for related pipelines on delete inference endpoint"
+area: Machine Learning
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -180,6 +180,7 @@ public class TransportVersions {
     public static final TransportVersion ADD_METADATA_FLATTENED_TO_ROLES = def(8_671_00_0);
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
     public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
+    public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+public class DeleteInferenceEndpointAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteInferenceEndpointAction INSTANCE = new DeleteInferenceEndpointAction();
+    public static final String NAME = "cluster:admin/xpack/inference/delete";
+
+    public DeleteInferenceEndpointAction() {
+        super(NAME);
+    }
+
+    public static class Request extends AcknowledgedRequest<DeleteInferenceEndpointAction.Request> {
+
+        private final String inferenceEndpointId;
+        private final TaskType taskType;
+        private final boolean forceDelete;
+        private final boolean dryRun;
+
+        public Request(String inferenceEndpointId, TaskType taskType, boolean forceDelete, boolean dryRun) {
+            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+            this.inferenceEndpointId = inferenceEndpointId;
+            this.taskType = taskType;
+            this.forceDelete = forceDelete;
+            this.dryRun = dryRun;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.inferenceEndpointId = in.readString();
+            this.taskType = TaskType.fromStream(in);
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+                this.forceDelete = Boolean.TRUE.equals(in.readOptionalBoolean());
+                this.dryRun = Boolean.TRUE.equals(in.readOptionalBoolean());
+            } else {
+                this.forceDelete = false;
+                this.dryRun = false;
+            }
+        }
+
+        public String getInferenceEndpointId() {
+            return inferenceEndpointId;
+        }
+
+        public TaskType getTaskType() {
+            return taskType;
+        }
+
+        public boolean isForceDelete() {
+            return forceDelete;
+        }
+
+        public boolean isDryRun() {
+            return dryRun;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(inferenceEndpointId);
+            taskType.writeTo(out);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+                out.writeOptionalBoolean(forceDelete);
+                out.writeOptionalBoolean(dryRun);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DeleteInferenceEndpointAction.Request request = (DeleteInferenceEndpointAction.Request) o;
+            return Objects.equals(inferenceEndpointId, request.inferenceEndpointId)
+                && taskType == request.taskType
+                && forceDelete == request.forceDelete
+                && dryRun == request.dryRun;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(inferenceEndpointId, taskType, forceDelete, dryRun);
+        }
+    }
+
+    public static class Response extends AcknowledgedResponse {
+
+        private final String PIPELINE_IDS = "pipelines";
+        Set<String> pipelineIds;
+
+        public Response(boolean acknowledged, Set<String> pipelineIds) {
+            super(acknowledged);
+            this.pipelineIds = pipelineIds;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            pipelineIds = in.readCollectionAsSet(StreamInput::readString);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeCollection(pipelineIds, StreamOutput::writeString);
+        }
+
+        @Override
+        protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
+            super.addCustomFields(builder, params);
+            builder.field(PIPELINE_IDS, pipelineIds);
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder returnable = new StringBuilder();
+            returnable.append("acknowledged: ").append(this.acknowledged);
+            returnable.append(", pipelineIdsByEndpoint: ");
+            for (String entry : pipelineIds) {
+                returnable.append(entry).append(", ");
+            }
+            return returnable.toString();
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceModelAction.java

@@ -1,74 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.action;
-
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.inference.TaskType;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class DeleteInferenceModelAction extends ActionType<AcknowledgedResponse> {
-
-    public static final DeleteInferenceModelAction INSTANCE = new DeleteInferenceModelAction();
-    public static final String NAME = "cluster:admin/xpack/inference/delete";
-
-    public DeleteInferenceModelAction() {
-        super(NAME);
-    }
-
-    public static class Request extends AcknowledgedRequest<DeleteInferenceModelAction.Request> {
-
-        private final String inferenceEntityId;
-        private final TaskType taskType;
-
-        public Request(String inferenceEntityId, TaskType taskType) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            this.inferenceEntityId = inferenceEntityId;
-            this.taskType = taskType;
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.inferenceEntityId = in.readString();
-            this.taskType = TaskType.fromStream(in);
-        }
-
-        public String getInferenceEntityId() {
-            return inferenceEntityId;
-        }
-
-        public TaskType getTaskType() {
-            return taskType;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeString(inferenceEntityId);
-            taskType.writeTo(out);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            DeleteInferenceModelAction.Request request = (DeleteInferenceModelAction.Request) o;
-            return Objects.equals(inferenceEntityId, request.inferenceEntityId) && taskType == request.taskType;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(inferenceEntityId, taskType);
-        }
-    }
-}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorConstants.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.utils;
+
+/**
+ * Provides access to constants for core, ml, and inference plugins
+ */
+public class InferenceProcessorConstants {
+    public static final String TYPE = "inference";
+    public static final String TARGET_FIELD = "target_field";
+    public static final String FIELD_MAP = "field_map";
+    public static final String INFERENCE_CONFIG = "inference_config";
+
+    private InferenceProcessorConstants() {}
+}

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractor.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.ml.utils;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.cluster.ClusterState;
@@ -16,6 +16,7 @@ import org.elasticsearch.ingest.Pipeline;
 import org.elasticsearch.transport.Transports;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -24,13 +25,11 @@ import java.util.function.Consumer;
 
 import static org.elasticsearch.inference.InferenceResults.MODEL_ID_RESULTS_FIELD;
 import static org.elasticsearch.ingest.Pipeline.PROCESSORS_KEY;
-import static org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor.TYPE;
 
 /**
  * Utilities for extracting information around inference processors from IngestMetadata
  */
 public final class InferenceProcessorInfoExtractor {
-
     private static final String FOREACH_PROCESSOR_NAME = "foreach";
     // Any more than 10 nestings of processors, we stop searching for inference processor definitions
     private static final int MAX_INFERENCE_PROCESSOR_SEARCH_RECURSIONS = 10;
@@ -131,6 +130,38 @@ public final class InferenceProcessorInfoExtractor {
         return pipelineIdsByModelIds;
     }
 
+    /**
+     * @param state Current {@link ClusterState}
+     * @return a map from Model or Deployment IDs or Aliases to each pipeline referencing them.
+     */
+    @SuppressWarnings("unchecked")
+    public static Set<String> pipelineIdsForResource(ClusterState state, Set<String> ids) {
+        assert Transports.assertNotTransportThread("non-trivial nested loops over cluster state structures");
+        Set<String> pipelineIds = new HashSet<>();
+        Metadata metadata = state.metadata();
+        if (metadata == null) {
+            return pipelineIds;
+        }
+        IngestMetadata ingestMetadata = metadata.custom(IngestMetadata.TYPE);
+        if (ingestMetadata == null) {
+            return pipelineIds;
+        }
+        ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
+            Map<String, Object> configMap = configuration.getConfigAsMap();
+            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            for (Map<String, Object> processorConfigWithKey : processorConfigs) {
+                for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
+                    addModelsAndPipelines(entry.getKey(), pipelineId, (Map<String, Object>) entry.getValue(), pam -> {
+                        if (ids.contains(pam.modelIdOrAlias)) {
+                            pipelineIds.add(pipelineId);
+                        }
+                    }, 0);
+                }
+            }
+        });
+        return pipelineIds;
+    }
+
     @SuppressWarnings("unchecked")
     private static void addModelsAndPipelines(
         String processorType,
@@ -146,7 +177,7 @@ public final class InferenceProcessorInfoExtractor {
         if (processorType == null || processorDefinition == null) {
             return;
         }
-        if (TYPE.equals(processorType)) {
+        if (InferenceProcessorConstants.TYPE.equals(processorType)) {
             String modelId = (String) processorDefinition.get(MODEL_ID_RESULTS_FIELD);
             if (modelId != null) {
                 handler.accept(new PipelineAndModel(pipelineId, modelId));

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractorTests.java → x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractorTests.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.ml.utils;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -23,7 +23,6 @@ import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
-import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -217,15 +216,15 @@ public class InferenceProcessorInfoExtractorTests extends ESTestCase {
     }
 
     private static Map<String, Object> inferenceProcessorForModel(String modelId) {
-        return Collections.singletonMap(InferenceProcessor.TYPE, new HashMap<>() {
+        return Collections.singletonMap(InferenceProcessorConstants.TYPE, new HashMap<>() {
             {
                 put(InferenceResults.MODEL_ID_RESULTS_FIELD, modelId);
                 put(
-                    InferenceProcessor.INFERENCE_CONFIG,
+                    InferenceProcessorConstants.INFERENCE_CONFIG,
                     Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap())
                 );
-                put(InferenceProcessor.TARGET_FIELD, "new_field");
-                put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
+                put(InferenceProcessorConstants.TARGET_FIELD, "new_field");
+                put(InferenceProcessorConstants.FIELD_MAP, Collections.singletonMap("source", "dest"));
             }
         });
     }

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java

@@ -113,6 +113,13 @@ public class InferenceBaseRestTest extends ESRestTestCase {
         assertOkOrCreated(response);
     }
 
+    protected Response deleteModel(String modelId, String queryParams) throws IOException {
+        var request = new Request("DELETE", "_inference/" + modelId + "?" + queryParams);
+        var response = client().performRequest(request);
+        assertOkOrCreated(response);
+        return response;
+    }
+
     protected void deleteModel(String modelId, TaskType taskType) throws IOException {
         var request = new Request("DELETE", Strings.format("_inference/%s/%s", taskType, modelId));
         var response = client().performRequest(request);
@@ -124,6 +131,29 @@ public class InferenceBaseRestTest extends ESRestTestCase {
         return putRequest(endpoint, modelConfig);
     }
 
+    protected Map<String, Object> putPipeline(String pipelineId, String modelId) throws IOException {
+        String endpoint = Strings.format("_ingest/pipeline/%s", pipelineId);
+        String body = """
+            {
+              "description": "Test pipeline",
+              "processors": [
+                {
+                  "inference": {
+                    "model_id": "%s"
+                  }
+                }
+              ]
+            }
+            """.formatted(modelId);
+        return putRequest(endpoint, body);
+    }
+
+    protected void deletePipeline(String pipelineId) throws IOException {
+        var request = new Request("DELETE", Strings.format("_ingest/pipeline/%s", pipelineId));
+        var response = client().performRequest(request);
+        assertOkOrCreated(response);
+    }
+
     /**
      * Task type should be in modelConfig
      */

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.xpack.inference;
 
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.inference.TaskType;
@@ -115,4 +116,34 @@ public class InferenceCrudIT extends InferenceBaseRestTest {
         // We would expect an error about the invalid API key if the validation occurred
         putModel("unvalidated", openAiConfigWithBadApiKey, TaskType.TEXT_EMBEDDING);
     }
+
+    public void testDeleteEndpointWhileReferencedByPipeline() throws IOException {
+        String endpointId = "endpoint_referenced_by_pipeline";
+        putModel(endpointId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING);
+        var pipelineId = "pipeline_referencing_model";
+        putPipeline(pipelineId, endpointId);
+
+        {
+            var e = expectThrows(ResponseException.class, () -> deleteModel(endpointId));
+            assertThat(
+                e.getMessage(),
+                containsString(
+                    "Inference endpoint endpoint_referenced_by_pipeline is referenced by pipelines and cannot be deleted. "
+                        + "Use `force` to delete it anyway, or use `dry_run` to list the pipelines that reference it."
+                )
+            );
+        }
+        {
+            var response = deleteModel(endpointId, "dry_run=true");
+            var entityString = EntityUtils.toString(response.getEntity());
+            assertThat(entityString, containsString(pipelineId));
+            assertThat(entityString, containsString("\"acknowledged\":false"));
+        }
+        {
+            var response = deleteModel(endpointId, "force=true");
+            var entityString = EntityUtils.toString(response.getEntity());
+            assertThat(entityString, containsString("\"acknowledged\":true"));
+        }
+        deletePipeline(pipelineId);
+    }
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java

@@ -38,12 +38,12 @@ import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction;
-import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceAction;
@@ -60,7 +60,7 @@ import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
 import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
-import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.inference.rest.RestGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.rest.RestGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.rest.RestInferenceAction;
@@ -125,7 +125,7 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP
             new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class),
             new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class),
             new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class),
-            new ActionHandler<>(DeleteInferenceModelAction.INSTANCE, TransportDeleteInferenceModelAction.class),
+            new ActionHandler<>(DeleteInferenceEndpointAction.INSTANCE, TransportDeleteInferenceEndpointAction.class),
             new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class),
             new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class)
         );
@@ -147,7 +147,7 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP
             new RestInferenceAction(),
             new RestGetInferenceModelAction(),
             new RestPutInferenceModelAction(),
-            new RestDeleteInferenceModelAction(),
+            new RestDeleteInferenceEndpointAction(),
             new RestGetInferenceDiagnosticsAction()
         );
     }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.ingest.IngestMetadata;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
+import org.elasticsearch.xpack.inference.common.InferenceExceptions;
+import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+
+import java.util.Set;
+
+public class TransportDeleteInferenceEndpointAction extends AcknowledgedTransportMasterNodeAction<DeleteInferenceEndpointAction.Request> {
+
+    private final ModelRegistry modelRegistry;
+    private final InferenceServiceRegistry serviceRegistry;
+    private static final Logger logger = LogManager.getLogger(TransportDeleteInferenceEndpointAction.class);
+
+    @Inject
+    public TransportDeleteInferenceEndpointAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ModelRegistry modelRegistry,
+        InferenceServiceRegistry serviceRegistry
+    ) {
+        super(
+            DeleteInferenceEndpointAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            DeleteInferenceEndpointAction.Request::new,
+            indexNameExpressionResolver,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.modelRegistry = modelRegistry;
+        this.serviceRegistry = serviceRegistry;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        DeleteInferenceEndpointAction.Request request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> masterListener
+    ) {
+        SubscribableListener.<ModelRegistry.UnparsedModel>newForked(modelConfigListener -> {
+            // Get the model from the registry
+
+            modelRegistry.getModel(request.getInferenceEndpointId(), modelConfigListener);
+        }).<Boolean>andThen((listener, unparsedModel) -> {
+            // Validate the request & issue the stop request to the service
+
+            if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) {
+                // specific task type in request does not match the models
+                listener.onFailure(InferenceExceptions.mismatchedTaskTypeException(request.getTaskType(), unparsedModel.taskType()));
+                return;
+            }
+
+            if (request.isDryRun()) {
+                masterListener.onResponse(
+                    new DeleteInferenceEndpointAction.Response(
+                        false,
+                        InferenceProcessorInfoExtractor.pipelineIdsForResource(state, Set.of(request.getInferenceEndpointId()))
+                    )
+                );
+                return;
+            } else if (request.isForceDelete() == false
+                && endpointIsReferencedInPipelines(state, request.getInferenceEndpointId(), listener)) {
+                    return;
+                }
+
+            var service = serviceRegistry.getService(unparsedModel.service());
+            if (service.isPresent()) {
+                service.get().stop(request.getInferenceEndpointId(), listener);
+            } else {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "No service found for this inference endpoint " + request.getInferenceEndpointId(),
+                        RestStatus.NOT_FOUND
+                    )
+                );
+            }
+        }).<Boolean>andThen((listener, didStop) -> {
+            if (didStop) {
+                modelRegistry.deleteModel(request.getInferenceEndpointId(), listener);
+            } else {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "Failed to stop inference endpoint " + request.getInferenceEndpointId(),
+                        RestStatus.INTERNAL_SERVER_ERROR
+                    )
+                );
+            }
+        })
+            .addListener(
+                masterListener.delegateFailure((l3, didDeleteModel) -> masterListener.onResponse(AcknowledgedResponse.of(didDeleteModel)))
+            );
+    }
+
+    private static boolean endpointIsReferencedInPipelines(
+        final ClusterState state,
+        final String inferenceEndpointId,
+        ActionListener<Boolean> listener
+    ) {
+        Metadata metadata = state.getMetadata();
+        if (metadata == null) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    " Could not determine if the endpoint is referenced in a pipeline as cluster state metadata was unexpectedly null. "
+                        + "Use `force` to delete it anyway",
+                    RestStatus.INTERNAL_SERVER_ERROR
+                )
+            );
+            // Unsure why the ClusterState metadata would ever be null, but in this case it seems safer to assume the endpoint is referenced
+            return true;
+        }
+        IngestMetadata ingestMetadata = metadata.custom(IngestMetadata.TYPE);
+        if (ingestMetadata == null) {
+            logger.debug("No ingest metadata found in cluster state while attempting to delete inference endpoint");
+        } else {
+            Set<String> modelIdsReferencedByPipelines = InferenceProcessorInfoExtractor.getModelIdsFromInferenceProcessors(ingestMetadata);
+            if (modelIdsReferencedByPipelines.contains(inferenceEndpointId)) {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "Inference endpoint "
+                            + inferenceEndpointId
+                            + " is referenced by pipelines and cannot be deleted. "
+                            + "Use `force` to delete it anyway, or use `dry_run` to list the pipelines that reference it.",
+                        RestStatus.CONFLICT
+                    )
+                );
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(DeleteInferenceEndpointAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
+    }
+
+}

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java

@@ -1,104 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.action;
-
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.SubscribableListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.inference.InferenceServiceRegistry;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
-import org.elasticsearch.xpack.inference.common.InferenceExceptions;
-import org.elasticsearch.xpack.inference.registry.ModelRegistry;
-
-public class TransportDeleteInferenceModelAction extends AcknowledgedTransportMasterNodeAction<DeleteInferenceModelAction.Request> {
-
-    private final ModelRegistry modelRegistry;
-    private final InferenceServiceRegistry serviceRegistry;
-
-    @Inject
-    public TransportDeleteInferenceModelAction(
-        TransportService transportService,
-        ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        ModelRegistry modelRegistry,
-        InferenceServiceRegistry serviceRegistry
-    ) {
-        super(
-            DeleteInferenceModelAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
-            actionFilters,
-            DeleteInferenceModelAction.Request::new,
-            indexNameExpressionResolver,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
-        );
-        this.modelRegistry = modelRegistry;
-        this.serviceRegistry = serviceRegistry;
-    }
-
-    @Override
-    protected void masterOperation(
-        Task task,
-        DeleteInferenceModelAction.Request request,
-        ClusterState state,
-        ActionListener<AcknowledgedResponse> listener
-    ) {
-        SubscribableListener.<ModelRegistry.UnparsedModel>newForked(modelConfigListener -> {
-            modelRegistry.getModel(request.getInferenceEntityId(), modelConfigListener);
-        }).<Boolean>andThen((l1, unparsedModel) -> {
-
-            if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) {
-                // specific task type in request does not match the models
-                l1.onFailure(InferenceExceptions.mismatchedTaskTypeException(request.getTaskType(), unparsedModel.taskType()));
-                return;
-            }
-            var service = serviceRegistry.getService(unparsedModel.service());
-            if (service.isPresent()) {
-                service.get().stop(request.getInferenceEntityId(), l1);
-            } else {
-                l1.onFailure(
-                    new ElasticsearchStatusException("No service found for model " + request.getInferenceEntityId(), RestStatus.NOT_FOUND)
-                );
-            }
-        }).<Boolean>andThen((l2, didStop) -> {
-            if (didStop) {
-                modelRegistry.deleteModel(request.getInferenceEntityId(), l2);
-            } else {
-                l2.onFailure(
-                    new ElasticsearchStatusException(
-                        "Failed to stop model " + request.getInferenceEntityId(),
-                        RestStatus.INTERNAL_SERVER_ERROR
-                    )
-                );
-            }
-        }).addListener(listener.delegateFailure((l3, didDeleteModel) -> listener.onResponse(AcknowledgedResponse.of(didDeleteModel))));
-    }
-
-    @Override
-    protected ClusterBlockException checkBlock(DeleteInferenceModelAction.Request request, ClusterState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
-    }
-
-}

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceModelAction.java → x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java

@@ -14,7 +14,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
 
 import java.util.List;
 
@@ -25,7 +25,10 @@ import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_INFERENCE_I
 import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID;
 
 @ServerlessScope(Scope.PUBLIC)
-public class RestDeleteInferenceModelAction extends BaseRestHandler {
+public class RestDeleteInferenceEndpointAction extends BaseRestHandler {
+
+    private String FORCE_DELETE_NAME = "force";
+    private String DRY_RUN_NAME = "dry_run";
 
     @Override
     public String getName() {
@@ -41,6 +44,9 @@ public class RestDeleteInferenceModelAction extends BaseRestHandler {
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String inferenceEntityId;
         TaskType taskType;
+        boolean forceDelete = false;
+        boolean dryRun = false;
+
         if (restRequest.hasParam(INFERENCE_ID)) {
             inferenceEntityId = restRequest.param(INFERENCE_ID);
             taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID));
@@ -49,7 +55,11 @@ public class RestDeleteInferenceModelAction extends BaseRestHandler {
             taskType = TaskType.ANY;
         }
 
-        var request = new DeleteInferenceModelAction.Request(inferenceEntityId, taskType);
-        return channel -> client.execute(DeleteInferenceModelAction.INSTANCE, request, new RestToXContentListener<>(channel));
+        forceDelete = restRequest.paramAsBoolean(FORCE_DELETE_NAME, false);
+
+        dryRun = restRequest.paramAsBoolean(DRY_RUN_NAME, false);
+
+        var request = new DeleteInferenceEndpointAction.Request(inferenceEntityId, taskType, forceDelete, dryRun);
+        return channel -> client.execute(DeleteInferenceEndpointAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java

@@ -47,6 +47,7 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 import static org.elasticsearch.xpack.ml.integration.ClassificationIT.KEYWORD_FIELD;
 import static org.elasticsearch.xpack.ml.integration.MlNativeDataFrameAnalyticsIntegTestCase.buildAnalytics;
 import static org.elasticsearch.xpack.ml.integration.PyTorchModelIT.BASE_64_ENCODED_MODEL;
@@ -55,7 +56,6 @@ import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDataf
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -475,7 +475,7 @@ import java.util.function.UnaryOperator;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX;
 import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 
 public class MachineLearning extends Plugin
     implements

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java

@@ -42,9 +42,9 @@ import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.ModelAliasMetadata;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.ArrayList;
 import java.util.HashMap;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java

@@ -33,8 +33,8 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAliasAction;
 import org.elasticsearch.xpack.core.ml.inference.ModelAliasMetadata;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.HashMap;
 import java.util.Locale;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java

@@ -74,7 +74,7 @@ import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.pipelineIdsByResource;
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.pipelineIdsByResource;
 
 public class TransportGetTrainedModelsStatsAction extends TransportAction<
     GetTrainedModelsStatsAction.Request,

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java

@@ -36,10 +36,10 @@ import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignme
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentClusterService;
 import org.elasticsearch.xpack.ml.inference.deployment.TrainedModelDeploymentTask;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.List;
 import java.util.Objects;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java

@@ -55,9 +55,10 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassifica
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorConstants;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -85,15 +86,15 @@ public class InferenceProcessor extends AbstractProcessor {
         Setting.Property.NodeScope
     );
 
-    public static final String TYPE = "inference";
+    public static final String TYPE = InferenceProcessorConstants.TYPE;
     public static final String MODEL_ID = "model_id";
-    public static final String INFERENCE_CONFIG = "inference_config";
+    public static final String INFERENCE_CONFIG = InferenceProcessorConstants.INFERENCE_CONFIG;
     public static final String IGNORE_MISSING = "ignore_missing";
 
     // target field style mappings
-    public static final String TARGET_FIELD = "target_field";
+    public static final String TARGET_FIELD = InferenceProcessorConstants.TARGET_FIELD;
     public static final String FIELD_MAPPINGS = "field_mappings";
-    public static final String FIELD_MAP = "field_map";
+    public static final String FIELD_MAP = InferenceProcessorConstants.FIELD_MAP;
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     // input field config