[ML] Addressing feedback for storing secrets in a separate net-new system index (#100148)

* Adding bwc change and index changes

* Fixing tests

* Trying to get the rest tests to work

* Making progress on the rest tests

* cluster starting

* failing on elser now

* Removing rest tests

x-pack/plugin/inference/build.gradle

@@ -4,10 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.OS
-
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 

x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/MockInferenceServiceIT.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.inference.integration;
 
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ModelConfigurations;
@@ -17,6 +18,8 @@ import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.inference.InferencePlugin;
@@ -26,6 +29,7 @@ import org.elasticsearch.xpack.inference.action.PutInferenceModelAction;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.List;
@@ -79,6 +83,24 @@ public class MockInferenceServiceIT extends ESIntegTestCase {
         inferOnMockService(modelId, TaskType.SPARSE_EMBEDDING, randomAlphaOfLength(10));
     }
 
+    public void testMockService_DoesNotReturnSecretsInGetResponse() throws IOException {
+        String modelId = "test-mock";
+        putMockService(modelId, TaskType.SPARSE_EMBEDDING);
+        ModelConfigurations readModel = getModel(modelId, TaskType.SPARSE_EMBEDDING);
+
+        assertThat(readModel.getServiceSettings(), instanceOf(TestInferenceServicePlugin.TestServiceSettings.class));
+
+        var serviceSettings = (TestInferenceServicePlugin.TestServiceSettings) readModel.getServiceSettings();
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
+        serviceSettings.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is("""
+            {
+              "model" : "my_model"
+            }"""));
+    }
+
     public void testGetUnparsedModelMap_ForTestServiceModel_ReturnsSecretsPopulated() {
         String modelId = "test-unparsed";
         putMockService(modelId, TaskType.SPARSE_EMBEDDING);

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceIndex.java

@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -62,7 +61,6 @@ public class InferenceIndex {
             return jsonBuilder().startObject()
                 .startObject(SINGLE_MAPPING_NAME)
                 .startObject("_meta")
-                .field("version", Version.CURRENT)
                 .field(SystemIndexDescriptor.VERSION_META_KEY, INDEX_MAPPING_VERSION)
                 .endObject()
                 .field("dynamic", "strict")

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java

@@ -17,9 +17,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.SecureSetting;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.env.Environment;
@@ -62,8 +59,6 @@ public class InferencePlugin extends Plugin implements ActionPlugin, InferenceSe
 
     public static final String NAME = "inference";
 
-    public static final Setting<SecureString> ENCRYPTION_KEY_SETTING = SecureSetting.secureString("xpack.inference.encryption_key", null);
-
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return List.of(
@@ -113,11 +108,6 @@ public class InferencePlugin extends Plugin implements ActionPlugin, InferenceSe
         return List.of(modelRegistry);
     }
 
-    @Override
-    public List<Setting<?>> getSettings() {
-        return List.of(ENCRYPTION_KEY_SETTING);
-    }
-
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         return List.of(

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceSecretsIndex.java

@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -23,7 +22,7 @@ public class InferenceSecretsIndex {
 
     private InferenceSecretsIndex() {}
 
-    public static final String INDEX_NAME = ".infer-secrets";
+    public static final String INDEX_NAME = ".secrets-inference";
     public static final String INDEX_PATTERN = INDEX_NAME + "*";
 
     // Increment this version number when the mappings change
@@ -62,7 +61,6 @@ public class InferenceSecretsIndex {
             return jsonBuilder().startObject()
                 .startObject(SINGLE_MAPPING_NAME)
                 .startObject("_meta")
-                .field("version", Version.CURRENT)
                 .field(SystemIndexDescriptor.VERSION_META_KEY, INDEX_MAPPING_VERSION)
                 .endObject()
                 .field("dynamic", "strict")

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java

@@ -89,7 +89,7 @@ public class ModelRegistryTests extends ESTestCase {
 
     public void testGetUnparsedModelMap_ThrowsIllegalStateException_WhenUnableToFindInferenceEntry() {
         var client = mockClient();
-        var inferenceSecretsHit = SearchHit.createFromMap(Map.of("_index", ".infer-secrets"));
+        var inferenceSecretsHit = SearchHit.createFromMap(Map.of("_index", ".secrets-inference"));
         mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceSecretsHit }));
 
         var registry = new ModelRegistry(client);
@@ -124,7 +124,7 @@ public class ModelRegistryTests extends ESTestCase {
     public void testGetUnparsedModelMap_ReturnsModelConfigMap_WhenBothInferenceAndSecretsHitsAreFound() {
         var client = mockClient();
         var inferenceHit = SearchHit.createFromMap(Map.of("_index", ".inference"));
-        var inferenceSecretsHit = SearchHit.createFromMap(Map.of("_index", ".infer-secrets"));
+        var inferenceSecretsHit = SearchHit.createFromMap(Map.of("_index", ".secrets-inference"));
 
         mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceHit, inferenceSecretsHit }));