Deprecate `_type` in simulate pipeline requests (#37949)

As mapping types are being removed throughout Elasticsearch, the use of
`_type` in pipeline simulation requests is deprecated. Additionally, the
default `_type` used if one is not supplied has been changed to `_doc` for
consistency with the rest of Elasticsearch.

client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java

@@ -130,7 +130,6 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
             {
                 builder.startObject()
                     .field("_index", "index")
-                    .field("_type", "doc")
                     .field("_id", "doc_" + 1)
                     .startObject("_source").field("foo", "rab_" + 1).field("rank", rankValue).endObject()
                     .endObject();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java

@@ -296,8 +296,8 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
                         "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" +
                     "}," +
                     "\"docs\":[" +
-                        "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," +
-                        "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" +
+                        "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," +
+                        "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" +
                     "]" +
                 "}";
             SimulatePipelineRequest request = new SimulatePipelineRequest(
@@ -353,8 +353,8 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
                     "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" +
                     "}," +
                     "\"docs\":[" +
-                    "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," +
-                    "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" +
+                    "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," +
+                    "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" +
                     "]" +
                     "}";
             SimulatePipelineRequest request = new SimulatePipelineRequest(

docs/reference/ingest/apis/simulate-pipeline.asciidoc

@@ -65,7 +65,6 @@ POST _ingest/pipeline/_simulate
   "docs": [
     {
       "_index": "index",
-      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "bar"
@@ -73,7 +72,6 @@ POST _ingest/pipeline/_simulate
     },
     {
       "_index": "index",
-      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "rab"
@@ -158,7 +156,6 @@ POST _ingest/pipeline/_simulate?verbose
   "docs": [
     {
       "_index": "index",
-      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "bar"
@@ -166,7 +163,6 @@ POST _ingest/pipeline/_simulate?verbose
     },
     {
       "_index": "index",
-      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "rab"

docs/reference/ingest/processors/date-index-name.asciidoc

@@ -112,7 +112,7 @@ and the result:
       "doc" : {
         "_id" : "_id",
         "_index" : "<myindex-{2016-04-25||/M{yyyy-MM-dd|UTC}}>",
-        "_type" : "_type",
+        "_type" : "_doc",
         "_source" : {
           "date1" : "2016-04-25T12:02:01.789Z"
         },

docs/reference/ingest/processors/grok.asciidoc

@@ -193,7 +193,7 @@ response:
   "docs": [
     {
       "doc": {
-        "_type": "_type",
+        "_type": "_doc",
         "_index": "_index",
         "_id": "_id",
         "_source": {
@@ -254,7 +254,7 @@ POST _ingest/pipeline/_simulate
   "docs": [
     {
       "doc": {
-        "_type": "_type",
+        "_type": "_doc",
         "_index": "_index",
         "_id": "_id",
         "_source": {

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

@@ -19,11 +19,14 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -43,6 +46,9 @@ import java.util.Objects;
 
 public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject {
 
+    private static final Logger logger = LogManager.getLogger(SimulatePipelineRequest.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+
     private String id;
     private boolean verbose;
     private BytesReference source;
@@ -178,8 +184,12 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent
                 dataMap, Fields.SOURCE);
             String index = ConfigurationUtils.readStringOrIntProperty(null, null,
                 dataMap, MetaData.INDEX.getFieldName(), "_index");
+            if (dataMap.containsKey(MetaData.TYPE.getFieldName())) {
+                deprecationLogger.deprecatedAndMaybeLog("simulate_pipeline_with_types",
+                    "[types removal] specifying _type in pipeline simulation requests is deprecated");
+            }
             String type = ConfigurationUtils.readStringOrIntProperty(null, null,
-                dataMap, MetaData.TYPE.getFieldName(), "_type");
+                dataMap, MetaData.TYPE.getFieldName(), "_doc");
             String id = ConfigurationUtils.readStringOrIntProperty(null, null,
                 dataMap, MetaData.ID.getFieldName(), "_id");
             String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null,

server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

@@ -19,15 +19,6 @@
 
 package org.elasticsearch.action.ingest;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.ingest.CompoundProcessor;
 import org.elasticsearch.ingest.IngestDocument;
@@ -38,6 +29,15 @@ import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields;
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.ID;
@@ -67,7 +67,15 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         when(ingestService.getProcessorFactories()).thenReturn(registry);
     }
 
-    public void testParseUsingPipelineStore() throws Exception {
+    public void testParseUsingPipelineStoreNoType() throws Exception {
+        innerTestParseUsingPipelineStore(false);
+    }
+
+    public void testParseUsingPipelineStoreWithType() throws Exception {
+        innerTestParseUsingPipelineStore(true);
+    }
+
+    private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Exception {
         int numDocs = randomIntBetween(1, 10);
 
         Map<String, Object> requestContent = new HashMap<>();
@@ -80,7 +88,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             String type = randomAlphaOfLengthBetween(1, 10);
             String id = randomAlphaOfLengthBetween(1, 10);
             doc.put(INDEX.getFieldName(), index);
-            doc.put(TYPE.getFieldName(), type);
+            if (useExplicitType) {
+                doc.put(TYPE.getFieldName(), type);
+            }
             doc.put(ID.getFieldName(), id);
             String fieldName = randomAlphaOfLengthBetween(1, 10);
             String fieldValue = randomAlphaOfLengthBetween(1, 10);
@@ -88,7 +98,11 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             docs.add(doc);
             Map<String, Object> expectedDoc = new HashMap<>();
             expectedDoc.put(INDEX.getFieldName(), index);
-            expectedDoc.put(TYPE.getFieldName(), type);
+            if (useExplicitType) {
+                expectedDoc.put(TYPE.getFieldName(), type);
+            } else {
+                expectedDoc.put(TYPE.getFieldName(), "_doc");
+            }
             expectedDoc.put(ID.getFieldName(), id);
             expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
             expectedDocs.add(expectedDoc);
@@ -111,9 +125,20 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
         assertThat(actualRequest.getPipeline().getDescription(), nullValue());
         assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1));
+        if (useExplicitType) {
+            assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated");
+        }
+    }
+
+    public void testParseWithProvidedPipelineNoType() throws Exception {
+        innerTestParseWithProvidedPipeline(false);
     }
 
-    public void testParseWithProvidedPipeline() throws Exception {
+    public void testParseWithProvidedPipelineWithType() throws Exception {
+        innerTestParseWithProvidedPipeline(true);
+    }
+
+    private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws Exception {
         int numDocs = randomIntBetween(1, 10);
 
         Map<String, Object> requestContent = new HashMap<>();
@@ -135,6 +160,14 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
                     );
                     doc.put(field.getFieldName(), value);
                     expectedDoc.put(field.getFieldName(), value);
+                } else if (field == TYPE) {
+                    if (useExplicitType) {
+                        String value = randomAlphaOfLengthBetween(1, 10);
+                        doc.put(field.getFieldName(), value);
+                        expectedDoc.put(field.getFieldName(), value);
+                    } else {
+                        expectedDoc.put(field.getFieldName(), "_doc");
+                    }
                 } else {
                     if (randomBoolean()) {
                         String value = randomAlphaOfLengthBetween(1, 10);
@@ -191,7 +224,6 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             Map<String, Object> expectedDocument = expectedDocsIterator.next();
             Map<IngestDocument.MetaData, Object> metadataMap = ingestDocument.extractMetadata();
             assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName())));
-            assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName())));
             assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName())));
             assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName())));
             assertThat(metadataMap.get(VERSION), equalTo(expectedDocument.get(VERSION.getFieldName())));
@@ -202,6 +234,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
         assertThat(actualRequest.getPipeline().getDescription(), nullValue());
         assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors));
+        if (useExplicitType) {
+            assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated");
+        }
     }
 
     public void testNullPipelineId() {