Remove types from internal monitoring templates and bump to api 7 (#39888)

This commit removes the "doc" type from monitoring internal indexes.
The template still carries the "_doc" type since that is needed for
the internal representation.

This change impacts the following templates:
monitoring-alerts.json
monitoring-beats.json
monitoring-es.json
monitoring-kibana.json
monitoring-logstash.json

As part of the required changes, the system_api_version has been
bumped from "6" to "7" and support for version "2" has been dropped.

A new empty pipeline is now introduced for the version "7", and
the formerly empty "6" pipeline will now remove the type and re-direct
the request to the "7" index.

Additionally, to due to a difference in the internal representation
(which requires the inclusion of "_doc" type) and external representation
(which requires the exclusion of any type) a helper method is introduced
to help convert internal to external representation, and used by the
monitoring HTTP template exporter.

Relates #38637

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -345,6 +345,19 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
             builder.endObject();
         }
 
+        /**
+         * Removes the nested type in the xContent representation of {@link IndexTemplateMetaData}.
+         *
+         * This method is useful to help bridge the gap between an the internal representation which still uses (the legacy format) a
+         * nested type in the mapping, and the external representation which does not use a nested type in the mapping.
+         */
+        public static void removeType(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder) throws IOException {
+            builder.startObject();
+            toInnerXContent(indexTemplateMetaData, builder,
+                new ToXContent.MapParams(Collections.singletonMap("reduce_mappings", "true")), false);
+            builder.endObject();
+        }
+
         /**
          * Serializes the template to xContent, making sure not to nest mappings under the
          * type name.
@@ -361,6 +374,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
             builder.endObject();
         }
 
+
         static void toInnerXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
                                              XContentBuilder builder,
                                              ToXContent.Params params) throws IOException {

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java

@@ -148,7 +148,7 @@ public class AutoFollowStatsMonitoringDocTests extends BaseMonitoringDocTestCase
         Map<String, Object> template =
             XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
         Map<?, ?> autoFollowStatsMapping =
-            (Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_auto_follow_stats.properties", template);
+            (Map<?, ?>) XContentMapValues.extractValue("mappings._doc.properties.ccr_auto_follow_stats.properties", template);
 
         assertThat(serializedStatus.size(), equalTo(autoFollowStatsMapping.size()));
         for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java

@@ -237,7 +237,8 @@ public class FollowStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Fol
 
         Map<String, Object> template =
             XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
-        Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_stats.properties", template);
+        Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues
+            .extractValue("mappings._doc.properties.ccr_stats.properties", template);
         assertThat(serializedStatus.size(), equalTo(followStatsMapping.size()));
         for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {
             String fieldName = entry.getKey();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java

@@ -66,14 +66,13 @@ public class MonitoringBulkRequest extends ActionRequest {
      * Parses a monitoring bulk request and builds the list of documents to be indexed.
      */
     public MonitoringBulkRequest add(final MonitoredSystem system,
-                                     final String defaultType,
                                      final BytesReference content,
                                      final XContentType xContentType,
                                      final long timestamp,
                                      final long intervalMillis) throws IOException {
 
         // MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest
-        new BulkRequestParser(false).parse(content, null, defaultType, null, null, null, true, xContentType,
+        new BulkRequestParser(false).parse(content, null, null, null, null, true, xContentType,
                 indexRequest -> {
                     // we no longer accept non-timestamped indexes from Kibana, LS, or Beats because we do not use the data
                     // and it was duplicated anyway; by simply dropping it, we allow BWC for older clients that still send it

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequestBuilder.java

@@ -26,12 +26,11 @@ public class MonitoringBulkRequestBuilder
     }
 
     public MonitoringBulkRequestBuilder add(final MonitoredSystem system,
-                                            final String type,
                                             final BytesReference content,
                                             final XContentType xContentType,
                                             final long timestamp,
                                             final long intervalMillis) throws IOException {
-        request.add(system, type, content, xContentType, timestamp, intervalMillis);
+        request.add(system, content, xContentType, timestamp, intervalMillis);
         return this;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java

@@ -33,18 +33,19 @@ public final class MonitoringTemplateUtils {
 
     /**
      * Current version of templates used in their name to differentiate from breaking changes (separate from product version).
+     * Version 7 has the same structure as version 6, but uses the `_doc` type.
      */
-    public static final String TEMPLATE_VERSION = "6";
+    public static final String TEMPLATE_VERSION = "7";
     /**
      * The previous version of templates, which we still support via the REST /_monitoring/bulk endpoint because
      * nothing changed for those documents.
      */
-    public static final String OLD_TEMPLATE_VERSION = "2";
+    public static final String OLD_TEMPLATE_VERSION = "6";
 
     /**
      * IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate}.
      */
-    public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash", "beats" };
+    public static final String[] TEMPLATE_IDS = { "alerts-7", "es", "kibana", "logstash", "beats" };
 
     /**
      * IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a
@@ -54,7 +55,7 @@ public final class MonitoringTemplateUtils {
      * instances will attempt to create a named template based on the templates that they expect (e.g., ".monitoring-es-2") and not the
      * ones that we are creating.
      */
-    public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash", "alerts" };
+    public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash" }; //excluding alerts since 6.x watches use it
 
     /**
      * IDs of pipelines that can be used with
@@ -99,7 +100,7 @@ public final class MonitoringTemplateUtils {
      * @see #OLD_TEMPLATE_VERSION
      */
     public static String createEmptyTemplate(final String id) {
-        // e.g., { "index_patterns": [ ".monitoring-data-2*" ], "version": 6000002 }
+        // e.g., { "index_patterns": [ ".monitoring-data-6*" ], "version": 6000002 }
         return "{\"index_patterns\":[\".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION + "*\"],\"version\":" + LAST_UPDATED_VERSION + "}";
     }
 
@@ -120,7 +121,7 @@ public final class MonitoringTemplateUtils {
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *
-     * @param id The API version (e.g., "2") to use
+     * @param id The API version (e.g., "6") to use
      * @param type The type of data you want to format for the request
      * @return Never {@code null}. Always an ended-object.
      * @throws IllegalArgumentException if {@code apiVersion} is unrecognized
@@ -131,7 +132,7 @@ public final class MonitoringTemplateUtils {
             case TEMPLATE_VERSION:
                 return emptyPipeline(type);
             case OLD_TEMPLATE_VERSION:
-                return pipelineForApiVersion2(type);
+                return pipelineForApiVersion6(type);
         }
 
         throw new IllegalArgumentException("unrecognized pipeline API version [" + id + "]");
@@ -139,13 +140,6 @@ public final class MonitoringTemplateUtils {
 
     /**
      * Create a pipeline to upgrade documents from {@link MonitoringTemplateUtils#OLD_TEMPLATE_VERSION}
-     * <pre><code>
-     * {
-     *   "description" : "This pipeline upgrades documents ...",
-     *   "version": 6000001,
-     *   "processors": [ ]
-     * }
-     * </code></pre>
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *
@@ -153,81 +147,39 @@ public final class MonitoringTemplateUtils {
      * @return Never {@code null}. Always an ended-object.
      * @see #LAST_UPDATED_VERSION
      */
-    static XContentBuilder pipelineForApiVersion2(final XContentType type) {
+    static XContentBuilder pipelineForApiVersion6(final XContentType type) {
         try {
-            // For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
-            //  to pipelines, then it would better serve this use case
             return XContentBuilder.builder(type.xContent()).startObject()
                     .field("description", "This pipeline upgrades documents from the older version of the Monitoring API to " +
-                                                "the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
-                                                "changes in those older documents before they are indexed from the older version (" +
-                                                OLD_TEMPLATE_VERSION + ").")
+                        "the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
+                        "changes in those older documents before they are indexed from the older version (" +
+                        OLD_TEMPLATE_VERSION + ").")
                     .field("version", LAST_UPDATED_VERSION)
                     .startArray("processors")
                         .startObject()
-                            // Drop the .monitoring-data-2 index and effectively drop unnecessary data (duplicate or simply unused)
+                            // remove the type
                             .startObject("script")
-                                .field("source",
-                                       "boolean legacyIndex = ctx._index == '.monitoring-data-2';" +
-                                       "if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {" +
-                                         "if (ctx._type == 'cluster_info') {" +
-                                           "ctx._type = 'cluster_stats';" +
-                                           "ctx._id = null;" +
-                                         "} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {" +
-                                           "String index = ctx._index;" +
-                                           "Object clusterUuid = ctx.cluster_uuid;" +
-                                           "Object timestamp = ctx.timestamp;" +
-
-                                           "ctx.clear();" +
-
-                                           "ctx._id = 'xpack_monitoring_2_drop_bucket';" +
-                                           "ctx._index = index;" +
-                                           "ctx._type = 'legacy_data';" +
-                                           "ctx.timestamp = timestamp;" +
-                                           "ctx.cluster_uuid = clusterUuid;" +
-                                         "}" +
-                                         "if (legacyIndex) {" +
-                                           "ctx._index = '<.monitoring-es-" + TEMPLATE_VERSION + "-{now}>';" +
-                                         "}" +
-                                       "}")
-                            .endObject()
-                        .endObject()
-                        .startObject()
-                            .startObject("rename")
-                                .field("field", "_type")
-                                .field("target_field", "type")
-                            .endObject()
-                        .endObject()
-                        .startObject()
-                            .startObject("set")
-                                .field("field", "_type")
-                                .field("value", "doc")
+                                .field("source","ctx._type = null" )
                             .endObject()
                         .endObject()
                         .startObject()
+                            // ensure the data lands in the correct index
                             .startObject("gsub")
                                 .field("field", "_index")
-                                .field("pattern", "(.monitoring-\\w+-)2(-.+)")
+                                .field("pattern", "(.monitoring-\\w+-)6(-.+)")
                                 .field("replacement", "$1" + TEMPLATE_VERSION + "$2")
                             .endObject()
                         .endObject()
                     .endArray()
-                    .endObject();
+                .endObject();
         } catch (final IOException e) {
             throw new RuntimeException("Failed to create pipeline to upgrade from older version [" + OLD_TEMPLATE_VERSION +
-                                       "] to the newer version [" + TEMPLATE_VERSION + "].", e);
+                "] to the newer version [" + TEMPLATE_VERSION + "].", e);
         }
     }
 
     /**
      * Create an empty pipeline.
-     * <pre><code>
-     * {
-     *   "description" : "This is a placeholder pipeline ...",
-     *   "version": 6000001,
-     *   "processors": [ ]
-     * }
-     * </code></pre>
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *

x-pack/plugin/core/src/main/resources/monitoring-alerts.json → x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json

@@ -6,12 +6,12 @@
       "number_of_shards": 1,
       "number_of_replicas": 0,
       "auto_expand_replicas": "0-1",
-      "format": 6,
+      "format": 7,
       "codec": "best_compression"
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "timestamp": {

x-pack/plugin/core/src/main/resources/monitoring-beats.json

@@ -5,13 +5,13 @@
   "settings": {
     "index.auto_expand_replicas": "0-1",
     "index.codec": "best_compression",
-    "index.format": 6,
+    "index.format": 7,
     "index.number_of_replicas": 0,
     "index.number_of_shards": 1
   },
   "version": 7000099,
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "beats_state": {

x-pack/plugin/core/src/main/resources/monitoring-es.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "date_detection": false,
       "dynamic": false,
       "properties": {

x-pack/plugin/core/src/main/resources/monitoring-kibana.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "cluster_uuid": {

x-pack/plugin/core/src/main/resources/monitoring-logstash.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "cluster_uuid": {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -592,7 +592,8 @@ public class HttpExporter extends Exporter {
             resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, templateLoader));
         }
 
-        // add old templates, like ".monitoring-data-2" and ".monitoring-es-2" so that other versions can continue to work
+        // Add dummy templates (e.g. ".monitoring-es-6") to enable the ability to check which version of the actual
+        // index template (e.g. ".monitoring-es") should be applied.
         boolean createLegacyTemplates =
                 TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
         if (createLegacyTemplates) {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java

@@ -12,19 +12,26 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
-
 /**
  * {@code TemplateHttpResource}s allow the checking and uploading of templates to a remote cluster.
  * <p>
@@ -87,9 +94,8 @@ public class TemplateHttpResource extends PublishableHttpResource {
      */
     @Override
     protected void doPublish(final RestClient client, final ActionListener<Boolean> listener) {
-        Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
         putResource(client, listener, logger,
-                    "/_template", templateName, parameters, this::templateToHttpEntity, "monitoring template",
+                    "/_template", templateName, Collections.emptyMap(), this::templateToHttpEntity, "monitoring template",
                     resourceOwnerName, "monitoring cluster");
     }
 
@@ -98,8 +104,17 @@ public class TemplateHttpResource extends PublishableHttpResource {
      *
      * @return Never {@code null}.
      */
-    HttpEntity templateToHttpEntity() {
-        return new StringEntity(template.get(), ContentType.APPLICATION_JSON);
+     HttpEntity templateToHttpEntity() {
+        // the internal representation of a template has type nested under mappings.
+        // this uses xContent to help remove the type before sending to the remote cluster
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
+            return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
+        }
     }
 
 }

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.monitoring.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -40,7 +41,8 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
     public static final String MONITORING_ID = "system_id";
     public static final String MONITORING_VERSION = "system_api_version";
     public static final String INTERVAL = "interval";
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestMonitoringBulkAction.class));
+    private static final Logger logger = LogManager.getLogger(RestMonitoringBulkAction.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
     private final Map<MonitoredSystem, List<String>> supportedApiVersions;
 
     public RestMonitoringBulkAction(Settings settings, RestController controller) {
@@ -50,10 +52,6 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
             POST, "/_xpack/monitoring/_bulk", deprecationLogger);
         controller.registerWithDeprecatedHandler(PUT, "/_monitoring/bulk", this,
             PUT, "/_xpack/monitoring/_bulk", deprecationLogger);
-        controller.registerWithDeprecatedHandler(POST, "/_monitoring/{type}/bulk", this,
-            POST, "/_xpack/monitoring/{type}/_bulk", deprecationLogger);
-        controller.registerWithDeprecatedHandler(PUT, "/_monitoring/{type}/bulk", this,
-            PUT, "/_xpack/monitoring/{type}/_bulk", deprecationLogger);
 
         final List<String> allVersions = Arrays.asList(
                 MonitoringTemplateUtils.TEMPLATE_VERSION,
@@ -63,8 +61,7 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
         final Map<MonitoredSystem, List<String>> versionsMap = new HashMap<>();
         versionsMap.put(MonitoredSystem.KIBANA, allVersions);
         versionsMap.put(MonitoredSystem.LOGSTASH, allVersions);
-        // Beats did not report data in the 5.x timeline, so it should never send the original version
-        versionsMap.put(MonitoredSystem.BEATS, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
+        versionsMap.put(MonitoredSystem.BEATS, allVersions);
         supportedApiVersions = Collections.unmodifiableMap(versionsMap);
     }
 
@@ -75,7 +72,6 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
 
     @Override
     public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
-        final String defaultType = request.param("type");
 
         final String id = request.param(MONITORING_ID);
         if (Strings.isEmpty(id)) {
@@ -106,7 +102,7 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
         final long intervalMillis = parseTimeValue(intervalAsString, INTERVAL).getMillis();
 
         final MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
-        requestBuilder.add(system, defaultType, request.content(), request.getXContentType(), timestamp, intervalMillis);
+        requestBuilder.add(system, request.content(), request.getXContentType(), timestamp, intervalMillis);
         return channel -> requestBuilder.execute(new RestBuilderListener<MonitoringBulkResponse>(channel) {
             @Override
             public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json

@@ -2,7 +2,7 @@
   "metadata": {
     "name": "X-Pack Monitoring: Cluster Status (${monitoring.watch.cluster_uuid})",
     "xpack": {
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "link": "elasticsearch/indices",
       "severity": 2100,
@@ -71,7 +71,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -99,7 +99,7 @@
               "request": {
                 "search_type": "query_then_fetch",
                 "indices": [
-                  ".monitoring-kibana-6-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -140,7 +140,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
+        "index": ".monitoring-alerts-7",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json

@@ -2,7 +2,7 @@
   "metadata": {
     "name": "X-Pack Monitoring: Nodes Changed (${monitoring.watch.cluster_uuid})",
     "xpack": {
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "link": "elasticsearch/nodes",
       "severity": 1999,
@@ -76,7 +76,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -116,7 +116,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-kibana-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -157,7 +157,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6"
+        "index": ".monitoring-alerts-7"
       }
     },
     "send_email_to_admin": {

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json

@@ -2,7 +2,7 @@
   "metadata": {
     "name": "X-Pack Monitoring: Elasticsearch Version Mismatch (${monitoring.watch.cluster_uuid})",
     "xpack": {
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "link": "elasticsearch/nodes",
       "severity": 1000,
@@ -67,7 +67,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -95,7 +95,7 @@
               "request": {
                 "search_type": "query_then_fetch",
                 "indices": [
-                  ".monitoring-kibana-6-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -136,7 +136,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
+        "index": ".monitoring-alerts-7",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json

@@ -2,7 +2,7 @@
   "metadata": {
     "name": "X-Pack Monitoring: Kibana Version Mismatch (${monitoring.watch.cluster_uuid})",
     "xpack": {
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "link": "kibana/instances",
       "severity": 1000,
@@ -87,7 +87,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -115,7 +115,7 @@
               "request": {
                 "search_type": "query_then_fetch",
                 "indices": [
-                  ".monitoring-kibana-6-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -156,7 +156,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
+        "index": ".monitoring-alerts-7",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json

@@ -2,7 +2,7 @@
   "metadata": {
     "name": "X-Pack Monitoring: Logstash Version Mismatch (${monitoring.watch.cluster_uuid})",
     "xpack": {
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "link": "logstash/instances",
       "severity": 1000,
@@ -87,7 +87,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -115,7 +115,7 @@
               "request": {
                 "search_type": "query_then_fetch",
                 "indices": [
-                  ".monitoring-kibana-6-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -156,7 +156,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
+        "index": ".monitoring-alerts-7",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json

@@ -5,7 +5,7 @@
       "link": "license",
       "expires_days": [ 60, 30, 14, 7 ],
       "severity": 0,
-      "alert_index": ".monitoring-alerts-6",
+      "alert_index": ".monitoring-alerts-7",
       "cluster_uuid": "${monitoring.watch.cluster_uuid}",
       "type": "monitoring",
       "version_created": 7000099,
@@ -72,7 +72,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-alerts-6"
+                  ".monitoring-alerts-7"
                 ],
                 "body": {
                   "size": 1,
@@ -99,7 +99,7 @@
             "search": {
               "request": {
                 "indices": [
-                  ".monitoring-kibana-6-*"
+                  ".monitoring-kibana-7-*"
                 ],
                 "body": {
                   "size": 1,
@@ -140,7 +140,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
+        "index": ".monitoring-alerts-7",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java

@@ -124,7 +124,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
         final long interval = randomNonNegativeLong();
 
         final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
-        bulkRequest.add(system, defaultType, content.bytes(), xContentType, timestamp, interval);
+        bulkRequest.add(system, content.bytes(), xContentType, timestamp, interval);
 
         final Collection<MonitoringBulkDoc> bulkDocs = bulkRequest.getDocs();
         assertNotNull(bulkDocs);
@@ -184,7 +184,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
 
         final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-            bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
+            bulkRequest.add(randomFrom(MonitoredSystem.values()), content.bytes(), xContentType, 0L, 0L)
         );
 
         assertThat(e.getMessage(), containsString("source is missing for monitoring document [][doc][" + nbDocs + "]"));
@@ -221,7 +221,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
 
         final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-                bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
+                bulkRequest.add(randomFrom(MonitoredSystem.values()), content.bytes(), xContentType, 0L, 0L)
         );
 
         assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]"));

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtilsTests.java

@@ -58,7 +58,7 @@ public class MonitoringTemplateUtilsTests extends ESTestCase {
         assertTemplate(source, equalTo("{\n" +
                 "  \"index_patterns\": \".monitoring-data-" + TEMPLATE_VERSION + "\",\n" +
                 "  \"mappings\": {\n" +
-                "    \"doc\": {\n" +
+                "    \"_doc\": {\n" +
                 "      \"_meta\": {\n" +
                 "        \"template.version\": \"" + TEMPLATE_VERSION + "\"\n" +
                 "      }\n" +
@@ -115,5 +115,4 @@ public class MonitoringTemplateUtilsTests extends ESTestCase {
         assertThat(indexName(formatter, MonitoredSystem.BEATS, timestamp),
                 equalTo(".monitoring-beats-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
     }
-
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

@@ -13,6 +13,7 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -21,8 +22,14 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.license.XPackLicenseState;
@@ -60,7 +67,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.indexName;
@@ -288,9 +294,8 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                     recordedRequest = secondWebServer.takeRequest();
                     assertThat(recordedRequest.getMethod(), equalTo("PUT"));
                     assertThat(recordedRequest.getUri().getPath(), equalTo(resourcePrefix + template.v1()));
-                    final Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
-                    assertMonitorVersionQueryString(recordedRequest.getUri().getQuery(), parameters);
-                    assertThat(recordedRequest.getBody(), equalTo(template.v2()));
+                    assertMonitorVersionQueryString(recordedRequest.getUri().getQuery(), Collections.emptyMap());
+                    assertThat(recordedRequest.getBody(), equalTo(getExternalTemplateRepresentation(template.v2())));
                 }
             }
             assertMonitorPipelines(secondWebServer, !pipelineExistsAlready, null, null);
@@ -474,11 +479,13 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
 
                 assertThat(putRequest.getMethod(), equalTo("PUT"));
                 assertThat(putRequest.getUri().getPath(), equalTo(pathPrefix + resourcePrefix + resource.v1()));
-                Map<String, String> parameters = resourcePrefix.startsWith("/_template")
-                    ? Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true")
-                    : Collections.emptyMap();
+                Map<String, String> parameters = Collections.emptyMap();
                 assertMonitorVersionQueryString(putRequest.getUri().getQuery(), parameters);
-                assertThat(putRequest.getBody(), equalTo(resource.v2()));
+                if (resourcePrefix.startsWith("/_template")) {
+                    assertThat(putRequest.getBody(), equalTo(getExternalTemplateRepresentation(resource.v2())));
+                } else {
+                    assertThat(putRequest.getBody(), equalTo(resource.v2()));
+                }
                 assertHeaders(putRequest, customHeaders);
             }
         }
@@ -908,4 +915,12 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
         return expectedTemplateNames;
     }
 
+    private String getExternalTemplateRepresentation(String internalRepresentation) throws IOException {
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, internalRepresentation)) {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, ""), builder);
+            return BytesReference.bytes(builder).utf8ToString();
+        }
+    }
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResourceTests.java

@@ -14,10 +14,8 @@ import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collections;
-import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -26,14 +24,20 @@ import static org.hamcrest.Matchers.is;
 public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTestCase {
 
     private final String templateName = ".my_template";
-    private final String templateValue = "{\"template\":\".xyz-*\",\"mappings\":{}}";
-    private final Supplier<String> template = () -> templateValue;
+
+    //the internal representation has the type, the external representation should not
+    private final String templateValueInternal = "{\"order\":0,\"index_patterns\":[\".xyz-*\"],\"settings\":{},\"mappings\":{\"_doc\"" +
+        ":{\"properties\":{\"one\":{\"properties\":{\"two\":{\"properties\":{\"name\":{\"type\":\"keyword\"}}}}}}}},\"aliases\":{}}";
+    private final String templateValueExternal = "{\"order\":0,\"index_patterns\":[\".xyz-*\"],\"settings\":{},\"mappings\"" +
+        ":{\"properties\":{\"one\":{\"properties\":{\"two\":{\"properties\":{\"name\":{\"type\":\"keyword\"}}}}}}},\"aliases\":{}}";
+    private final Supplier<String> template = () -> templateValueInternal;
     private final int minimumVersion = Math.min(MonitoringTemplateUtils.LAST_UPDATED_VERSION, Version.CURRENT.id);
 
     private final TemplateHttpResource resource = new TemplateHttpResource(owner, masterTimeout, templateName, template);
 
     public void testTemplateToHttpEntity() throws IOException {
-        final byte[] templateValueBytes = templateValue.getBytes(ContentType.APPLICATION_JSON.getCharset());
+        //the internal representation is converted to the external representation for the resource
+        final byte[] templateValueBytes = templateValueExternal.getBytes(ContentType.APPLICATION_JSON.getCharset());
         final HttpEntity entity = resource.templateToHttpEntity();
 
         assertThat(entity.getContentType().getValue(), is(ContentType.APPLICATION_JSON.toString()));
@@ -80,13 +84,11 @@ public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTe
     }
 
     public void testDoPublishTrue() {
-        Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
-        assertPublishSucceeds(resource, "/_template", templateName, parameters, StringEntity.class);
+        assertPublishSucceeds(resource, "/_template", templateName, Collections.emptyMap(), StringEntity.class);
     }
 
     public void testDoPublishFalseWithException() {
-        Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
-        assertPublishWithException(resource, "/_template", templateName, parameters, StringEntity.class);
+        assertPublishWithException(resource, "/_template", templateName, Collections.emptyMap(), StringEntity.class);
     }
 
     public void testParameters() {

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java

@@ -218,7 +218,7 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
      */
     private void checkMonitoringTemplates() {
         final Set<String> templates = new HashSet<>();
-        templates.add(".monitoring-alerts");
+        templates.add(".monitoring-alerts-7");
         templates.add(".monitoring-es");
         templates.add(".monitoring-kibana");
         templates.add(".monitoring-logstash");

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java

@@ -81,8 +81,8 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
                 .field("index.number_of_replicas", 0)
             .endObject()
             .startObject("mappings")
-                // Still need use type, RestPutIndexTemplateAction#prepareRequestSource has logic that adds type if missing
-                .startObject("doc")
+                // The internal representation still requires a default type of _doc
+                .startObject("_doc")
                     .startObject("_meta")
                         .field("test", true)
                     .endObject()
@@ -194,7 +194,7 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
         final String name = MonitoringTemplateUtils.templateName(system.getSystem());
 
         for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
-            final String docMapping = template.getMappings().get("doc").toString();
+            final String docMapping = template.getMappings().get("_doc").toString();
 
             assertThat(docMapping, notNullValue());
             assertThat(docMapping, containsString("test"));

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -105,13 +105,13 @@ public class MonitoringIT extends ESSingleNodeTestCase {
     }
 
     private String createBulkEntity() {
-        return "{\"index\":{}}\n" +
-               "{\"foo\":{\"bar\":0}}\n" +
-               "{\"index\":{}}\n" +
-               "{\"foo\":{\"bar\":1}}\n" +
-               "{\"index\":{}}\n" +
-               "{\"foo\":{\"bar\":2}}\n" +
-               "\n";
+        return  "{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
+                "{\"foo\":{\"bar\":0}}\n" +
+                "{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
+                "{\"foo\":{\"bar\":1}}\n" +
+                "{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
+                "{\"foo\":{\"bar\":2}}\n" +
+                "\n";
     }
 
     /**
@@ -128,7 +128,7 @@ public class MonitoringIT extends ESSingleNodeTestCase {
 
             final MonitoringBulkResponse bulkResponse =
                     new MonitoringBulkRequestBuilder(client())
-                            .add(system, "monitoring_data_type", new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
+                            .add(system, new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
                                  System.currentTimeMillis(), interval.millis())
                     .get();
 

x-pack/plugin/monitoring/src/test/resources/monitoring-test.json

@@ -1,7 +1,7 @@
 {
   "index_patterns": ".monitoring-data-${monitoring.template.version}",
   "mappings": {
-    "doc": {
+    "_doc": {
       "_meta": {
         "template.version": "${monitoring.template.version}"
       }