x-pack/plugin/apm-data: map some APM fields as flattened and fix error.grouping_name script (#103032)

Various fixes to the apm-data plugin:

- fix the `error.grouping_name` script to not throw an exception when an `error.exception` object lacks a `message` attribute
- statically map various APM-specific fields as `flattened` to prevent mapping conflicts

Also expand the YAML REST test coverage.

docs/changelog/103032.yaml

@@ -0,0 +1,5 @@
+pr: 103032
+summary: "x-pack/plugin/apm-data: Map some APM fields as flattened and fix error.grouping_name script"
+area: Data streams
+type: enhancement
+issues: []

x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml

@@ -6,21 +6,29 @@ _meta:
 template:
   mappings:
     properties:
+      # error.*
       error.custom:
         type: flattened
       error.exception.attributes:
         type: flattened
       error.exception.stacktrace:
         type: flattened
+      error.log.stacktrace:
+        type: flattened
       error.grouping_name:
         type: keyword
         script: |
           def logMessage = params['_source'].error?.log?.message;
-          if (logMessage != null) {
+          if (logMessage != null && logMessage != "") {
             emit(logMessage);
             return;
           }
           def exception = params['_source'].error?.exception;
-          if (exception != null && exception.length > 0) {
+          def exceptionMessage = exception != null && exception.length > 0 ? exception[0]?.message : null;
+          if (exceptionMessage != null && exceptionMessage != "") {
             emit(exception[0].message);
           }
+
+      # http.*
+      http.request.body:
+        type: flattened

x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml

@@ -6,15 +6,22 @@ _meta:
 template:
   mappings:
     properties:
+      # NOTE(axw) processor.event may be either "span" or "transaction".
+      #
+      # This field should eventually be removed, and we should end up
+      # with only spans. Some of those spans may be identified as local
+      # roots, equivalent in concept to transactions.
       processor.event:
         type: keyword
+
+      # event.*
       event.success_count:
         type: byte
         index: false
-      span.duration.us:
-        type: long
-      transaction.duration.us:
-        type: long
+
+      # http.*
+      http.request.body:
+        type: flattened
       http.response.transfer_size:
         type: long
         index: false
@@ -24,10 +31,22 @@ template:
       http.response.decoded_body_size:
         type: long
         index: false
+
+      # span.*
+      span.duration.us:
+        type: long
       span.representative_count:
         type: scaled_float
         scaling_factor: 1000
         index: false
+      span.stacktrace:
+        type: flattened
+
+      # transaction.*
+      transaction.custom:
+        type: flattened
+      transaction.duration.us:
+        type: long
       transaction.representative_count:
         type: scaled_float
         scaling_factor: 1000

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml

@@ -82,110 +82,3 @@ setup:
   - length: {hits.hits: 1}
   - match: {hits.hits.0.fields.event\.success_count: [1]}
   - match: {hits.hits.0.fields.span\.duration\.us: [123]}
-
----
-"Test metrics-apm.internal-* data stream rerouting":
-  - do:
-      bulk:
-        index: metrics-apm.internal-testing
-        refresh: true
-        body:
-          - create: {}
-          - "@timestamp": "2017-06-22"
-            data_stream.type: metrics
-            data_stream.dataset: apm.internal
-            data_stream.namespace: testing
-            metricset:
-              name: transaction
-          - create: {}
-          - "@timestamp": "2017-06-22"
-            data_stream.type: metrics
-            data_stream.dataset: apm.internal
-            data_stream.namespace: testing
-            metricset:
-              name: service_destination
-          - create: {}
-          - "@timestamp": "2017-06-22"
-            data_stream.type: metrics
-            data_stream.dataset: apm.internal
-            data_stream.namespace: testing
-            metricset:
-              name: app_config # should not be rerouted
-  - do:
-      indices.get_data_stream:
-        name: metrics-apm.transaction.1m-testing
-  - do:
-      indices.get_data_stream:
-        name: metrics-apm.service_destination.1m-testing
-  - do:
-      indices.get_data_stream:
-        name: metrics-apm.internal-testing
-  - do:
-      search:
-        index: metrics-apm*
-  - length: {hits.hits: 3}
-  - match: {hits.hits.0._source.data_stream.dataset: "apm.internal"}
-  - match: {hits.hits.1._source.data_stream.dataset: "apm.service_destination.1m"}
-  - match: {hits.hits.1._source.metricset.interval: "1m"}
-  - match: {hits.hits.2._source.data_stream.dataset: "apm.transaction.1m"}
-  - match: {hits.hits.2._source.metricset.interval: "1m"}
-
----
-"Test metrics-apm.app-* dynamic mapping":
-  - do:
-      bulk:
-        index: metrics-apm.app.svc1-testing
-        refresh: true
-        body:
-          - create: {}
-          - "@timestamp": "2017-06-22"
-            data_stream.type: metrics
-            data_stream.dataset: apm.app.svc1
-            data_stream.namespace: testing
-            metricset:
-              name: app
-              samples:
-                - name: double_metric
-                  type: gauge
-                  value: 123
-                - name: summary_metric
-                  type: summary
-                  value_count: 123
-                  sum: 456.789
-                - name: histogram_metric
-                  type: histogram
-                  counts: [1, 2, 3]
-                  values: [1.5, 2.5, 3.5]
-  - set:
-      items.0.create._index: index
-  - do:
-      # Wait for cluster state changes to be applied before
-      # querying field mappings.
-      cluster.health:
-        wait_for_events: languid
-  - do:
-      indices.get_field_mapping:
-        index: metrics-apm.app.svc1-testing
-        fields: [double_metric, summary_metric, histogram_metric]
-  - match:
-      $body:
-        $index:
-          mappings:
-            double_metric:
-              full_name: double_metric
-              mapping:
-                double_metric:
-                  type: double
-                  index: false
-            summary_metric:
-              full_name: summary_metric
-              mapping:
-                summary_metric:
-                  type: aggregate_metric_double
-                  metrics : [sum, value_count]
-                  default_metric: value_count
-            histogram_metric:
-              full_name: histogram_metric
-              mapping:
-                histogram_metric:
-                  type: histogram

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml

@@ -0,0 +1,56 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test logs-apm.error-* error grouping":
+  - do:
+      bulk:
+        index: logs-apm.error-testing
+        refresh: true
+        body:
+          # No error object field
+          - create: {}
+          - '{"@timestamp": "2017-06-22"}'
+
+          # Empty error object
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {}}'
+
+          # Empty error.log object
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"log": {}}}'
+
+          # Empty error.exception array
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"exception": []}}'
+
+          # Empty error.exception object
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"exception": [{}]}}'
+
+          # Non-empty error.log.message used
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"log": {"message": "log_used"}, "exception": [{"message": "ignored"}]}}'
+
+          # Non-empty error.exception.message used
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"log": {"message": ""}, "exception": [{"message": "exception_used"}]}}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: logs-apm.error-testing
+        body:
+          fields: ["error.grouping_name"]
+  - length: { hits.hits: 7 }
+  - match: { hits.hits.0.fields: null }
+  - match: { hits.hits.1.fields: null }
+  - match: { hits.hits.2.fields: null }
+  - match: { hits.hits.3.fields: null }
+  - match: { hits.hits.4.fields: null }
+  - match: { hits.hits.5.fields: {"error.grouping_name": ["log_used"]} }
+  - match: { hits.hits.6.fields: {"error.grouping_name": ["exception_used"]} }

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_flattened_fields.yml

@@ -0,0 +1,107 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test traces-apm-* flattened fields":
+  - do:
+      bulk:
+        index: traces-apm-testing
+        refresh: true
+        body:
+          # http.request.body should be mapped as flattened, allowing
+          # differing types to be used in http.request.body.original.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "http.request.body": {"original": "text"}}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "http.request.body": {"original": {"field": "value"}}}'
+
+          # span.stacktrace is a complex object whose structure may
+          # change over time, and which is always treated as an object.
+          # Moreover, stacktraces may contain dynamic "vars" whose
+          # types may change from one document to the next.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "span.stacktrace": [{"vars": {"a": 123}}]}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "span.stacktrace": [{"vars": {"a": "b"}}]}'
+
+          # transaction.custom is a complex object of fields with
+          # arbitrary field types that may change from one document
+          # to the next.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "transaction.custom": {"a": {"b": 123}}}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "transaction.custom": {"a": "b"}}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: traces-apm-testing
+        body:
+          fields: ["http.request.body", "span.stacktrace", "transaction.custom"]
+  - length: { hits.hits: 6 }
+  - match: { hits.hits.0.fields: {"http.request.body": [{"original": "text"}]} }
+  - match: { hits.hits.1.fields: {"http.request.body": [{"original": {"field": "value"}}]} }
+  - match: { hits.hits.2.fields: {"span.stacktrace": [{"vars": {"a": 123}}]} }
+  - match: { hits.hits.3.fields: {"span.stacktrace": [{"vars": {"a": "b"}}]} }
+  - match: { hits.hits.4.fields: {"transaction.custom": [{"a": {"b": 123}}]} }
+  - match: { hits.hits.5.fields: {"transaction.custom": [{"a": "b"}]} }
+
+---
+"Test logs-apm.error-* flattened fields":
+  - do:
+      bulk:
+        index: logs-apm.error-testing
+        refresh: true
+        body:
+          # http.request.body has the same requirements as http.request.body
+          # in traces-apm-* data streams.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "http.request.body": {"original": "text"}}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "http.request.body": {"original": {"field": "value"}}}'
+
+          # error.{exception,log}.stacktrace have the same requirements as span.stacktrace.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.exception.stacktrace": [{"vars": {"a": 123}}]}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.exception.stacktrace": [{"vars": {"a": "b"}}]}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.log.stacktrace": [{"vars": {"a": 123}}]}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.log.stacktrace": [{"vars": {"a": "b"}}]}'
+
+          # error.exception.attributes is a complex object with arbitrary field types
+          # that may change from one document to the next.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.exception": [{"attributes": {"a": 123}}]}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.exception": [{"attributes": {"a": "b"}}]}'
+
+          # error.custom has the same requirements as transaction.custom.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.custom": {"a": {"b": 123}}}'
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error.custom": {"a": "b"}}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: logs-apm.error-testing
+        body:
+          fields: ["http.request.body", "error.log.*", "error.exception.*", "error.custom"]
+  - length: { hits.hits: 10 }
+  - match: { hits.hits.0.fields: {"http.request.body": [{"original": "text"}]} }
+  - match: { hits.hits.1.fields: {"http.request.body": [{"original": {"field": "value"}}]} }
+  - match: { hits.hits.2.fields: {"error.exception.stacktrace": [{"vars": {"a": 123}}]} }
+  - match: { hits.hits.3.fields: {"error.exception.stacktrace": [{"vars": {"a": "b"}}]} }
+  - match: { hits.hits.4.fields: {"error.log.stacktrace": [{"vars": {"a": 123}}]} }
+  - match: { hits.hits.5.fields: {"error.log.stacktrace": [{"vars": {"a": "b"}}]} }
+  - match: { hits.hits.6.fields: {"error.exception.attributes": [{"a": 123}]} }
+  - match: { hits.hits.7.fields: {"error.exception.attributes": [{"a": "b"}]} }
+  - match: { hits.hits.8.fields: {"error.custom": [{"a": {"b": 123}}]} }
+  - match: { hits.hits.9.fields: {"error.custom": [{"a": "b"}]} }

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_mapping.yml

@@ -0,0 +1,65 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test metrics-apm.app-* dynamic mapping":
+  - do:
+      bulk:
+        index: metrics-apm.app.svc1-testing
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": "2017-06-22"
+            data_stream.type: metrics
+            data_stream.dataset: apm.app.svc1
+            data_stream.namespace: testing
+            metricset:
+              name: app
+              samples:
+                - name: double_metric
+                  type: gauge
+                  value: 123
+                - name: summary_metric
+                  type: summary
+                  value_count: 123
+                  sum: 456.789
+                - name: histogram_metric
+                  type: histogram
+                  counts: [1, 2, 3]
+                  values: [1.5, 2.5, 3.5]
+  - set:
+      items.0.create._index: index
+  - do:
+      # Wait for cluster state changes to be applied before
+      # querying field mappings.
+      cluster.health:
+        wait_for_events: languid
+  - do:
+      indices.get_field_mapping:
+        index: metrics-apm.app.svc1-testing
+        fields: [double_metric, summary_metric, histogram_metric]
+  - match:
+      $body:
+        $index:
+          mappings:
+            double_metric:
+              full_name: double_metric
+              mapping:
+                double_metric:
+                  type: double
+                  index: false
+            summary_metric:
+              full_name: summary_metric
+              mapping:
+                summary_metric:
+                  type: aggregate_metric_double
+                  metrics : [sum, value_count]
+                  default_metric: value_count
+            histogram_metric:
+              full_name: histogram_metric
+              mapping:
+                histogram_metric:
+                  type: histogram

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_rerouting.yml

@@ -0,0 +1,52 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test metrics-apm.internal-* data stream rerouting":
+  - do:
+      bulk:
+        index: metrics-apm.internal-testing
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": "2017-06-22"
+            data_stream.type: metrics
+            data_stream.dataset: apm.internal
+            data_stream.namespace: testing
+            metricset:
+              name: transaction
+          - create: {}
+          - "@timestamp": "2017-06-22"
+            data_stream.type: metrics
+            data_stream.dataset: apm.internal
+            data_stream.namespace: testing
+            metricset:
+              name: service_destination
+          - create: {}
+          - "@timestamp": "2017-06-22"
+            data_stream.type: metrics
+            data_stream.dataset: apm.internal
+            data_stream.namespace: testing
+            metricset:
+              name: app_config # should not be rerouted
+  - do:
+      indices.get_data_stream:
+        name: metrics-apm.transaction.1m-testing
+  - do:
+      indices.get_data_stream:
+        name: metrics-apm.service_destination.1m-testing
+  - do:
+      indices.get_data_stream:
+        name: metrics-apm.internal-testing
+  - do:
+      search:
+        index: metrics-apm*
+  - length: {hits.hits: 3}
+  - match: {hits.hits.0._source.data_stream.dataset: "apm.internal"}
+  - match: {hits.hits.1._source.data_stream.dataset: "apm.service_destination.1m"}
+  - match: {hits.hits.1._source.metricset.interval: "1m"}
+  - match: {hits.hits.2._source.data_stream.dataset: "apm.transaction.1m"}
+  - match: {hits.hits.2._source.metricset.interval: "1m"}

x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_ingest.yml

@@ -0,0 +1,99 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test traces-apm-* processor.event inference":
+  - do:
+      bulk:
+        index: traces-apm-testing
+        refresh: true
+        body:
+          # `processor.event: transaction` is inferred from presence of `transaction.type`
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "transaction": {"type": "foo"}}'
+
+          # `processor.event: span` is inferred otherwise
+          - create: {}
+          - '{"@timestamp": "2017-06-22"}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: traces-apm-testing
+        body:
+          fields: ["processor.event"]
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0.fields: {"processor.event": ["transaction"]} }
+  - match: { hits.hits.1.fields: {"processor.event": ["span"]} }
+
+---
+"Test traces-apm-* setting *.duration.us from event.duration":
+  - do:
+      bulk:
+        index: traces-apm-testing
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "transaction": {"type": "foo"}, "event": {"duration": 1234}}'
+
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "event": {"duration": 1234}}'
+
+          # If event.duration is omitted, it is assumed to be zero.
+          - create: {}
+          - '{"@timestamp": "2017-06-22"}'
+
+          # An existing field will not be overwritten.
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "span": {"duration": {"us": 789}}}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: traces-apm-testing
+        body:
+          fields: ["event.duration", "*.duration.us"]
+  - length: { hits.hits: 4 }
+  - match: { hits.hits.0.fields: {"transaction.duration.us": [1]} }
+  - match: { hits.hits.1.fields: {"span.duration.us": [1]} }
+  - match: { hits.hits.2.fields: {"span.duration.us": [0]} }
+  - match: { hits.hits.3.fields: {"span.duration.us": [789]} }
+
+---
+"Test traces-apm-* setting event.success_count from event.outcome":
+  - do:
+      bulk:
+        index: traces-apm-testing
+        refresh: true
+        body:
+          # No event.outcome, no event.success_count
+          - create: {}
+          - '{"@timestamp": "2017-06-22"}'
+
+          # event.outcome: unknown, no event.success_count
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "event": {"outcome": "unknown"}}'
+
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "event": {"outcome": "success"}}'
+
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "event": {"outcome": "failure"}}'
+
+  - is_false: errors
+
+  - do:
+      search:
+        index: traces-apm-testing
+        body:
+          fields: ["event.success_count"]
+  - length: { hits.hits: 4 }
+  - match: { hits.hits.0.fields: null }
+  - match: { hits.hits.1.fields: null }
+  - match: { hits.hits.2.fields: {"event.success_count": [1]} }
+  - match: { hits.hits.3.fields: {"event.success_count": [0]} }