introduce the JSON Processor (#20128)

introduce the JSON Processor

docs/reference/ingest/ingest-node.asciidoc

@@ -1272,6 +1272,28 @@ Throws an error when the field is not an array.
 }
 --------------------------------------------------
 
+[[json-processor]]
+=== JSON Processor
+Converts a JSON string into a structured JSON object.
+
+[[json-options]]
+.Json Options
+[options="header"]
+|======
+| Name           | Required  | Default  | Description
+| `field`        | yes       | -        | The field to be parsed
+| `target_field` | no        | `field`  | The field to insert the converted structured object into
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "json": {
+    "field": "{\"foo\": 2000}"
+  }
+}
+--------------------------------------------------
+
 [[lowercase-processor]]
 === Lowercase Processor
 Converts a string to its lowercase equivalent.

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.util.Map;
+
+/**
+ * Processor that serializes a string-valued field into a
+ * map of maps.
+ */
+public final class JsonProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "json";
+
+    private final String field;
+    private final String targetField;
+
+    JsonProcessor(String tag, String field, String targetField) {
+        super(tag);
+        this.field = field;
+        this.targetField = targetField;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public String getTargetField() {
+        return targetField;
+    }
+
+    @Override
+    public void execute(IngestDocument document) throws Exception {
+        String stringValue = document.getFieldValue(field, String.class);
+        try {
+            Map<String, Object> mapValue = JsonXContent.jsonXContent.createParser(stringValue).map();
+            document.setFieldValue(targetField, mapValue);
+        } catch (JsonParseException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static final class Factory implements Processor.Factory {
+        @Override
+        public JsonProcessor create(Map<String, Processor.Factory> registry, String processorTag,
+                                    Map<String, Object> config) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
+            return new JsonProcessor(processorTag, field, targetField);
+        }
+    }
+}
+

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class JsonProcessorFactoryTests extends ESTestCase {
+
+    private static final JsonProcessor.Factory FACTORY = new JsonProcessor.Factory();
+
+    public void testCreate() throws Exception {
+        String processorTag = randomAsciiOfLength(10);
+        String randomField = randomAsciiOfLength(10);
+        String randomTargetField = randomAsciiOfLength(5);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        config.put("target_field", randomTargetField);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
+        assertThat(jsonProcessor.getTag(), equalTo(processorTag));
+        assertThat(jsonProcessor.getField(), equalTo(randomField));
+        assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField));
+    }
+
+    public void testCreateWithDefaultTarget() throws Exception {
+        String processorTag = randomAsciiOfLength(10);
+        String randomField = randomAsciiOfLength(10);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", randomField);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
+        assertThat(jsonProcessor.getTag(), equalTo(processorTag));
+        assertThat(jsonProcessor.getField(), equalTo(randomField));
+        assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
+    }
+
+    public void testCreateWithMissingField() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        String processorTag = randomAsciiOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> FACTORY.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
+    }
+}

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
+
+public class JsonProcessorTests extends ESTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testExecute() throws Exception {
+        String processorTag = randomAsciiOfLength(3);
+        String randomField = randomAsciiOfLength(3);
+        String randomTargetField = randomAsciiOfLength(2);
+        JsonProcessor jsonProcessor = new JsonProcessor(processorTag, randomField, randomTargetField);
+        Map<String, Object> document = new HashMap<>();
+
+        Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random());
+        XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
+        String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
+        document.put(randomField, randomJson);
+
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        Map<String, Object> jsonified = ingestDocument.getFieldValue(randomTargetField, Map.class);
+        assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
+    }
+
+    public void testInvalidJson() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field");
+        Map<String, Object> document = new HashMap<>();
+        document.put("field", "invalid json");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("com.fasterxml.jackson.core.JsonParseException: Unrecognized token" +
+            " 'invalid': was expecting ('true', 'false' or 'null')\n" +
+            " at [Source: invalid json; line: 1, column: 8]"));
+    }
+
+    public void testFieldMissing() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field");
+        Map<String, Object> document = new HashMap<>();
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [field] not present as part of path [field]"));
+    }
+}