new ScriptProcessor for Ingest (#18193)

add new ScriptProcessor for executing ES Scripts within pipelines

core/src/main/java/org/elasticsearch/ingest/IngestService.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Processor;
@@ -55,8 +56,8 @@ public class IngestService implements Closeable {
         return pipelineExecutionService;
     }
 
-    public void setScriptService(ScriptService scriptService) {
-        pipelineStore.buildProcessorFactoryRegistry(processorsRegistryBuilder, scriptService);
+    public void buildProcessorsFactoryRegistry(ScriptService scriptService, ClusterService clusterService) {
+        pipelineStore.buildProcessorFactoryRegistry(processorsRegistryBuilder, scriptService, clusterService);
     }
 
     public IngestInfo info() {

core/src/main/java/org/elasticsearch/ingest/PipelineStore.java

@@ -66,9 +66,8 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
         super(settings);
     }
 
-    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService) {
-        TemplateService templateService = new InternalTemplateService(scriptService);
-        this.processorRegistry = processorsRegistryBuilder.build(templateService);
+    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService, ClusterService clusterService) {
+        this.processorRegistry = processorsRegistryBuilder.build(scriptService, clusterService);
     }
 
     @Override

core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java

@@ -20,9 +20,10 @@
 package org.elasticsearch.ingest;
 
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
 import org.elasticsearch.ingest.core.TemplateService;
+import org.elasticsearch.script.ScriptService;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -31,21 +32,39 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.BiFunction;
+import java.util.function.Function;
 
 public final class ProcessorsRegistry implements Closeable {
 
     private final Map<String, Processor.Factory> processorFactories;
+    private final TemplateService templateService;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
 
-    private ProcessorsRegistry(TemplateService templateService,
-                               Map<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> providers) {
+    private ProcessorsRegistry(ScriptService scriptService, ClusterService clusterService,
+                               Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers) {
+        this.templateService = new InternalTemplateService(scriptService);
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
         Map<String, Processor.Factory> processorFactories = new HashMap<>();
-        for (Map.Entry<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
-            processorFactories.put(entry.getKey(), entry.getValue().apply(templateService, this));
+        for (Map.Entry<String, Function<ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
+            processorFactories.put(entry.getKey(), entry.getValue().apply(this));
         }
         this.processorFactories = Collections.unmodifiableMap(processorFactories);
     }
 
+    public TemplateService getTemplateService() {
+        return templateService;
+    }
+
+    public ScriptService getScriptService() {
+        return scriptService;
+    }
+
+    public ClusterService getClusterService() {
+        return clusterService;
+    }
+
     public Processor.Factory getProcessorFactory(String name) {
         return processorFactories.get(name);
     }
@@ -68,20 +87,20 @@ public final class ProcessorsRegistry implements Closeable {
 
     public static final class Builder {
 
-        private final Map<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
+        private final Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
 
         /**
          * Adds a processor factory under a specific name.
          */
-        public void registerProcessor(String name, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> provider) {
-            BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
+        public void registerProcessor(String name, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
+            Function<ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
             if (previous != null) {
                 throw new IllegalArgumentException("Processor factory already registered for name [" + name + "]");
             }
         }
 
-        public ProcessorsRegistry build(TemplateService templateService) {
-            return new ProcessorsRegistry(templateService, providers);
+        public ProcessorsRegistry build(ScriptService scriptService, ClusterService clusterService) {
+            return new ProcessorsRegistry(scriptService, clusterService, providers);
         }
 
     }

core/src/main/java/org/elasticsearch/node/NodeModule.java

@@ -23,11 +23,10 @@ import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.service.NodeService;
 
-import java.util.function.BiFunction;
+import java.util.function.Function;
 
 /**
  *
@@ -71,7 +70,7 @@ public class NodeModule extends AbstractModule {
     /**
      * Adds a processor factory under a specific type name.
      */
-    public void registerProcessor(String type, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> provider) {
+    public void registerProcessor(String type, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
         processorsRegistryBuilder.registerProcessor(type, provider);
     }
 }

core/src/main/java/org/elasticsearch/node/service/NodeService.java

@@ -62,6 +62,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final CircuitBreakerService circuitBreakerService;
     private final IngestService ingestService;
     private final SettingsFilter settingsFilter;
+    private ClusterService clusterService;
     private ScriptService scriptService;
 
     @Nullable
@@ -87,6 +88,7 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.version = version;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
+        this.clusterService = clusterService;
         this.ingestService = new IngestService(settings, threadPool, processorsRegistryBuilder);
         this.settingsFilter = settingsFilter;
         clusterService.add(ingestService.getPipelineStore());
@@ -97,7 +99,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     @Inject(optional = true)
     public void setScriptService(ScriptService scriptService) {
         this.scriptService = scriptService;
-        this.ingestService.setScriptService(scriptService);
+        this.ingestService.buildProcessorsFactoryRegistry(scriptService, clusterService);
     }
 
     public void setHttpServer(@Nullable HttpServer httpServer) {

core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
@@ -27,6 +28,7 @@ import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -58,8 +60,8 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor);
         Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
         ProcessorsRegistry.Builder processorRegistryBuilder = new ProcessorsRegistry.Builder();
-        processorRegistryBuilder.registerProcessor("mock_processor", ((templateService, registry) -> mock(Processor.Factory.class)));
-        ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(TestTemplateService.instance());
+        processorRegistryBuilder.registerProcessor("mock_processor", ((registry) -> mock(Processor.Factory.class)));
+        ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(mock(ScriptService.class), mock(ClusterService.class));
         store = mock(PipelineStore.class);
         when(store.get(SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
         when(store.getProcessorRegistry()).thenReturn(processorRegistry);

core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java

@@ -65,7 +65,7 @@ public class IngestCloseIT extends ESSingleNodeTestCase {
         }
 
         public void onModule(NodeModule nodeModule) {
-            nodeModule.registerProcessor("test", (templateService, registry) -> new Factory());
+            nodeModule.registerProcessor("test", (registry) -> new Factory());
         }
     }
 

core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -36,6 +37,7 @@ import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.ingest.core.ProcessorInfo;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -51,6 +53,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class PipelineStoreTests extends ESTestCase {
 
@@ -60,7 +63,7 @@ public class PipelineStoreTests extends ESTestCase {
     public void init() throws Exception {
         store = new PipelineStore(Settings.EMPTY);
         ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
-        registryBuilder.registerProcessor("set", (templateService, registry) -> config -> {
+        registryBuilder.registerProcessor("set", (registry) -> config -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             return new Processor() {
@@ -80,7 +83,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        registryBuilder.registerProcessor("remove", (templateService, registry) -> config -> {
+        registryBuilder.registerProcessor("remove", (registry) -> config -> {
             String field = (String) config.remove("field");
             return new Processor() {
                 @Override
@@ -99,7 +102,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        store.buildProcessorFactoryRegistry(registryBuilder, null);
+        store.buildProcessorFactoryRegistry(registryBuilder, mock(ScriptService.class), mock(ClusterService.class));
     }
 
     public void testUpdatePipelines() {

core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java

@@ -19,28 +19,31 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.mockito.Mockito.mock;
 
 public class ProcessorsRegistryTests extends ESTestCase {
 
     public void testBuildProcessorRegistry() {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         TestProcessor.Factory factory1 = new TestProcessor.Factory();
-        builder.registerProcessor("1", (templateService, registry) -> factory1);
+        builder.registerProcessor("1", (registry) -> factory1);
         TestProcessor.Factory factory2 = new TestProcessor.Factory();
-        builder.registerProcessor("2", (templateService, registry) -> factory2);
+        builder.registerProcessor("2", (registry) -> factory2);
         TestProcessor.Factory factory3 = new TestProcessor.Factory();
         try {
-            builder.registerProcessor("1", (templateService, registry) -> factory3);
+            builder.registerProcessor("1", (registry) -> factory3);
             fail("addProcessor should have failed");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Processor factory already registered for name [1]"));
         }
 
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
         assertThat(registry.getProcessorFactories().size(), equalTo(2));
         assertThat(registry.getProcessorFactory("1"), sameInstance(factory1));
         assertThat(registry.getProcessorFactory("2"), sameInstance(factory2));

core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java

@@ -20,9 +20,13 @@
 package org.elasticsearch.ingest.core;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.junit.Before;
 
 import java.util.ArrayList;
@@ -96,8 +100,8 @@ public class ConfigurationUtilsTests extends ESTestCase {
     public void testReadProcessors() throws Exception {
         Processor processor = mock(Processor.class);
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        builder.registerProcessor("test_processor", (templateService, registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        builder.registerProcessor("test_processor", (registry) -> config -> processor);
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
 
 
         List<Map<String, Map<String, Object>>> config = new ArrayList<>();

core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java

@@ -20,10 +20,14 @@
 package org.elasticsearch.ingest.core;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -34,6 +38,7 @@ import java.util.Map;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class PipelineFactoryTests extends ESTestCase {
 
@@ -152,8 +157,8 @@ public class PipelineFactoryTests extends ESTestCase {
     private ProcessorsRegistry createProcessorRegistry(Map<String, Processor.Factory> processorRegistry) {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         for (Map.Entry<String, Processor.Factory> entry : processorRegistry.entrySet()) {
-            builder.registerProcessor(entry.getKey(), ((templateService, registry) -> entry.getValue()));
+            builder.registerProcessor(entry.getKey(), ((registry) -> entry.getValue()));
         }
-        return builder.build(TestTemplateService.instance());
+        return builder.build(mock(ScriptService.class), mock(ClusterService.class));
     }
 }

docs/reference/ingest/ingest-node.asciidoc

@@ -1258,6 +1258,46 @@ Renames an existing field. If the field doesn't exist or the new name is already
 }
 --------------------------------------------------
 
+[[script-processor]]
+=== Script Processor
+
+Allows inline, stored, and file scripts to be executed within ingest pipelines.
+
+See <<modules-scripting-using, How to use scripts>> to learn more about writing scripts. The Script Processor
+leverages caching of compiled scripts for improved performance. Since the
+script specified within the processor is potentially re-compiled per document, it is important
+to understand how script caching works. To learn more about
+caching see <<modules-scripting-using-caching, Script Caching>>.
+
+[[script-options]]
+.Script Options
+[options="header"]
+|======
+| Name                   | Required  | Default | Description
+| `field`                | yes       | -       | The field to set
+| `lang`                 | no        | -       | The scripting language
+| `file`                 | no        | -       | The script file to refer to
+| `id`                   | no        | -       | The stored script id to refer to
+| `inline`               | no        | -       | An inline script to be executed
+|======
+
+You can access the current ingest document from within the script context by using the `ctx` variable.
+
+The following example sets a new field called `field_a_plus_b` to be the sum of two existing
+numeric fields `field_a` and `field_b`:
+
+[source,js]
+--------------------------------------------------
+{
+  "script": {
+    "field": "field_a_plus_b",
+    "lang": "painless",
+    "inline": "return ctx.field_a + ctx.field_b"
+  }
+}
+--------------------------------------------------
+
+
 [[set-processor]]
 === Set Processor
 Sets one field and associates it with the specified value. If the field already exists,

docs/reference/modules/scripting/using.asciidoc

@@ -43,7 +43,6 @@ GET my_index/_search
 -------------------------------------
 // CONSOLE
 
-
 [float]
 === Script Parameters
 
@@ -236,6 +235,18 @@ DELETE _scripts/groovy/calculate-score
 // CONSOLE
 // TEST[continued]
 
+[float]
+[[modules-scripting-using-caching]]
+=== Script Caching
+
+All scripts are cached by default so that they only need to be recompiled
+when updates occur. File scripts keep a static cache and will always reside
+in memory. Both inline and stored scripts are stored in a cache that can evict
+residing scripts. By default, scripts do not have a time-based expiration, but
+you can change this behavior by using the `script.cache.expire` setting.
+You can configure the size of this cache by using the `script.cache.max_size` setting.
+By default, the cache size is `100`.
+
 NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
 changed by setting `script.max_size_in_bytes` setting to increase that soft
 limit, but if scripts are really large then alternatives like

modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java

@@ -52,23 +52,25 @@ public class IngestCommonPlugin extends Plugin {
     }
 
     public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor(DateProcessor.TYPE, (templateService, registry) -> new DateProcessor.Factory());
-        nodeModule.registerProcessor(SetProcessor.TYPE, (templateService, registry) -> new SetProcessor.Factory(templateService));
-        nodeModule.registerProcessor(AppendProcessor.TYPE, (templateService, registry) -> new AppendProcessor.Factory(templateService));
-        nodeModule.registerProcessor(RenameProcessor.TYPE, (templateService, registry) -> new RenameProcessor.Factory());
-        nodeModule.registerProcessor(RemoveProcessor.TYPE, (templateService, registry) -> new RemoveProcessor.Factory(templateService));
-        nodeModule.registerProcessor(SplitProcessor.TYPE, (templateService, registry) -> new SplitProcessor.Factory());
-        nodeModule.registerProcessor(JoinProcessor.TYPE, (templateService, registry) -> new JoinProcessor.Factory());
-        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (templateService, registry) -> new UppercaseProcessor.Factory());
-        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (templateService, registry) -> new LowercaseProcessor.Factory());
-        nodeModule.registerProcessor(TrimProcessor.TYPE, (templateService, registry) -> new TrimProcessor.Factory());
-        nodeModule.registerProcessor(ConvertProcessor.TYPE, (templateService, registry) -> new ConvertProcessor.Factory());
-        nodeModule.registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
-        nodeModule.registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
-        nodeModule.registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
-        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
-        nodeModule.registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
-        nodeModule.registerProcessor(GrokProcessor.TYPE, (templateService, registry) -> new GrokProcessor.Factory(builtinPatterns));
+        nodeModule.registerProcessor(DateProcessor.TYPE, (registry) -> new DateProcessor.Factory());
+        nodeModule.registerProcessor(SetProcessor.TYPE, (registry) -> new SetProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(AppendProcessor.TYPE, (registry) -> new AppendProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(RenameProcessor.TYPE, (registry) -> new RenameProcessor.Factory());
+        nodeModule.registerProcessor(RemoveProcessor.TYPE, (registry) -> new RemoveProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(SplitProcessor.TYPE, (registry) -> new SplitProcessor.Factory());
+        nodeModule.registerProcessor(JoinProcessor.TYPE, (registry) -> new JoinProcessor.Factory());
+        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (registry) -> new UppercaseProcessor.Factory());
+        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (registry) -> new LowercaseProcessor.Factory());
+        nodeModule.registerProcessor(TrimProcessor.TYPE, (registry) -> new TrimProcessor.Factory());
+        nodeModule.registerProcessor(ConvertProcessor.TYPE, (registry) -> new ConvertProcessor.Factory());
+        nodeModule.registerProcessor(GsubProcessor.TYPE, (registry) -> new GsubProcessor.Factory());
+        nodeModule.registerProcessor(FailProcessor.TYPE, (registry) -> new FailProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(ForEachProcessor.TYPE, (registry) -> new ForEachProcessor.Factory(registry));
+        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (registry) -> new DateIndexNameProcessor.Factory());
+        nodeModule.registerProcessor(SortProcessor.TYPE, (registry) -> new SortProcessor.Factory());
+        nodeModule.registerProcessor(GrokProcessor.TYPE, (registry) -> new GrokProcessor.Factory(builtinPatterns));
+        nodeModule.registerProcessor(ScriptProcessor.TYPE, (registry) ->
+            new ScriptProcessor.Factory(registry.getScriptService(), registry.getClusterService()));
     }
 
     // Code for loading built-in grok patterns packaged with the jar file:

modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.ingest.core.AbstractProcessor;
+import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.Strings.hasLength;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.readOptionalStringProperty;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.script.ScriptService.ScriptType.FILE;
+import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
+import static org.elasticsearch.script.ScriptService.ScriptType.STORED;
+
+/**
+ * Processor that adds new fields with their corresponding values. If the field is already present, its value
+ * will be replaced with the provided one.
+ */
+public final class ScriptProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "script";
+
+    private final Script script;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
+    private final String field;
+
+    ScriptProcessor(String tag, Script script, ScriptService scriptService, ClusterService clusterService, String field)  {
+        super(tag);
+        this.script = script;
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
+        this.field = field;
+    }
+
+    @Override
+    public void execute(IngestDocument document) {
+        Map<String, Object> vars = new HashMap<>();
+        vars.put("ctx", document.getSourceAndMetadata());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST, emptyMap(), clusterService.state());
+        ExecutableScript executableScript = scriptService.executable(compiledScript, vars);
+        Object value = executableScript.run();
+        if (field != null) {
+            document.setFieldValue(field, value);
+        }
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static final class Factory extends AbstractProcessorFactory<ScriptProcessor> {
+
+        private final ScriptService scriptService;
+        private final ClusterService clusterService;
+
+        public Factory(ScriptService scriptService, ClusterService clusterService) {
+            this.scriptService = scriptService;
+            this.clusterService = clusterService;
+        }
+
+        @Override
+        public ScriptProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+            String field = readOptionalStringProperty(TYPE, processorTag, config, "field");
+            String lang = readStringProperty(TYPE, processorTag, config, "lang");
+            String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
+            String file = readOptionalStringProperty(TYPE, processorTag, config, "file");
+            String id = readOptionalStringProperty(TYPE, processorTag, config, "id");
+
+            boolean containsNoScript = !hasLength(file) && !hasLength(id) && !hasLength(inline);
+            if (containsNoScript) {
+                throw newConfigurationException(TYPE, processorTag, null, "Need [file], [id], or [inline] parameter to refer to scripts");
+            }
+
+            boolean moreThanOneConfigured = (Strings.hasLength(file) && Strings.hasLength(id)) ||
+                (Strings.hasLength(file) && Strings.hasLength(inline)) || (Strings.hasLength(id) && Strings.hasLength(inline));
+            if (moreThanOneConfigured) {
+                throw newConfigurationException(TYPE, processorTag, null, "Only one of [file], [id], or [inline] may be configured");
+            }
+
+            final Script script;
+            if (Strings.hasLength(file)) {
+                script = new Script(file, FILE, lang, emptyMap());
+            } else if (Strings.hasLength(inline)) {
+                script = new Script(inline, INLINE, lang, emptyMap());
+            } else if (Strings.hasLength(id)) {
+                script = new Script(id, STORED, lang, emptyMap());
+            } else {
+                throw newConfigurationException(TYPE, processorTag, null, "Could not initialize script");
+            }
+
+            return new ScriptProcessor(processorTag, script, scriptService, clusterService, field);
+        }
+    }
+}

modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -27,13 +29,15 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.mockito.Mockito.mock;
+
 public class ForEachProcessorFactoryTests extends ESTestCase {
 
     public void testCreate() throws Exception {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         Processor processor = new TestProcessor(ingestDocument -> {});
-        builder.registerProcessor("_name", (templateService, registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        builder.registerProcessor("_name", (registry) -> config -> processor);
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(registry);
 
         Map<String, Object> config = new HashMap<>();

modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class ScriptProcessorFactoryTests extends ESTestCase {
+
+    private ScriptProcessor.Factory factory;
+
+    @Before
+    public void init() {
+        factory = new ScriptProcessor.Factory(mock(ScriptService.class), mock(ClusterService.class));
+    }
+
+
+    public void testFactoryValidationForMultipleScriptingTypes() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        String randomType = randomFrom("id", "inline", "file");
+        String otherRandomType = randomFrom("id", "inline", "file");
+        while (randomType.equals(otherRandomType)) {
+            otherRandomType = randomFrom("id", "inline", "file");
+        }
+
+        configMap.put(randomType, "foo");
+        configMap.put(otherRandomType, "bar");
+        configMap.put("field", "my_field");
+        configMap.put("lang", "mockscript");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+
+        assertThat(exception.getMessage(), is("[null] Only one of [file], [id], or [inline] may be configured"));
+    }
+
+    public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        configMap.put("field", "my_field");
+        configMap.put("lang", "mockscript");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+
+        assertThat(exception.getMessage(), is("[null] Need [file], [id], or [inline] parameter to refer to scripts"));
+    }
+}

modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ScriptProcessorTests extends ESTestCase {
+
+    public void testScripting() throws Exception {
+        int randomInt = randomInt();
+        ScriptService scriptService = mock(ScriptService.class);
+        ClusterService clusterService = mock(ClusterService.class);
+        CompiledScript compiledScript = mock(CompiledScript.class);
+        Script script = mock(Script.class);
+        when(scriptService.compile(any(), any(), any(), any())).thenReturn(compiledScript);
+        ExecutableScript executableScript = mock(ExecutableScript.class);
+        when(scriptService.executable(any(), any())).thenReturn(executableScript);
+        when(executableScript.run()).thenReturn(randomInt);
+
+        ScriptProcessor processor = new ScriptProcessor(randomAsciiOfLength(10), script,
+            scriptService, clusterService, "bytes_total");
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("bytes_in", 1234);
+        document.put("bytes_out", 4321);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_in"));
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_out"));
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_total"));
+        assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(randomInt));
+    }
+}

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml

@@ -21,8 +21,9 @@
     - match:  { nodes.$master.ingest.processors.9.type: lowercase }
     - match:  { nodes.$master.ingest.processors.10.type: remove }
     - match:  { nodes.$master.ingest.processors.11.type: rename }
-    - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: sort }
-    - match:  { nodes.$master.ingest.processors.14.type: split }
-    - match:  { nodes.$master.ingest.processors.15.type: trim }
-    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.12.type: script }
+    - match:  { nodes.$master.ingest.processors.13.type: set }
+    - match:  { nodes.$master.ingest.processors.14.type: sort }
+    - match:  { nodes.$master.ingest.processors.15.type: split }
+    - match:  { nodes.$master.ingest.processors.16.type: trim }
+    - match:  { nodes.$master.ingest.processors.17.type: uppercase }

plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java

@@ -51,7 +51,7 @@ public class IngestGeoIpPlugin extends Plugin {
     public void onModule(NodeModule nodeModule) throws IOException {
         Path geoIpConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-geoip");
         Map<String, DatabaseReader> databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
-        nodeModule.registerProcessor(GeoIpProcessor.TYPE, (templateService, registry) -> new GeoIpProcessor.Factory(databaseReaders));
+        nodeModule.registerProcessor(GeoIpProcessor.TYPE, (registry) -> new GeoIpProcessor.Factory(databaseReaders));
     }
 
     public static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory) throws IOException {

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -23,11 +23,14 @@ dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
     testCompile project(path: ':plugins:ingest-geoip', configuration: 'runtime')
     testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
+    testCompile project(path: ':modules:lang-painless', configuration: 'runtime')
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
 
 integTest {
     cluster {
         plugin 'ingest-geoip', project(':plugins:ingest-geoip')
+        setting 'script.inline', 'true'
+        setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
     }
-}
+}

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java → qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java

@@ -29,10 +29,12 @@ import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
+import org.mockito.internal.util.collections.Sets;
 
+import java.util.Arrays;
 import java.util.Collections;
 
-public abstract class AbstractMustacheTestCase extends ESTestCase {
+public abstract class AbstractScriptTestCase extends ESTestCase {
 
     protected TemplateService templateService;
 
@@ -43,14 +45,13 @@ public abstract class AbstractMustacheTestCase extends ESTestCase {
             .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .build();
         MustacheScriptEngineService mustache = new MustacheScriptEngineService(settings);
-        ScriptEngineRegistry scriptEngineRegistry =
-                new ScriptEngineRegistry(Collections.singletonList(
-                                new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
-                                                                                  MustacheScriptEngineService.NAME,
-                                                                                  true)));
+
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(
+            new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService scriptService = new ScriptService(settings, new Environment(settings), Collections.singleton(mustache), null,
+
+        ScriptService scriptService = new ScriptService(settings, new Environment(settings), Sets.newSet(mustache), null,
                 scriptEngineRegistry, scriptContextRegistry, scriptSettings);
         templateService = new InternalTemplateService(scriptService);
     }

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java

@@ -31,7 +31,7 @@ import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class IngestDocumentMustacheIT extends AbstractMustacheTestCase {
+public class IngestDocumentMustacheIT extends AbstractScriptTestCase {
 
     public void testAccessMetaDataViaTemplate() {
         Map<String, Object> document = new HashMap<>();

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java

@@ -27,7 +27,7 @@ import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class TemplateServiceIT extends AbstractMustacheTestCase {
+public class TemplateServiceIT extends AbstractScriptTestCase {
 
     public void testTemplates() {
         Map<String, Object> model = new HashMap<>();

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java

@@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
-public class ValueSourceMustacheIT extends AbstractMustacheTestCase {
+public class ValueSourceMustacheIT extends AbstractScriptTestCase {
 
     public void testValueSourceWithTemplates() {
         Map<String, Object> model = new HashMap<>();

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithMustacheIT.java → qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java

@@ -27,9 +27,9 @@ import org.elasticsearch.test.rest.parser.RestTestParseException;
 
 import java.io.IOException;
 
-public class IngestWithMustacheIT extends ESRestTestCase {
+public class IngestWithDependenciesIT extends ESRestTestCase {
 
-    public IngestWithMustacheIT(@Name("yaml") RestTestCandidate testCandidate) {
+    public IngestWithDependenciesIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
 

qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml

@@ -0,0 +1,120 @@
+---
+"Test script processor with inline script":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "inline": "return ctx.bytes_in + ctx.bytes_out"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }
+
+---
+"Test script processor with file script":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "file": "master"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }
+
+---
+"Test script processor with stored script":
+  - do:
+      put_script:
+        id: "sum_bytes"
+        lang: "painless"
+        body: >
+          {
+            "script" : "return ctx.bytes_in + ctx.bytes_out"
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "id" : "sum_bytes"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }

qa/smoke-test-ingest-with-all-dependencies/src/test/resources/scripts/master.painless

@@ -0,0 +1 @@
+return ctx.bytes_in + ctx.bytes_out

test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java

@@ -38,7 +38,7 @@ public class IngestTestPlugin extends Plugin {
     }
 
     public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor("test", (templateService, registry) -> config ->
+        nodeModule.registerProcessor("test", (registry) -> config ->
                 new TestProcessor("id", "test", doc -> {
                     doc.setFieldValue("processed", true);
                     if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {