EQL: implement between function (#54277)

* EQL: implement between function

* Address WIP TODOs. Add more tests

* Fix linter complaint

* Add more query folder tests to cover invalid parameter types

* Update Between toDefault. Fix typo in BetweenFunctionProcessor. Add more tests.

* Remove unneeded null checks in BetweenFunctionProcessor::doProcess

* Address code review comments

* Address additional code review comments

* Remove dependency on QL from eql/qa project

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java

@@ -117,6 +117,7 @@ public abstract class CommonEqlActionTestCase extends ESRestTestCase {
 
         // Load EQL validation specs
         List<EqlSpec> specs = EqlSpecLoader.load("/test_queries.toml", true);
+        specs.addAll(EqlSpecLoader.load("/test_queries_supported.toml", true));
         List<EqlSpec> unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false);
 
         // Validate only currently supported specs

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -0,0 +1,15 @@
+# This file is populated with additional EQL queries that were not present in the original EQL python implementation
+# test_queries.toml file in order to keep the original unchanges and easier to sync with the EQL reference implementation tests.
+
+
+[[queries]]
+expected_event_ids = [95]
+query = '''
+file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something"
+'''
+
+[[queries]]
+expected_event_ids = [95]
+query = '''
+file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
+'''

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

@@ -1040,48 +1040,17 @@ query = "file where serial_event_id / 2 == 41"
 expected_event_ids = [82]
 query = "file where serial_event_id % 40 == 2"
 
-[[queries]]
-expected_event_ids = [1, 2]
-query = '''
-process where between(process_name, "s", "e") == "yst"
-'''
-
-[[queries]]
-expected_event_ids = [1, 2]
-query = '''
-process where between(process_name, "s", "e", false) == "yst"
-'''
-
-[[queries]]
-expected_event_ids = []
-query = '''
-process where between(process_name, "s", "e", false, true) == "yst"
-'''
-
-[[queries]]
-expected_event_ids = [1, 2, 42]
-query = '''
-process where between(process_name, "s", "e", false, true) == "t"
-'''
-
-[[queries]]
-expected_event_ids = [1, 2]
-query = '''
-process where between(process_name, "S", "e", false, true) == "yst"
-'''
-
-[[queries]]
-expected_event_ids = [1]
-query = '''
-process where between(process_name, "s", "e", true) == "ystem Idle Proc"
-'''
-
+# The following two "between" queries behave slightly different with elasticsearch
+# due to comparison on keyword field would be case-sensitive and would need to be
+# file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something"
+# blacklisted, check for modified query in test_queries_supported.toml
 [[queries]]
 expected_event_ids = [95]
 query = '''
 file where between(file_path, "dev", ".json", false) == "\\testlogs\\something"
 '''
 
+# blacklisted, check for modified query in test_queries_supported.toml
 [[queries]]
 expected_event_ids = [95]
 query = '''

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
@@ -27,6 +28,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
         // Scalar functions
         // String
             new FunctionDefinition[] {
+                def(Between.class, Between::new, 2, "between"),
                 def(EndsWith.class, EndsWith::new, "endswith"),
                 def(Length.class, Length::new, "length"),
                 def(StartsWith.class, StartsWith::new, "startswith"),

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.function.OptionalArgument;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isBoolean;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific between function.
+ * between(source, left, right[, greedy=false, case_sensitive=false])
+ * Extracts a substring from source that’s between left and right substrings
+ */
+public class Between extends ScalarFunction implements OptionalArgument {
+
+    private final Expression source, left, right, greedy, caseSensitive;
+
+    public Between(Source source, Expression src, Expression left, Expression right, Expression greedy, Expression caseSensitive) {
+        super(source, Arrays.asList(src, left, right, toDefault(greedy), toDefault(caseSensitive)));
+        this.source = src;
+        this.left = left;
+        this.right = right;
+        this.greedy = arguments().get(3);
+        this.caseSensitive = arguments().get(4);
+    }
+
+    private static Expression toDefault(Expression exp) {
+        return exp != null ? exp : Literal.FALSE;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isStringAndExact(source, sourceText(), Expressions.ParamOrdinal.FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        resolution = isStringAndExact(left, sourceText(), Expressions.ParamOrdinal.SECOND);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        resolution = isStringAndExact(right, sourceText(), Expressions.ParamOrdinal.THIRD);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        resolution = isBoolean(greedy, sourceText(), Expressions.ParamOrdinal.FOURTH);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        return isBoolean(caseSensitive, sourceText(), Expressions.ParamOrdinal.FIFTH);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new BetweenFunctionPipe(source(), this, Expressions.pipe(source),
+                Expressions.pipe(left), Expressions.pipe(right),
+                Expressions.pipe(greedy), Expressions.pipe(caseSensitive));
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable() && left.foldable() && right.foldable() && greedy.foldable() && caseSensitive.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), left.fold(), right.fold(), greedy.fold(), caseSensitive.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Between::new, source, left, right, greedy, caseSensitive);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate leftScript = asScript(left);
+        ScriptTemplate rightScript = asScript(right);
+        ScriptTemplate greedyScript = asScript(greedy);
+        ScriptTemplate caseSensitiveScript = asScript(caseSensitive);
+
+        return asScriptFrom(sourceScript, leftScript, rightScript, greedyScript, caseSensitiveScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate leftScript,
+                                          ScriptTemplate rightScript, ScriptTemplate greedyScript, ScriptTemplate caseSensitiveScript) {
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s,%s,%s,%s)"),
+                "between",
+                sourceScript.template(),
+                leftScript.template(),
+                rightScript.template(),
+                greedyScript.template(),
+                caseSensitiveScript.template()),
+                paramsBuilder()
+                        .script(sourceScript.params())
+                        .script(leftScript.params())
+                        .script(rightScript.params())
+                        .script(greedyScript.params())
+                        .script(caseSensitiveScript.params())
+                        .build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
+                paramsBuilder().variable(field.exactAttribute().name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 5) {
+            throw new IllegalArgumentException("expected [5] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Between(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3), newChildren.get(4));
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class BetweenFunctionPipe extends Pipe {
+
+    private final Pipe source, left, right, greedy, caseSensitive;
+
+    public BetweenFunctionPipe(Source source, Expression expression, Pipe src, Pipe left, Pipe right, Pipe greedy, Pipe caseSensitive) {
+        super(source, expression, Arrays.asList(src, left, right, greedy, caseSensitive));
+        this.source = src;
+        this.left = left;
+        this.right = right;
+        this.greedy = greedy;
+        this.caseSensitive = caseSensitive;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        if (newChildren.size() != 5) {
+            throw new IllegalArgumentException("expected [5] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3), newChildren.get(4));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newSource = source.resolveAttributes(resolver);
+        Pipe newLeft = left.resolveAttributes(resolver);
+        Pipe newRight = right.resolveAttributes(resolver);
+        Pipe newGreedy = greedy.resolveAttributes(resolver);
+        Pipe newCaseSensitive = caseSensitive.resolveAttributes(resolver);
+        if (newSource == source && newLeft == left && newRight == right && newGreedy == greedy && newCaseSensitive == caseSensitive) {
+            return this;
+        }
+        return replaceChildren(newSource, newLeft, newRight, newGreedy, newCaseSensitive);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() && left.supportedByAggsOnlyQuery() && right.supportedByAggsOnlyQuery()
+                && greedy.supportedByAggsOnlyQuery() && caseSensitive.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && left.resolved() && right.resolved() && greedy.resolved() && caseSensitive.resolved();
+    }
+
+    protected Pipe replaceChildren(Pipe newSource, Pipe newLeft, Pipe newRight, Pipe newGreedy, Pipe newCaseSensitive) {
+        return new BetweenFunctionPipe(source(), expression(), newSource, newLeft, newRight, newGreedy, newCaseSensitive);
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        left.collectFields(sourceBuilder);
+        right.collectFields(sourceBuilder);
+        greedy.collectFields(sourceBuilder);
+        caseSensitive.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<BetweenFunctionPipe> info() {
+        return NodeInfo.create(this, BetweenFunctionPipe::new, expression(), source, left, right, greedy, caseSensitive);
+    }
+
+    @Override
+    public BetweenFunctionProcessor asProcessor() {
+        return new BetweenFunctionProcessor(source.asProcessor(), left.asProcessor(), right.asProcessor(),
+                greedy.asProcessor(), caseSensitive.asProcessor());
+    }
+
+    public Pipe src() {
+        return source;
+    }
+
+    public Pipe left() {
+        return left;
+    }
+
+    public Pipe right() {
+        return right;
+    }
+
+    public Pipe greedy() {
+        return greedy;
+    }
+
+    public Pipe caseSensitive() {
+        return caseSensitive;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), left(), right(), greedy(), caseSensitive());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BetweenFunctionPipe other = (BetweenFunctionPipe) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right())
+                && Objects.equals(greedy(), other.greedy())
+                && Objects.equals(caseSensitive(), other.caseSensitive());
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.util.Check;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class BetweenFunctionProcessor implements Processor {
+
+    public static final String NAME = "sbtw";
+
+    private final Processor source, left, right, greedy, caseSensitive;
+
+    public BetweenFunctionProcessor(Processor source, Processor left, Processor right, Processor greedy, Processor caseSensitive) {
+        this.source = source;
+        this.left = left;
+        this.right = right;
+        this.greedy = greedy;
+        this.caseSensitive = caseSensitive;
+    }
+
+    public BetweenFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        left = in.readNamedWriteable(Processor.class);
+        right = in.readNamedWriteable(Processor.class);
+        greedy = in.readNamedWriteable(Processor.class);
+        caseSensitive = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(left);
+        out.writeNamedWriteable(right);
+        out.writeNamedWriteable(greedy);
+        out.writeNamedWriteable(caseSensitive);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input), left.process(input), right.process(input),
+                greedy.process(input), caseSensitive.process(input));
+    }
+
+    public static Object doProcess(Object source, Object left, Object right, Object greedy, Object caseSensitive) {
+        if (source == null) {
+            return null;
+        }
+
+        Check.isString(source);
+        Check.isString(left);
+        Check.isString(right);
+
+        Check.isBoolean(greedy);
+        Check.isBoolean(caseSensitive);
+
+        String str = source.toString();
+        String strRight = right.toString();
+        String strLeft = left.toString();
+        boolean bGreedy = ((Boolean) greedy).booleanValue();
+        boolean bCaseSensitive = ((Boolean) caseSensitive).booleanValue();
+        return StringUtils.between(str, strLeft, strRight, bGreedy, bCaseSensitive);
+    }
+
+    protected Processor source() {
+        return source;
+    }
+
+    public Processor left() {
+        return left;
+    }
+
+    public Processor right() {
+        return right;
+    }
+
+    public Processor greedy() {
+        return greedy;
+    }
+
+    public Processor caseSensitive() {
+        return caseSensitive;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), left(), right(), greedy(), caseSensitive());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BetweenFunctionProcessor other = (BetweenFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(left(), other.left())
+                && Objects.equals(right(), other.right())
+                && Objects.equals(greedy(), other.greedy())
+                && Objects.equals(caseSensitive(), other.caseSensitive());
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java

@@ -8,12 +8,58 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string;
 
 import org.elasticsearch.common.Strings;
 
+import java.util.Locale;
+
 import static org.elasticsearch.common.Strings.hasLength;
+import static org.elasticsearch.xpack.ql.util.StringUtils.EMPTY;
 
 final class StringUtils {
 
     private StringUtils() {}
 
+    /**
+     * Extracts a substring from string between left and right strings.
+     * Port of "between" function from the original EQL python implementation.
+     *
+     * @param string        string to search.
+     * @param left          left bounding substring to search for.
+     * @param right         right bounding substring to search for.
+     * @param greedy        match the longest substring if true.
+     * @param caseSensitive match case when searching for {@code left} and {@code right} strings.
+     * @return the substring in between {@code left} and {@code right} strings.
+     */
+    static String between(String string, String left, String right, boolean greedy, boolean caseSensitive) {
+        if (hasLength(string) == false || hasLength(left) == false || hasLength(right) == false) {
+            return string;
+        }
+
+        String matchString = string;
+        if (caseSensitive == false) {
+            matchString = matchString.toLowerCase(Locale.ROOT);
+            left = left.toLowerCase(Locale.ROOT);
+            right = right.toLowerCase(Locale.ROOT);
+        }
+
+        int idx = matchString.indexOf(left);
+        if (idx == -1) {
+            return EMPTY;
+        }
+
+        int start = idx + left.length();
+
+        if (greedy) {
+            idx = matchString.lastIndexOf(right);
+        } else {
+            idx = matchString.indexOf(right, start);
+        }
+
+        if (idx == -1) {
+            return EMPTY;
+        }
+
+        return string.substring(start, idx);
+    }
+
     /**
      * Returns a substring using the Python slice semantics, meaning
      * start and end can be negative

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor;
@@ -21,6 +22,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
 
     InternalEqlScriptUtils() {}
 
+    public static String between(String s, String left, String right, Boolean greedy, Boolean caseSensitive) {
+        return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive);
+    }
+
     public static Boolean endsWith(String s, String pattern) {
         return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -55,6 +55,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 #
 # ASCII Functions
 # 
+  String  between(String, String, String, Boolean, Boolean)
   Boolean endsWith(String, String)
   Integer length(String)
   Boolean startsWith(String, String)

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -135,12 +135,8 @@ public class VerifierTests extends ESTestCase {
                 error("process where serial_event_id == number('5')"));
         assertEquals("1:15: Unknown function [concat]",
                 error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
-        assertEquals("1:15: Unknown function [between]",
-                error("process where between(process_name, \"s\", \"e\") == \"yst\""));
         assertEquals("1:15: Unknown function [cidrMatch]",
                 error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")"));
-        assertEquals("1:22: Unknown function [between]",
-                error("process where length(between(process_name, 'g', 'e')) > 0"));
     }
 
     // Test unsupported array indexes

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.util.StringUtils;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class BetweenFunctionProcessorTests extends ESTestCase {
+    public void testNullOrEmptyParameters() throws Exception {
+        String left = randomBoolean() ? null : randomAlphaOfLength(10);
+        String right = randomBoolean() ? null : randomAlphaOfLength(10);
+        Boolean greedy = randomBoolean() ? null : randomBoolean();
+        Boolean caseSensitive = randomBoolean() ? null : randomBoolean();
+
+        String source = randomBoolean() ? null : StringUtils.EMPTY;
+
+        // The source parameter can be null. Expect exception if any of other parameters is null.
+        if ((source != null) && (left == null || right == null || greedy == null || caseSensitive == null)) {
+            QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
+                    () -> BetweenFunctionProcessor.doProcess(source, left, right, greedy, caseSensitive));
+            if (left == null || right == null) {
+                assertThat(e.getMessage(), equalTo("A string/char is required; received [null]"));
+            } else {
+                assertThat(e.getMessage(), equalTo("A boolean is required; received [null]"));
+            }
+        } else {
+            assertThat(BetweenFunctionProcessor.doProcess(source, left, right, greedy, caseSensitive), equalTo(source));
+        }
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java

@@ -9,6 +9,8 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StringUtils.substringSlice;
+import static org.elasticsearch.xpack.ql.util.StringUtils.EMPTY;
+import static org.hamcrest.Matchers.equalTo;
 
 public class StringUtilsTests extends ESTestCase {
 
@@ -72,4 +74,68 @@ public class StringUtilsTests extends ESTestCase {
     public void testNullValue() {
         assertNull(substringSlice(null, 0, 0));
     }
+
+    public void testBetweenNullOrEmptyString() throws Exception {
+        String left = randomAlphaOfLength(10);
+        String right = randomAlphaOfLength(10);
+        boolean greedy = randomBoolean();
+        boolean caseSensitive = randomBoolean();
+
+        String string = randomBoolean() ? null : EMPTY;
+        assertThat(StringUtils.between(string, left, right, greedy, caseSensitive), equalTo(string));
+    }
+
+    public void testBetweenEmptyNullLeftRight() throws Exception {
+        String string = randomAlphaOfLength(10);
+        String left = randomBoolean() ? null : "";
+        String right = randomBoolean() ? null : "";
+        boolean greedy = randomBoolean();
+        boolean caseSensitive = randomBoolean();
+        assertThat(StringUtils.between(string, left, right, greedy, caseSensitive), equalTo(string));
+    }
+
+    // Test from EQL doc https://eql.readthedocs.io/en/latest/query-guide/functions.html
+    public void testBetweenBasicEQLExamples() {
+        assertThat(StringUtils.between("welcome to event query language", " ", " ", false, false),
+                equalTo("to"));
+        assertThat(StringUtils.between("welcome to event query language", " ", " ", true, false),
+                equalTo("to event query"));
+        assertThat(StringUtils.between("System Idle Process", "s", "e", true, false),
+                equalTo("ystem Idle Proc"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", false, false),
+                equalTo("\\TestLogs\\something"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", true, false),
+                equalTo("\\TestLogs\\something"));
+
+        assertThat(StringUtils.between("System Idle Process", "s", "e", false, false),
+                equalTo("yst"));
+
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", false, true),
+                equalTo("\\TestLogs\\something"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "Test", ".json", false, true),
+                equalTo("Logs\\something"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "test", ".json", false, true),
+                equalTo(""));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", true, true),
+                equalTo("\\TestLogs\\something"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "Test", ".json", true, true),
+                equalTo("Logs\\something"));
+
+        assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "test", ".json", true, true),
+                equalTo(""));
+
+        assertThat(StringUtils.between("System Idle Process", "S", "e", false, true),
+                equalTo("yst"));
+
+        assertThat(StringUtils.between("System Idle Process", "Y", "e", false, true),
+                equalTo(""));
+
+    }
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -80,4 +80,49 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
         assertEquals("Found 1 problem\n" +
             "line 1:15: first argument of [wildcard(pid, '*.exe')] must be [string], found value [pid] type [long]", msg);
     }
+
+    public void testBetweenMissingOrNullParams() {
+        final String[] queries = {
+            "process where between() == \"yst\"",
+            "process where between(process_name) == \"yst\"",
+            "process where between(process_name, \"s\") == \"yst\"",
+            "process where between(null) == \"yst\"",
+            "process where between(process_name, null) == \"yst\"",
+            "process where between(process_name, \"s\", \"e\", false, false, true) == \"yst\"",
+        };
+
+        for (String query : queries) {
+            ParsingException e = expectThrows(ParsingException.class,
+                    () -> plan(query));
+            assertEquals("line 1:16: error building [between]: expects between three and five arguments", e.getMessage());
+        }
+    }
+
+    private String error(String query) {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan(query));
+
+        assertTrue(e.getMessage().startsWith("Found "));
+        final String header = "Found 1 problem\nline ";
+        return e.getMessage().substring(header.length());
+    }
+
+    public void testBetweenWrongTypeParams() {
+        assertEquals("1:15: second argument of [between(process_name, 1, 2)] must be [string], found value [1] type [integer]",
+                error("process where between(process_name, 1, 2)"));
+
+        assertEquals("1:15: third argument of [between(process_name, \"s\", 2)] must be [string], found value [2] type [integer]",
+                error("process where between(process_name, \"s\", 2)"));
+
+        assertEquals("1:15: fourth argument of [between(process_name, \"s\", \"e\", 1)] must be [boolean], found value [1] type [integer]",
+                error("process where between(process_name, \"s\", \"e\", 1)"));
+
+        assertEquals("1:15: fourth argument of [between(process_name, \"s\", \"e\", \"true\")] must be [boolean], " +
+                        "found value [\"true\"] type [keyword]",
+                error("process where between(process_name, \"s\", \"e\", \"true\")"));
+
+        assertEquals("1:15: fifth argument of [between(process_name, \"s\", \"e\", false, 2)] must be [boolean], " +
+                        "found value [2] type [integer]",
+                error("process where between(process_name, \"s\", \"e\", false, 2)"));
+    }
 }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -103,6 +103,13 @@ InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),p
 "params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"}
 
 
+betweenFunction
+process where between(process_name, "s", "e") == "yst"
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3,params.v4),params.v5))",
+"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}
+
+
 wildcardFunctionSingleArgument
 process where wildcard(process_path, "*\\red_ttp\\wininit.*")
 "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
@@ -119,4 +126,3 @@ process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def
 "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
 "wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
 "wildcard":{"process_path":{"wildcard":"*def*"
-

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java

@@ -31,7 +31,8 @@ public final class Expressions {
         FIRST,
         SECOND,
         THIRD,
-        FOURTH;
+        FOURTH,
+        FIFTH;
 
         public static ParamOrdinal fromIndex(int index) {
             switch (index) {
@@ -39,6 +40,7 @@ public final class Expressions {
                 case 1: return ParamOrdinal.SECOND;
                 case 2: return ParamOrdinal.THIRD;
                 case 3: return ParamOrdinal.FOURTH;
+                case 4: return ParamOrdinal.FIFTH;
                 default: return ParamOrdinal.DEFAULT;
             }
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java

@@ -33,6 +33,16 @@ import static java.util.stream.Collectors.toList;
 
 public class FunctionRegistry {
 
+    // Translation table for error messaging in the following function
+    private static final String[] NUM_NAMES = {
+            "zero",
+            "one",
+            "two",
+            "three",
+            "four",
+            "five",
+    };
+
     // list of functions grouped by type of functions (aggregate, statistics, math etc) and ordered alphabetically inside each group
     // a single function will have one entry for itself with its name associated to its instance and, also, one entry for each alias
     // it has with the alias name associated to the FunctionDefinition instance
@@ -403,6 +413,36 @@ public class FunctionRegistry {
         T build(Source source, Expression src, Expression exp1, Expression exp2, Expression exp3);
     }
 
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    public static <T extends Function> FunctionDefinition def(Class<T> function,
+                                                              FiveParametersFunctionBuilder<T> ctorRef,
+                                                              int numOptionalParams, String... names) {
+        FunctionBuilder builder = (source, children, distinct, cfg) -> {
+            final int NUM_TOTAL_PARAMS = 5;
+            boolean hasOptionalParams = OptionalArgument.class.isAssignableFrom(function);
+            if (hasOptionalParams && (children.size() > NUM_TOTAL_PARAMS || children.size() < NUM_TOTAL_PARAMS - numOptionalParams)) {
+                throw new QlIllegalArgumentException("expects between " + NUM_NAMES[NUM_TOTAL_PARAMS - numOptionalParams]
+                        + " and " + NUM_NAMES[NUM_TOTAL_PARAMS] + " arguments");
+            } else if (hasOptionalParams == false && children.size() != NUM_TOTAL_PARAMS) {
+                throw new QlIllegalArgumentException("expects exactly " + NUM_NAMES[NUM_TOTAL_PARAMS] + " arguments");
+            }
+            if (distinct) {
+                throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified");
+            }
+            return ctorRef.build(source,
+                    children.size() > 0 ? children.get(0) : null,
+                    children.size() > 1 ? children.get(1) : null,
+                    children.size() > 2 ? children.get(2) : null,
+                    children.size() > 3 ? children.get(3) : null,
+                    children.size() > 4 ? children.get(4) : null);
+        };
+        return def(function, builder, false, names);
+    }
+
+    protected interface FiveParametersFunctionBuilder<T> {
+        T build(Source source, Expression src, Expression exp1, Expression exp2, Expression exp3, Expression exp4);
+    }
+
     /**
      * Special method to create function definition for Cast as its
      * signature is not compatible with {@link UnresolvedFunction}

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java

@@ -35,4 +35,16 @@ public abstract class Check {
             throw new QlIllegalArgumentException(message, values);
         }
     }
+
+    public static void isString(Object obj) {
+        if (!(obj instanceof String || obj instanceof Character)) {
+            throw new QlIllegalArgumentException("A string/char is required; received [{}]", obj);
+        }
+    }
+
+    public static void isBoolean(Object obj) {
+        if (!(obj instanceof Boolean)) {
+            throw new QlIllegalArgumentException("A boolean is required; received [{}]", obj);
+        }
+    }
 }