EQL: startsWith function implementation (#54400)

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

@@ -786,44 +786,12 @@ process where original_file_name == process_name
 expected_event_ids  = [97, 98, 75273, 75303]
 description = "check that case insensitive comparisons are performed for fields."
 
-[[queries]]
-query = '''
-file where opcode=0 and startsWith(file_name, 'exploRER.')
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-query = '''
-file where opcode=0 and startsWith(file_name, 'expLORER.exe')
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
 [[queries]]
 query = '''
 file where opcode=0 and endsWith(file_name, 'loREr.exe')'''
 expected_event_ids  = [88]
 description = "check built-in string functions"
 
-[[queries]]
-query = '''
-file where opcode=0 and startsWith(file_name, 'explORER.EXE')'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-query = '''
-file where opcode=0 and startsWith('explorer.exeaaaaaaaa', file_name)'''
-expected_event_ids  = [88]
-description = "check built-in string functions"
-
-[[queries]]
-query = '''
-file where opcode=0 and serial_event_id = 88 and startsWith('explorer.exeaAAAA', 'EXPLORER.exe')'''
-expected_event_ids  = [88]
-description = "check built-in string functions"
-
 [[queries]]
 query = '''
 file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name)

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.expression.function;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
@@ -25,6 +26,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
         // String
             new FunctionDefinition[] {
                 def(Length.class, Length::new, "length"),
+                def(StartsWith.class, StartsWith::new, "startswith"),
                 def(Substring.class, Substring::new, "substring")
             }
         };

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * Function that checks if first parameter starts with the second parameter. Both parameters should be strings
+ * and the function returns a boolean value. The function is case insensitive.
+ */
+public class StartsWith extends ScalarFunction {
+
+    private final Expression source;
+    private final Expression pattern;
+
+    public StartsWith(Source source, Expression src, Expression pattern) {
+        super(source, Arrays.asList(src, pattern));
+        this.source = src;
+        this.pattern = pattern;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST);
+        if (sourceResolution.unresolved()) {
+            return sourceResolution;
+        }
+
+        return isStringAndExact(pattern, sourceText(), ParamOrdinal.SECOND);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new StartsWithFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(pattern));
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable() && pattern.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), pattern.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, StartsWith::new, source, pattern);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate patternScript = asScript(pattern);
+
+        return asScriptFrom(sourceScript, patternScript);
+    }
+    
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate patternScript) {
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s)"),
+                "startsWith",
+                sourceScript.template(),
+                patternScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params())
+                    .script(patternScript.params())
+                    .build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
+                paramsBuilder().variable(field.exactAttribute().name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 2) {
+            throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]");
+        }
+
+        return new StartsWith(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class StartsWithFunctionPipe extends Pipe {
+
+    private final Pipe source;
+    private final Pipe pattern;
+
+    public StartsWithFunctionPipe(Source source, Expression expression, Pipe src, Pipe pattern) {
+        super(source, expression, Arrays.asList(src, pattern));
+        this.source = src;
+        this.pattern = pattern;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        if (newChildren.size() != 2) {
+            throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newSource = source.resolveAttributes(resolver);
+        Pipe newPattern = pattern.resolveAttributes(resolver);
+        if (newSource == source && newPattern == pattern) {
+            return this;
+        }
+        return replaceChildren(newSource, newPattern);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() && pattern.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && pattern.resolved();
+    }
+
+    protected Pipe replaceChildren(Pipe newSource, Pipe newPattern) {
+        return new StartsWithFunctionPipe(source(), expression(), newSource, newPattern);
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        pattern.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<StartsWithFunctionPipe> info() {
+        return NodeInfo.create(this, StartsWithFunctionPipe::new, expression(), source, pattern);
+    }
+
+    @Override
+    public StartsWithFunctionProcessor asProcessor() {
+        return new StartsWithFunctionProcessor(source.asProcessor(), pattern.asProcessor());
+    }
+    
+    public Pipe src() {
+        return source;
+    }
+
+    public Pipe pattern() {
+        return pattern;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source, pattern);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        StartsWithFunctionPipe other = (StartsWithFunctionPipe) obj;
+        return Objects.equals(source, other.source)
+                && Objects.equals(pattern, other.pattern);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class StartsWithFunctionProcessor implements Processor {
+
+    public static final String NAME = "sstw";
+
+    private final Processor source;
+    private final Processor pattern;
+
+    public StartsWithFunctionProcessor(Processor source, Processor pattern) {
+        this.source = source;
+        this.pattern = pattern;
+    }
+
+    public StartsWithFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        pattern = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(pattern);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input), pattern.process(input));
+    }
+
+    public static Object doProcess(Object source, Object pattern) {
+        if (source == null) {
+            return null;
+        }
+        if (source instanceof String == false && source instanceof Character == false) {
+            throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (pattern == null) {
+            return null;
+        }
+        if (pattern instanceof String == false && pattern instanceof Character == false) {
+            throw new EqlIllegalArgumentException("A string/char is required; received [{}]", pattern);
+        }
+
+        return source.toString().toLowerCase(Locale.ROOT).startsWith(pattern.toString().toLowerCase(Locale.ROOT));
+    }
+    
+    protected Processor source() {
+        return source;
+    }
+
+    protected Processor pattern() {
+        return pattern;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        StartsWithFunctionProcessor other = (StartsWithFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(pattern(), other.pattern());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), pattern());
+    }
+    
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
 
@@ -23,6 +24,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
         return (Integer) LengthFunctionProcessor.doProcess(s);
     }
 
+    public static Boolean startsWith(String s, String pattern) {
+        return (Boolean) StartsWithFunctionProcessor.doProcess(s, pattern);
+    }
+
     public static String substring(String s, Number start, Number end) {
         return (String) SubstringFunctionProcessor.doProcess(s, start, end);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -56,5 +56,6 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 # ASCII Functions
 # 
   Integer length(String)
+  Boolean startsWith(String, String)
   String  substring(String, Number, Number)
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -131,10 +131,8 @@ public class VerifierTests extends ESTestCase {
 
     // Test the known EQL functions that are not supported
     public void testFunctionVerificationUnknown() {
-        assertEquals("1:25: Unknown function [endsWith]",
+        assertEquals("1:25: Unknown function [endsWith], did you mean [startswith]?",
                 error("file where opcode=0 and endsWith(file_name, 'loREr.exe')"));
-        assertEquals("1:25: Unknown function [startsWith]",
-                error("file where opcode=0 and startsWith(file_name, 'explORER.EXE')"));
         assertEquals("1:25: Unknown function [stringContains]",
                 error("file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name)"));
         assertEquals("1:25: Unknown function [indexOf]",

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.LiteralTests;
+
+import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
+import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
+import static org.hamcrest.Matchers.startsWith;
+
+public class StartsWithProcessorTests extends ESTestCase {
+
+    public void testStartsWithFunctionWithValidInput() {
+        assertEquals(true, new StartsWith(EMPTY, l("foobarbar"), l("f")).makePipe().asProcessor().process(null));
+        assertEquals(false, new StartsWith(EMPTY, l("foobar"), l("bar")).makePipe().asProcessor().process(null));
+        assertEquals(false, new StartsWith(EMPTY, l("foo"), l("foobar")).makePipe().asProcessor().process(null));
+        assertEquals(true, new StartsWith(EMPTY, l("foobar"), l("")).makePipe().asProcessor().process(null));
+        assertEquals(true, new StartsWith(EMPTY, l("foo"), l("foo")).makePipe().asProcessor().process(null));
+        assertEquals(true, new StartsWith(EMPTY, l("foo"), l("FO")).makePipe().asProcessor().process(null));
+        assertEquals(true, new StartsWith(EMPTY, l("foo"), l("FOo")).makePipe().asProcessor().process(null));
+        assertEquals(true, new StartsWith(EMPTY, l('f'), l('f')).makePipe().asProcessor().process(null));
+        assertEquals(false, new StartsWith(EMPTY, l(""), l("bar")).makePipe().asProcessor().process(null));
+        assertEquals(null, new StartsWith(EMPTY, l(null), l("bar")).makePipe().asProcessor().process(null));
+        assertEquals(null, new StartsWith(EMPTY, l("foo"), l(null)).makePipe().asProcessor().process(null));
+        assertEquals(null, new StartsWith(EMPTY, l(null), l(null)).makePipe().asProcessor().process(null));
+    }
+    
+    public void testStartsWithFunctionInputsValidation() {
+        QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new StartsWith(EMPTY, l(5), l("foo")).makePipe().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new StartsWith(EMPTY, l("bar"), l(false)).makePipe().asProcessor().process(null));
+        assertEquals("A string/char is required; received [false]", siae.getMessage());
+    }
+
+    public void testStartsWithFunctionWithRandomInvalidDataType() {
+        Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral());
+        QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new StartsWith(EMPTY, literal, l("foo")).makePipe().asProcessor().process(null));
+        assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
+        siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new StartsWith(EMPTY, l("foo"), literal).makePipe().asProcessor().process(null));
+        assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -32,4 +32,12 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
         assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
                 + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
     }
+
+    public void testStartsWithFunctionWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where startsWith(plain_text, \"foo\") == true"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
+                + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
 }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -82,6 +82,13 @@ InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),par
 "params":{"v0":"constant_keyword","v1":5}
 
 
+startsWithFunction
+process where startsWith(user_name, 'A')
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.startsWith(
+InternalQlScriptUtils.docValue(doc,params.v0),params.v1))",
+"params":{"v0":"user_name","v1":"A"}
+
+
 substringFunction
 process where substring(file_name, -4) == '.exe'
 "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java

@@ -91,8 +91,8 @@ public class FunctionRegistry {
     }
 
     public String resolveAlias(String alias) {
-        String upperCase = normalize(alias);
-        return aliases.getOrDefault(upperCase, upperCase);
+        String normalized = normalize(alias);
+        return aliases.getOrDefault(normalized, normalized);
     }
 
     public boolean functionExists(String functionName) {