EQL: Give a name to all toml tests and enforce the naming of new tests (#59283)

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java

@@ -26,7 +26,9 @@ import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.stream.Collectors.toList;
@@ -61,9 +63,10 @@ public abstract class CommonEqlActionTestCase extends ESRestTestCase {
     public static List<Object[]> readTestSpecs() throws Exception {
 
         // Load EQL validation specs
-        List<EqlSpec> specs = EqlSpecLoader.load("/test_queries.toml", true);
-        specs.addAll(EqlSpecLoader.load("/additional_test_queries.toml", true));
-        List<EqlSpec> unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false);
+        Set<String> uniqueTestNames = new HashSet<>();
+        List<EqlSpec> specs = EqlSpecLoader.load("/test_queries.toml", true, uniqueTestNames);
+        specs.addAll(EqlSpecLoader.load("/additional_test_queries.toml", true, uniqueTestNames));
+        List<EqlSpec> unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false, uniqueTestNames);
 
         // Validate only currently supported specs
         List<EqlSpec> filteredSpecs = new ArrayList<>();
@@ -89,7 +92,7 @@ public abstract class CommonEqlActionTestCase extends ESRestTestCase {
     private static List<Object[]> asArray(List<EqlSpec> specs) {
         AtomicInteger counter = new AtomicInteger();
         return specs.stream().map(spec -> {
-            String name = spec.description();
+            String name = spec.name();
             if (Strings.isNullOrEmpty(name)) {
                 name = spec.note();
             }

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java

@@ -12,6 +12,7 @@ import java.util.Arrays;
 import java.util.Objects;
 
 public class EqlSpec {
+    private String name;
     private String description;
     private String note;
     private String[] tags;
@@ -24,6 +25,14 @@ public class EqlSpec {
     // FALSE -> case insensitive
     private Boolean caseSensitive = null;
 
+    public String name() {
+        return name;
+    }
+
+    public void name(String name) {
+        this.name = name;
+    }
+
     public String description() {
         return description;
     }
@@ -76,6 +85,7 @@ public class EqlSpec {
     public String toString() {
         String str = "";
         str = appendWithComma(str, "query", query);
+        str = appendWithComma(str, "name", name);
         str = appendWithComma(str, "description", description);
         str = appendWithComma(str, "note", note);
 

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java

@@ -9,26 +9,40 @@ package org.elasticsearch.test.eql;
 import io.ous.jtoml.JToml;
 import io.ous.jtoml.Toml;
 import io.ous.jtoml.TomlTable;
+
 import org.elasticsearch.common.Strings;
 
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 public class EqlSpecLoader {
-    public static List<EqlSpec> load(String path, boolean supported) throws Exception {
+    public static List<EqlSpec> load(String path, boolean supported, Set<String> uniqueTestNames) throws Exception {
         try (InputStream is = EqlSpecLoader.class.getResourceAsStream(path)) {
-            return readFromStream(is, supported);
+            return readFromStream(is, supported, uniqueTestNames);
         }
     }
 
-    private static void validateAndAddSpec(List<EqlSpec> specs, EqlSpec spec, boolean supported) throws Exception {
+    private static void validateAndAddSpec(List<EqlSpec> specs, EqlSpec spec, boolean supported,
+        Set<String> uniqueTestNames) throws Exception {
+        if (Strings.isNullOrEmpty(spec.name())) {
+            throw new IllegalArgumentException("Read a test without a name value");
+        }
+
         if (Strings.isNullOrEmpty(spec.query())) {
             throw new IllegalArgumentException("Read a test without a query value");
         }
 
-        if (supported && spec.expectedEventIds() == null) {
-            throw new IllegalArgumentException("Read a test without a expected_event_ids value");
+        if (supported) {
+            if (spec.expectedEventIds() == null) {
+                throw new IllegalArgumentException("Read a test without a expected_event_ids value");
+            }
+            if (uniqueTestNames.contains(spec.name())) {
+                throw new IllegalArgumentException("Found a test with the same name as another test: " + spec.name());
+            } else {
+                uniqueTestNames.add(spec.name());
+            }
         }
 
         specs.add(spec);
@@ -42,7 +56,7 @@ public class EqlSpecLoader {
         return null;
     }
 
-    private static List<EqlSpec> readFromStream(InputStream is, boolean supported) throws Exception {
+    private static List<EqlSpec> readFromStream(InputStream is, boolean supported, Set<String> uniqueTestNames) throws Exception {
         List<EqlSpec> testSpecs = new ArrayList<>();
 
         EqlSpec spec;
@@ -52,6 +66,7 @@ public class EqlSpecLoader {
         for (TomlTable table : queries) {
             spec = new EqlSpec();
             spec.query(getTrimmedString(table, "query"));
+            spec.name(getTrimmedString(table, "name"));
             spec.note(getTrimmedString(table, "note"));
             spec.description(getTrimmedString(table, "description"));
 
@@ -88,7 +103,7 @@ public class EqlSpecLoader {
                 }
                 spec.expectedEventIds(expectedEventIds);
             }
-            validateAndAddSpec(testSpecs, spec, supported);
+            validateAndAddSpec(testSpecs, spec, supported, uniqueTestNames);
         }
 
         return testSpecs;

x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml

@@ -2,48 +2,56 @@
 # test_queries.toml file in order to keep the original unchanged and easier to sync with the EQL reference implementation tests.
 
 [[queries]]
+name = "betweenAdditional1"
 expected_event_ids = [95]
 query = '''
 file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something"
 '''
 
 [[queries]]
+name = "betweenAdditional2"
 expected_event_ids = [95]
 query = '''
 file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
 '''
 
 [[queries]]
+name = "stringCidrMatch1"
 expected_event_ids = [75304, 75305]
 query = '''
 network where cidrMatch(source_address, "10.6.48.157/8") == true
 '''
 
 [[queries]]
+name = "stringCidrMatch2"
 expected_event_ids = [75304, 75305]
 query = '''
 network where string(cidrMatch(source_address, "10.6.48.157/8")) == "true"
 '''
 
 [[queries]]
+name = "cidrMatchAdditional2"
 expected_event_ids = [75304, 75305]
 query = '''
 network where true == cidrMatch(source_address, "10.6.48.157/8")
 '''
 
 [[queries]]
+name = "cidrMatchAdditional3"
 expected_event_ids = []
 query = '''
 network where cidrMatch(source_address, "192.168.0.0/16") == true
 '''
 
 [[queries]]
+name = "cidrMatchAdditional4"
 expected_event_ids = [75304, 75305]
 query = '''
 network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") == true
 '''
 
 [[queries]]
+name = "cidrMatchAdditional5"
 expected_event_ids = [75304, 75305]
 query = '''
 network where cidrMatch(source_address, "0.0.0.0/0") == true
@@ -51,6 +59,7 @@ network where cidrMatch(source_address, "0.0.0.0/0") == true
 
 
 [[queries]]
+name = "concatEquals1"
 description = "test string concatenation. update test to avoid case-sensitivity issues"
 query = '''
 process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3'
@@ -59,60 +68,72 @@ expected_event_ids  = [5]
 
 
 [[queries]]
+name = "concatEquals2"
 query = 'process where concat(serial_event_id) = "1"'
 expected_event_ids  = [1]
 
 [[queries]]
+name = "concatWithCondition1"
 query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) != null'
 expected_event_ids  = [2, 3]
 
 
 [[queries]]
+name = "concatWithCondition2"
 query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) == null'
 expected_event_ids  = [1, 4]
 
 
 [[queries]]
+name = "concatWithCondition3"
 query = 'process where serial_event_id < 5 and concat(process_name, null, null) == null'
 expected_event_ids  = [1, 2, 3, 4]
 
 
 [[queries]]
+name = "concatWithCondition4"
 query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
 expected_event_ids  = [1, 2, 3, 4]
 
 
 [[queries]]
+name = "numberStringConversion1"
 query = 'process where string(serial_event_id) = "1"'
 expected_event_ids  = [1]
 
 
 [[queries]]
+name = "numberStringConversion2"
 query = 'any where number(string(serial_event_id)) == 17'
 expected_event_ids = [17]
 
 
 [[queries]]
+name = "numberStringConversion3"
 query = 'any where number(string(serial_event_id), null) == 17'
 expected_event_ids = [17]
 
 
 [[queries]]
+name = "numberStringConversion4"
 query = 'any where number(string(serial_event_id), 10) == 17'
 expected_event_ids = [17]
 
 
 [[queries]]
+name = "numberStringEquality"
 query = 'any where number(string(serial_event_id), 13) == number("31", 13)'
 expected_event_ids = [31]
 
 
 [[queries]]
+name = "numberStringConversion5"
 query = 'any where number(string(serial_event_id), 16) == 17'
 expected_event_ids = [11]
 
 
 [[queries]]
+name = "matchWithCharacterClasses1"
 expected_event_ids  = [98]
 notes = "regexp doesn't support character classes"
 query = '''
@@ -122,12 +143,14 @@ process where match(command_line, ?'.*?net1[ ]+localgroup.*?')
 '''
 
 [[queries]]
+name = "matchLiteAdditional"
 expected_event_ids  = [98]
 query = '''
 process where matchLite(command_line, ?'.*?net1.*?')
 '''
 
 [[queries]]
+name = "matchWithCharacterClasses2"
 expected_event_ids  = [98]
 notes = "regexp doesn't support predefined character classes (like \\s)"
 query = '''
@@ -138,18 +161,21 @@ process where match(command_line, ?'.*?net1[ ]+[a-z]{4,15}[ ]+.*?')
 
 
 [[queries]]
+name = "multiPatternMatch"
 expected_event_ids  = [50, 97, 98]
 query = '''
 process where match(command_line, '.*?net[1]?  localgroup.*?', '.*? myappserver.py .*?')
 '''
 
 [[queries]]
+name = "matchWithSubstring"
 expected_event_ids  = [50, 98]
 query = '''
 process where match(substring(command_line, 5), '.*?net[1]?  localgroup.*?', '.*? myappserver.py .*?')
 '''
 
 [[queries]]
+name = "moduloEqualsField"
 # Basic test for modulo function
 query = '''
 process where modulo(11, 10) == serial_event_id'''
@@ -157,36 +183,44 @@ expected_event_ids  = [1]
 description = "test built-in modulo math functions"
 
 [[queries]]
+name = "additionalMathWithFields1"
 # This query give a different result with ES EQL implementation because it doesn't convert to float data types for division
 expected_event_ids = [82, 83]
 query = "file where serial_event_id / 2 == 41"
 
 # Additional EQL queries with arithmetic operations that were not part of the original EQL implementation
 [[queries]]
+name = "additionalMathWithFields2"
 expected_event_ids = [82]
 query = "file where 83 - serial_event_id == 1"
 
 [[queries]]
+name = "additionalMathWithFields3"
 expected_event_ids = [82]
 query = "file where 1 + serial_event_id == 83"
 
 [[queries]]
+name = "additionalMathWithFields4"
 expected_event_ids = [82]
 query = "file where -serial_event_id + 100 == 18"
 
 [[queries]]
+name = "additionalMathWithFields5"
 expected_event_ids = [82]
 query = "file where 2 * serial_event_id == 164"
 
 [[queries]]
+name = "additionalMathWithFields6"
 expected_event_ids = [66]
 query = "file where 66.0 / serial_event_id == 1"
 
 [[queries]]
+name = "additionalMathWithFields7"
 expected_event_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 46]
 query = "process where serial_event_id + ((1 + 3) * 2 / (3 - 1)) * 2 == 54 or 70 + serial_event_id < 100"
 
 [[queries]]
+name = "twoSequencesAdditional1"
 query = '''
 sequence
   [process where serial_event_id = 1]
@@ -195,6 +229,7 @@ sequence
 expected_event_ids  = [1, 2]
 
 [[queries]]
+name = "twoSequencesAdditional2"
 query = '''
 sequence
   [process where serial_event_id=1] by unique_pid
@@ -202,6 +237,7 @@ sequence
 expected_event_ids  = [1, 2]
 
 [[queries]]
+name = "twoSequencesAdditional3"
 query = '''
 sequence
   [process where serial_event_id<3] by unique_pid
@@ -210,6 +246,7 @@ sequence
 expected_event_ids  = [1, 2, 2, 3]
 
 [[queries]]
+name = "twoSequencesAdditional4"
 query = '''
 sequence
   [process where false] by unique_pid