[DOCS] Refactor EQL docs (#60700)

Changes:

* Moves sample data to reusable rest test
* Combines EQL index, requirements, and run a search pages
* Combines EQL syntax and limitations pages
* Adds related redirects

docs/build.gradle

@@ -190,15 +190,42 @@ buildRestTests.setups['messages'] = '''
           refresh: true
           body: |
             {"index":{"_id": "0"}}
-            {"message": "trying out Elasticsearch" }
+            {"message": "trying out Elasticsearch"}
             {"index":{"_id": "1"}}
-            {"message": "some message with the number 1" }
+            {"message": "some message with the number 1"}
             {"index":{"_id": "2"}}
-            {"message": "some message with the number 2" }
+            {"message": "some message with the number 2"}
             {"index":{"_id": "3"}}
-            {"message": "some message with the number 3" }
+            {"message": "some message with the number 3"}
             {"index":{"_id": "4"}}
-            {"message": "some message with the number 4" }'''
+            {"message": "some message with the number 4"}'''
+
+// Used for EQL
+buildRestTests.setups['sec_logs'] = '''
+  - do:
+        indices.create:
+          index: my-index-000001
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+  - do:
+        bulk:
+          index: my-index-000001
+          refresh: true
+          body: |
+            {"index":{}}
+            {"@timestamp": "2020-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:08.000Z", "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}'''
 
 buildRestTests.setups['host'] = '''
   # Fetch the http host. We use the host of the master because we know there will always be a master.

docs/reference/eql/delete-async-eql-search-api.asciidoc

@@ -27,12 +27,12 @@ DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZM
 [[delete-async-eql-search-api-prereqs]]
 ==== {api-prereq-title}
 
-See <<eql-requirements,EQL requirements>>.
+See <<eql-required-fields>>.
 
 [[delete-async-eql-search-api-limitations]]
 ===== Limitations
 
-See <<eql-limitations,EQL limitations>>.
+See <<eql-syntax-limitations,EQL limitations>>.
 
 [[delete-async-eql-search-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/eql/eql-search-api.asciidoc

@@ -14,26 +14,6 @@ Returns search results for an <<eql,Event Query Language (EQL)>> query.
 In {es}, EQL assumes each document in a data stream or index corresponds to an
 event.
 
-////
-[source,console]
-----
-PUT /my-index-000001/_bulk?refresh
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-----
-// TESTSETUP
-////
-
 [source,console]
 ----
 GET /my-index-000001/_eql/search
@@ -43,6 +23,7 @@ GET /my-index-000001/_eql/search
   """
 }
 ----
+// TEST[setup:sec_logs]
 
 [[eql-search-api-request]]
 ==== {api-request-title}
@@ -54,12 +35,12 @@ GET /my-index-000001/_eql/search
 [[eql-search-api-prereqs]]
 ==== {api-prereq-title}
 
-See <<eql-requirements,EQL requirements>>.
+See <<eql-required-fields>>.
 
 [[eql-search-api-limitations]]
 ===== Limitations
 
-See <<eql-limitations,EQL limitations>>.
+See <<eql-syntax-limitations,EQL limitations>>.
 
 [[eql-search-api-path-params]]
 ==== {api-path-parms-title}
@@ -163,6 +144,9 @@ Field containing the event classification, such as `process`, `file`, or
 Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic
 Common Schema (ECS)]. If a data stream or index does not contain the
 `event.category` field, this value is required.
++
+The event category field is typically mapped as a <<keyword,`keyword`>> or
+<<constant-keyword,constant keyword>> field.
 
 `fetch_size`::
 (Optional, integer)
@@ -275,6 +259,9 @@ does not contain the `@timestamp` field, this value is required.
 Events in the API response are sorted by this field's value, converted to
 milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in
 ascending order.
+
+The timestamp field is typically mapped as a <<date,`date`>> or
+<<date_nanos,`date_nanos`>> field.
 --
 
 [[eql-search-api-wait-for-completion-timeout]]
@@ -506,17 +493,18 @@ The following EQL search request searches for events with an `event.category` of
 `file` that meet the following conditions:
 
 * A `file.name` of `cmd.exe`
-* An `agent.id` other than `8a4f526c`
+* An `process.pid` other than `2013`
 
 [source,console]
 ----
 GET /my-index-000001/_eql/search
 {
   "query": """
-    file where (file.name == "cmd.exe" and agent.id != "8a4f526c")
+    file where (file.name == "cmd.exe" and process.pid != 2013)
   """
 }
 ----
+// TEST[setup:sec_logs]
 // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
 
 The API returns the following response. Matching events in the `hits.events`
@@ -547,9 +535,6 @@ the events in ascending, lexicographic order.
         "_score": null,
         "_source": {
           "@timestamp": "2020-12-06T11:04:07.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
           "event": {
             "category": "file",
             "id": "dGCHwoeS",
@@ -564,7 +549,8 @@ the events in ascending, lexicographic order.
           },
           "process": {
             "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
           }
         }
       },
@@ -574,9 +560,6 @@ the events in ascending, lexicographic order.
         "_score": null,
         "_source": {
           "@timestamp": "2020-12-07T11:07:08.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
           "event": {
             "category": "file",
             "id": "bYA7gPay",
@@ -591,7 +574,8 @@ the events in ascending, lexicographic order.
           },
           "process": {
             "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
           }
         }
       }
@@ -614,7 +598,7 @@ that:
 --
 * An `event.category` of `file`
 * A `file.name` of `cmd.exe`
-* An `agent.id` other than `8a4f526c`
+* An `process.pid` other than `2013`
 --
 . Followed by an event with:
 +
@@ -623,29 +607,24 @@ that:
 * A `process.executable` that contains the substring `regsvr32`
 --
 
-These events must also share the same `agent.id` value.
+These events must also share the same `process.pid` value.
 
 [source,console]
 ----
 GET /my-index-000001/_eql/search
 {
   "query": """
-    sequence by agent.id
-      [ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ]
+    sequence by process.pid
+      [ file where file.name == "cmd.exe" and process.pid != 2013 ]
       [ process where stringContains(process.executable, "regsvr32") ]
   """
 }
 ----
+// TEST[setup:sec_logs]
 
-The API returns the following response. The `hits.sequences.join_keys` property
-contains the shared `agent.id` value for each matching event. Matching events in
-the `hits.sequences.events` property are sorted by
-<<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
-the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
-
-If two or more events share the same timestamp, the
-<<eql-search-api-tiebreaker-field,`tiebreaker_field`>> field is used to sort
-the events in ascending, lexicographic order.
+The API returns the following response. Matching sequences are included in the
+`hits.sequences` property. The `hits.sequences.join_keys` property contains the
+shared `process.pid` value for each matching event.
 
 [source,console-result]
 ----
@@ -662,7 +641,7 @@ the events in ascending, lexicographic order.
     "sequences": [
       {
         "join_keys": [
-          "8a4f500d"
+          "2012"
         ],
         "events": [
           {
@@ -674,9 +653,6 @@ the events in ascending, lexicographic order.
             "_score": null,
             "_source": {
               "@timestamp": "2020-12-07T11:07:08.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
               "event": {
                 "category": "file",
                 "id": "bYA7gPay",
@@ -689,9 +665,10 @@ the events in ascending, lexicographic order.
                 "type": "file",
                 "size": 16384
               },
-              "process": {
+              "process": { 
                 "name": "cmd.exe",
-                "executable": "C:\\Windows\\System32\\cmd.exe"
+                "executable": "C:\\Windows\\System32\\cmd.exe",
+                "pid": 2012
               }
             }
           },
@@ -704,17 +681,15 @@ the events in ascending, lexicographic order.
             "_score": null,
             "_source": {
               "@timestamp": "2020-12-07T11:07:09.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
               "event": {
                 "category": "process",
                 "id": "aR3NWVOs",
                 "sequence": 5
               },
-              "process": {
+              "process": { 
                 "name": "regsvr32.exe",
-                "executable": "C:\\Windows\\System32\\regsvr32.exe"
+                "executable": "C:\\Windows\\System32\\regsvr32.exe",
+                "pid": 2012
               }
             }
           }

docs/reference/eql/eql.asciidoc

@@ -0,0 +1,714 @@
+[role="xpack"]
+[testenv="basic"]
+[[eql]]
+= EQL search
+++++
+<titleabbrev>EQL</titleabbrev>
+++++
+
+experimental::[]
+
+{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for
+event-based, time-series data, such as logs.
+
+[discrete]
+[[eql-advantages]]
+== Advantages of EQL
+
+* *EQL lets you express relationships between events.* +
+Many query languages allow you to match only single events. EQL lets you match a
+sequence of events across different event categories and time spans.
+
+* *EQL has a low learning curve.* +
+EQL syntax looks like other query languages. It lets you write and read queries
+intuitively, which makes for quick, iterative searching.
+
+* *We designed EQL for security use cases.* +
+While you can use EQL for any event-based data, we created EQL for threat
+hunting. EQL not only supports indicator of compromise (IOC) searching but
+makes it easy to describe activity that goes beyond IOCs.
+
+[discrete]
+[[eql-required-fields]]
+== Required fields
+
+EQL assumes each document in a data stream or index corresponds to an event. To
+search using EQL, each document in the searched data stream or index must
+include a _timestamp_ field and an _event category_ field.
+
+{es} EQL uses the `@timestamp` and `event.category` fields from the
+{ecs-ref}[Elastic Common Schema (ECS)] as the default timestamp and event
+category fields. If your searched documents use a different timestamp or event
+category field, you must specify it in the search request. See
+<<specify-a-timestamp-or-event-category-field>>.
+
+[discrete]
+[[run-an-eql-search]]
+== Run an EQL search
+
+You can use the <<eql-search-api,EQL search API>> to run an EQL search.
+
+The following request searches `my-index-000001` for events with an
+`event.category` of `process` and a `process.name` of `cmd.exe`. Each document
+in `my-index-000001` includes a `@timestamp` and `event.category` field.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    process where process.name == "cmd.exe"
+  """
+}
+----
+// TEST[setup:sec_logs]
+// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
+
+The API returns the following response. Matching events are included in the
+`hits.events` property. These events are sorted by timestamp, converted to
+milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in
+ascending order.
+
+[source,console-result]
+----
+{
+  "is_partial": false,
+  "is_running": false,
+  "took": 60,
+  "timed_out": false,
+  "hits": {
+    "total": {
+      "value": 2,
+      "relation": "eq"
+    },
+    "events": [
+       {
+        "_index": "my-index-000001",
+        "_id": "OQmfCaduce8zoHT93o4H",
+        "_score": null,
+        "_source": {
+          "@timestamp": "2020-12-06T11:04:05.000Z",
+          "event": {
+            "category": "process",
+            "id": "edwCRnyD",
+            "sequence": 1
+          },
+          "process": {
+            "name": "cmd.exe",
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
+          }
+        }
+      },
+      {
+        "_index": "my-index-000001",
+        "_id": "xLkCaj4EujzdNSxfYLbO",
+        "_score": null,
+        "_source": {
+          "@timestamp": "2020-12-07T11:06:07.000Z",
+          "event": {
+            "category": "process",
+            "id": "cMyt5SZ2",
+            "sequence": 3
+          },
+          "process": {
+            "name": "cmd.exe",
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
+          }
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took": 60/"took": $body.took/]
+// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
+// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
+
+[discrete]
+[[eql-search-sequence]]
+=== Search for a sequence of events
+
+You can use EQL's <<eql-sequences,sequence syntax>> to search for an ordered
+series of events.
+
+The following EQL search request matches a sequence that:
+
+. Starts with an event with:
++
+--
+* An `event.category` of `file`
+* A `file.name` of `cmd.exe`
+--
+. Followed by an event with:
++
+--
+* An `event.category` of `process`
+* A `process.name` that contains the substring `regsvr32`
+--
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    sequence
+      [ file where file.name == "cmd.exe" ]
+      [ process where stringContains(process.name, "regsvr32") ]
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+The API returns the following response. Matching sequences are included in the
+`hits.sequences` property.
+
+[source,console-result]
+----
+{
+  "is_partial": false,
+  "is_running": false,
+  "took": 60,
+  "timed_out": false,
+  "hits": {
+    "total": {
+      "value": 1,
+      "relation": "eq"
+    },
+    "sequences": [
+      {
+        "events": [
+          {
+            "_index": "my-index-000001",
+            "_id": "AtOJ4UjUBAAx3XR5kcCM",
+            "_version" : 1,
+            "_seq_no" : 3,
+            "_primary_term" : 1,
+            "_score": null,
+            "_source": {
+              "@timestamp": "2020-12-07T11:07:08.000Z",
+              "event": {
+                "category": "file",
+                "id": "bYA7gPay",
+                "sequence": 4
+              },
+              "file": {
+                "accessed": "2020-12-07T11:07:08.000Z",
+                "name": "cmd.exe",
+                "path": "C:\\Windows\\System32\\cmd.exe",
+                "type": "file",
+                "size": 16384
+              },
+              "process": {
+                "name": "cmd.exe",
+                "executable": "C:\\Windows\\System32\\cmd.exe",
+                "pid": 2012
+              }
+            }
+          },
+          {
+            "_index": "my-index-000001",
+            "_id": "yDwnGIJouOYGBzP0ZE9n",
+            "_version" : 1,
+            "_seq_no" : 4,
+            "_primary_term" : 1,
+            "_score": null,
+            "_source": {
+              "@timestamp": "2020-12-07T11:07:09.000Z",
+              "event": {
+                "category": "process",
+                "id": "aR3NWVOs",
+                "sequence": 5
+              },
+              "process": {
+                "name": "regsvr32.exe",
+                "executable": "C:\\Windows\\System32\\regsvr32.exe",
+                "pid": 2012
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took": 60/"took": $body.took/]
+// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
+// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
+
+You can use the <<eql-with-maxspan-keywords,`with maxspan` keywords>> to
+constrain a sequence to a specified timespan.
+
+The following EQL search request adds `with maxspan=1h` to the previous query.
+This ensures all events in a matching sequence occur within `1h` (one hour) of
+the first event's timestamp.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    sequence with maxspan=1h
+      [ file where file.name == "cmd.exe" ]
+      [ process where stringContains(process.name, "regsvr32") ]
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+You can further constrain matching event sequences using the
+<<eql-by-keyword,`by` keyword>>.
+
+The following EQL search request adds `by process.pid` to each event item. This
+ensures events matching the sequence share the same `process.pid` field value.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    sequence with maxspan=1h
+      [ file where file.name == "cmd.exe" ] by process.pid
+      [ process where stringContains(process.name, "regsvr32") ] by process.pid
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+Because the `process.pid` field is shared across all events in the sequence, it
+can be included using `sequence by`. The following query is equivalent to the
+previous one.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    sequence by process.pid with maxspan=1h
+      [ file where file.name == "cmd.exe" ]
+      [ process where stringContains(process.name, "regsvr32") ]
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+The API returns the following response. The `hits.sequences.join_keys` property
+contains the shared `process.pid` value for each matching event.
+
+[source,console-result]
+----
+{
+  "is_partial": false,
+  "is_running": false,
+  "took": 60,
+  "timed_out": false,
+  "hits": {
+    "total": {
+      "value": 1,
+      "relation": "eq"
+    },
+    "sequences": [
+      {
+        "join_keys": [
+          "2012"
+        ],
+        "events": [
+          {
+            "_index": "my-index-000001",
+            "_id": "AtOJ4UjUBAAx3XR5kcCM",
+            "_version": 1,
+            "_seq_no": 3,
+            "_primary_term": 1,
+            "_score": null,
+            "_source": {
+              "@timestamp": "2020-12-07T11:07:08.000Z",
+              "event": {
+                "category": "file",
+                "id": "bYA7gPay",
+                "sequence": 4
+              },
+              "file": {
+                "accessed": "2020-12-07T11:07:08.000Z",
+                "name": "cmd.exe",
+                "path": "C:\\Windows\\System32\\cmd.exe",
+                "type": "file",
+                "size": 16384
+              },
+              "process": {
+                "name": "cmd.exe",
+                "executable": "C:\\Windows\\System32\\cmd.exe",
+                "pid": 2012
+              }
+            }
+          },
+          {
+            "_index": "my-index-000001",
+            "_id": "yDwnGIJouOYGBzP0ZE9n",
+            "_version": 1,
+            "_seq_no": 4,
+            "_primary_term": 1,
+            "_score": null,
+            "_source": {
+              "@timestamp": "2020-12-07T11:07:09.000Z",
+              "event": {
+                "category": "process",
+                "id": "aR3NWVOs",
+                "sequence": 5
+              },
+              "process": {
+                "name": "regsvr32.exe",
+                "executable": "C:\\Windows\\System32\\regsvr32.exe",
+                "pid": 2012
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took": 60/"took": $body.took/]
+// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
+// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
+
+You can use the <<eql-until-keyword,`until` keyword>> to specify an expiration
+event for sequences. Matching sequences must end before this event.
+
+The following request adds `until [ process where event.type == "termination" ]`
+to the previous query. This ensures matching sequences end before a `process`
+event with an `event.type` of `termination`.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "query": """
+    sequence by process.pid with maxspan=1h
+      [ file where file.name == "cmd.exe" ]
+      [ process where stringContains(process.name, "regsvr32") ]
+    until [ process where event.type == "termination" ]
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+[discrete]
+[[specify-a-timestamp-or-event-category-field]]
+=== Specify a timestamp or event category field
+
+By default, the EQL search API uses `@timestamp` and `event.category` as the
+required timestamp and event category fields. If your searched documents use
+a different timestamp or event category field, you must specify it in the search
+request using the `timestamp_field` or `event_category_field` parameters.
+
+The event category field is typically mapped as a <<keyword,`keyword`>> or
+<<constant-keyword,constant keyword>> field. The timestamp field is typically
+mapped as a <<date,`date`>> or <<date_nanos,`date_nanos`>> field.
+
+NOTE: You cannot use a <<nested,`nested`>> field or the sub-fields of a `nested`
+field as the timestamp or event category field. See <<eql-nested-fields>>.
+
+The following request uses the `timestamp_field` parameter to specify
+`file.accessed` as the timestamp field. The request also uses the
+`event_category_field` parameter to specify `file.type` as the event category
+field.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "timestamp_field": "file.accessed",
+  "event_category_field": "file.type",
+  "query": """
+    file where (file.size > 1 and file.type == "file")
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+[discrete]
+[[eql-search-filter-query-dsl]]
+=== Filter using query DSL
+
+You can use the `filter` parameter to specify an additional query using
+<<query-dsl,query DSL>>. This query filters the documents on which the EQL query
+runs.
+
+The following request uses a `range` query to filter `my-index-000001` to only
+documents with a `file.size` value greater than `1` but less than `1000000`
+bytes. The EQL query in `query` parameter then runs on these filtered documents.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "filter": {
+    "range" : {
+      "file.size" : {
+        "gte" : 1,
+        "lte" : 1000000
+      }
+    }
+  },
+  "query": """
+    file where (file.type == "file" and file.name == "cmd.exe")
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+[discrete]
+[[eql-search-case-sensitive]]
+=== Run a case-sensitive EQL search
+
+By default, matching for EQL queries is case-insensitive. You can use the
+`case_sensitive` parameter to toggle case sensitivity on or off.
+
+The following search request contains a query that matches `process` events
+with a `process.executable` containing `System32`.
+
+Because `case_sensitive` is `true`, this query only matches `process.executable`
+values containing `System32` with the exact same capitalization. A
+`process.executable` value containing `system32` or `SYSTEM32` would not match
+this query.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "keep_on_completion": true,
+  "case_sensitive": true,
+  "query": """
+    process where stringContains(process.executable, "System32")
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+[discrete]
+[[eql-search-async]]
+=== Run an async EQL search
+
+EQL searches are designed to run on large volumes of data quickly, often
+returning results in milliseconds. For this reason, EQL searches are
+_synchronous_ by default. The search request waits for complete results before
+returning a response. 
+
+However, complete results can take longer for searches across:
+
+* <<frozen-indices,Frozen indices>>
+* <<modules-cross-cluster-search,Multiple clusters>>
+* Many shards
+
+To avoid long waits, you can use the `wait_for_completion_timeout` parameter to
+run an _asynchronous_, or _async_, EQL search.
+
+Set `wait_for_completion_timeout` to a duration you'd like to wait
+for complete search results. If the search request does not finish within this
+period, the search becomes async and returns a response that includes:
+
+* A search ID, which can be used to monitor the progress of the async search.
+* An `is_partial` value of `true`, meaning the response does not contain
+  complete search results.
+* An `is_running` value of `true`, meaning the search is async and ongoing.
+
+The async search continues to run in the background without blocking
+other requests.
+
+The following request searches the `frozen-my-index-000001` index, which has been
+<<frozen-indices,frozen>> for storage and is rarely searched.
+
+Because searches on frozen indices are expected to take longer to complete, the
+request contains a `wait_for_completion_timeout` parameter value of `2s` (two
+seconds). If the request does not return complete results in two seconds, the
+search becomes async and returns a search ID.
+
+[source,console]
+----
+GET /frozen-my-index-000001/_eql/search
+{
+  "wait_for_completion_timeout": "2s",
+  "query": """
+    process where process.name == "cmd.exe"
+  """
+}
+----
+// TEST[setup:sec_logs]
+// TEST[s/frozen-my-index-000001/my-index-000001/]
+
+After two seconds, the request returns the following response. Note `is_partial`
+and `is_running` properties are `true`, indicating an async search.
+
+[source,console-result]
+----
+{
+  "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
+  "is_partial": true,
+  "is_running": true,
+  "took": 2000,
+  "timed_out": false,
+  "hits": ...
+}
+----
+// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
+// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/]
+// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
+// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+You can use the the search ID and the <<get-async-eql-search-api,get async EQL
+search API>> to check the progress of an async search.
+
+The get async EQL search API also accepts a `wait_for_completion_timeout`
+parameter. If ongoing search does not complete during this period, the response
+returns an `is_partial` value of `true` and no search results.
+
+The following get async EQL search API request checks the progress of the
+previous async EQL search. The request specifies a `wait_for_completion_timeout`
+query parameter value of `2s` (two seconds).
+
+[source,console]
+----
+GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s
+----
+// TEST[skip: no access to search ID]
+
+The request returns the following response. Note `is_partial` and `is_running`
+are `false`, indicating the async search has finished and the search results
+in the `hits` property are complete.
+
+[source,console-result]
+----
+{
+  "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
+  "is_partial": false,
+  "is_running": false,
+  "took": 2000,
+  "timed_out": false,
+  "hits": ...
+}
+----
+// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
+// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+[discrete]
+[[eql-search-store-async-eql-search]]
+=== Change the search retention period
+
+By default, the EQL search API stores async searches for five days. After this
+period, any searches and their results are deleted. You can use the `keep_alive`
+parameter to change this retention period.
+
+In the following EQL search request, the `keep_alive` parameter is `2d` (two
+days). If the search becomes async, its results
+are stored on the cluster for two days. After two days, the async
+search and its results are deleted, even if it's still ongoing.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "keep_alive": "2d",
+  "wait_for_completion_timeout": "2s",
+  "query": """
+    process where process.name == "cmd.exe"
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+You can use the <<get-async-eql-search-api,get async EQL search API>>'s
+`keep_alive`parameter to later change the retention period. The new
+retention period starts after the get request executes.
+
+The following request sets the `keep_alive` query parameter to `5d` (five days).
+The async search and its results are deleted five days after the get request
+executes.
+
+[source,console]
+----
+GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
+----
+// TEST[skip: no access to search ID]
+
+You can use the <<delete-async-eql-search-api,delete async EQL search API>> to
+manually delete an async EQL search before the `keep_alive` period ends. If the
+search is still ongoing, this cancels the search request.
+
+The following request deletes an async EQL search and its results.
+
+[source,console]
+----
+DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
+----
+// TEST[skip: no access to search ID]
+
+[discrete]
+[[eql-search-store-sync-eql-search]]
+=== Store synchronous EQL searches
+
+By default, the EQL search API only stores async searches that cannot be
+completed within the period set by `wait_for_completion_timeout`.
+
+To save the results of searches that complete during this period, set the
+`keep_on_completion` parameter to `true`.
+
+In the following search request, `keep_on_completion` is `true`. This means the
+search results are stored on the cluster, even if the search completes within
+the `2s` (two-second) period set by the `wait_for_completion_timeout` parameter.
+
+[source,console]
+----
+GET /my-index-000001/_eql/search
+{
+  "keep_on_completion": true,
+  "wait_for_completion_timeout": "2s",
+  "query": """
+    process where process.name == "cmd.exe"
+  """
+}
+----
+// TEST[setup:sec_logs]
+
+The API returns the following response. A search ID is provided in the `id`
+property. `is_partial` and `is_running` are `false`, indicating the EQL search
+was synchronous and returned complete results in `hits`.
+
+[source,console-result]
+----
+{
+  "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=",
+  "is_partial": false,
+  "is_running": false,
+  "took": 52,
+  "timed_out": false,
+  "hits": ...
+}
+----
+// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/]
+// TESTRESPONSE[s/"took": 52/"took": $body.took/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+You can use the search ID and the <<get-async-eql-search-api,get async EQL
+search API>> to retrieve the same results later.
+
+[source,console]
+----
+GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=
+----
+// TEST[skip: no access to search ID]
+
+Saved synchronous searches are still subject to the retention period set by the
+`keep_alive` parameter. After this period, the search and its results are
+deleted.
+
+You can also manually delete saved synchronous searches using the
+<<delete-async-eql-search-api,delete async EQL search API>>.
+
+include::syntax.asciidoc[]
+include::functions.asciidoc[]
+include::pipes.asciidoc[]

docs/reference/eql/get-async-eql-search-api.asciidoc

@@ -27,12 +27,12 @@ GET /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUTo
 [[get-async-eql-search-api-prereqs]]
 ==== {api-prereq-title}
 
-See <<eql-requirements,EQL requirements>>.
+See <<eql-required-fields>>.
 
 [[get-async-eql-search-api-limitations]]
 ===== Limitations
 
-See <<eql-limitations,EQL limitations>>.
+See <<eql-syntax-limitations,EQL limitations>>.
 
 [[get-async-eql-search-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/eql/index.asciidoc

@@ -1,61 +0,0 @@
-[role="xpack"]
-[testenv="basic"]
-[[eql]]
-= EQL for event-based search
-++++
-<titleabbrev>EQL</titleabbrev>
-++++
-
-experimental::[]
-
-{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for
-logs and other event-based data.
-
-You can use EQL in {es} to easily express relationships between events and
-quickly match events with shared properties. You can use EQL and query
-DSL together to better filter your searches.
-
-[discrete]
-[[eql-advantages]]
-=== Advantages of EQL
-
-* *EQL lets you express relationships between events.* +
-Many query languages allow you to match only single events. EQL lets you match a
-sequence of events across different event categories and time spans.
-
-* *EQL has a low learning curve.* +
-EQL syntax looks like other query languages. It lets you write and read queries
-intuitively, which makes for quick, iterative searching.
-
-* *We designed EQL for security use cases.* +
-While you can use EQL for any event-based data, we created EQL for threat
-hunting. EQL not only supports indicator of compromise (IOC) searching but
-makes it easy to describe activity that goes beyond IOCs.
-
-[discrete]
-[[when-to-use-eql]]
-=== When to use EQL
-
-Consider using EQL if you:
-
-* Use {es} for threat hunting or other security use cases
-* Search time-series data or logs, such as network or system logs
-* Want an easy way to explore relationships between events
-
-[discrete]
-[[eql-toc]]
-=== In this section
-
-* <<eql-requirements>>
-* <<eql-search>>
-* <<eql-syntax>>
-* <<eql-function-ref>>
-* <<eql-pipe-ref>>
-* <<eql-limitations>>
-
-include::requirements.asciidoc[]
-include::search.asciidoc[]
-include::syntax.asciidoc[]
-include::functions.asciidoc[]
-include::pipes.asciidoc[]
-include::limitations.asciidoc[]

docs/reference/eql/limitations.asciidoc

@@ -1,43 +0,0 @@
-[role="xpack"]
-[testenv="basic"]
-[[eql-limitations]]
-== EQL limitations
-++++
-<titleabbrev>Limitations</titleabbrev>
-++++
-
-experimental::[]
-
-[discrete]
-[[eql-nested-fields]]
-=== EQL search on nested fields is not supported
-
-You cannot use EQL to search the values of a <<nested,`nested`>> field or the
-sub-fields of a `nested` field. However, data streams and indices containing
-`nested` field mappings are otherwise supported.
-
-[discrete]
-[[eql-unsupported-syntax]]
-=== Unsupported syntax
-
-{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL
-queries that contain:
-
-* Array functions:
-** {eql-ref}/functions.html#arrayContains[`arrayContains`]
-** {eql-ref}/functions.html#arrayCount[`arrayCount`]
-** {eql-ref}/functions.html#arraySearch[`arraySearch`]
-
-* {eql-ref}/joins.html[Joins]
-
-* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]:
-** `child of`
-** `descendant of`
-** `event of`
-
-* The following {eql-ref}/pipes.html[pipes]:
-** {eql-ref}/pipes.html#count[`count`]
-** {eql-ref}/pipes.html#filter[`filter`]
-** {eql-ref}/pipes.html#sort[`sort`]
-** {eql-ref}/pipes.html#unique[`unique`]
-** {eql-ref}/pipes.html#unique-count[`unique_count`]

docs/reference/eql/requirements.asciidoc

@@ -1,43 +0,0 @@
-[role="xpack"]
-[testenv="basic"]
-[[eql-requirements]]
-== EQL requirements
-++++
-<titleabbrev>Requirements</titleabbrev>
-++++
-
-experimental::[]
-
-EQL is schema-less and works well with most common log formats.
-
-[TIP]
-====
-While no schema is required to use EQL in {es}, we recommend the
-{ecs-ref}[Elastic Common Schema (ECS)]. The <<eql-search-api,EQL search API>> is
-designed to work with core ECS fields by default.
-====
-
-[discrete]
-[[eql-required-fields]]
-=== Required fields
-
-In {es}, EQL assumes each document in a data stream or index corresponds to an
-event.
-
-To search a data stream or index using EQL, each document in the data stream or
-index must contain the following field archetypes:
-
-Event category::
-A field containing the event classification, such as `process`, `file`, or
-`network`. This is typically mapped as a <<keyword,`keyword`>> field.
-
-Timestamp::
-A field containing the date and/or time the event occurred. This is typically
-mapped as a <<date,`date`>> or <<date_nanos,`date_nanos`>> field.
-
-[NOTE]
-====
-You cannot use a <<nested,`nested`>> field data type or the sub-fields of a
-`nested` field as the timestamp or event category field. See
-<<eql-nested-fields>>.
-====

docs/reference/eql/search.asciidoc

@@ -1,827 +0,0 @@
-[role="xpack"]
-[testenv="basic"]
-[[eql-search]]
-== Run an EQL search
-
-experimental::[]
-
-To start using EQL in {es}, first ensure your event data meets
-<<eql-requirements,EQL requirements>>. You can then use the <<eql-search-api,EQL
-search API>> to search event data stored in one or more {es} data streams or
-indices. The API requires a query written in {es}'s supported <<eql-syntax,EQL
-syntax>>.
-
-To get started, ingest or add the data to an {es} data stream or index.
-
-The following <<docs-bulk,bulk API>> request adds some example log data to the
-`sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema
-(ECS)].
-
-[source,console]
-----
-PUT /sec_logs/_bulk?refresh
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-----
-// TESTSETUP
-
-[TIP]
-=====
-You also can set up {beats-ref}/getting-started.html[{beats}], such as
-{auditbeat-ref}/auditbeat-installation-configuration.html[{auditbeat}] or
-{winlogbeat-ref}/winlogbeat-installation-configuration.html[{winlogbeat}], to automatically
-send and index your event data in {es}. See
-{beats-ref}/getting-started.html[Getting started with {beats}].
-=====
-
-You can now use the EQL search API to search this index using an EQL query.
-
-The following request searches the `sec_logs` index using the EQL query
-specified in the `query` parameter. The EQL query matches events with an
-`event.category` of `process` that have a `process.name` of `cmd.exe`.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    process where process.name == "cmd.exe"
-  """
-}
-----
-// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
-
-Because the `sec_log` index follows the ECS, you don't need to specify the
-required <<eql-required-fields,event category or timestamp>> fields. The request
-uses the `event.category` and `@timestamp` fields by default.
-
-The API returns the following response containing the matching events. Events
-in the response are sorted by timestamp, converted to milliseconds since the
-https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
-
-[source,console-result]
-----
-{
-  "is_partial": false,
-  "is_running": false,
-  "took": 60,
-  "timed_out": false,
-  "hits": {
-    "total": {
-      "value": 2,
-      "relation": "eq"
-    },
-    "events": [
-       {
-        "_index": "sec_logs",
-        "_id": "OQmfCaduce8zoHT93o4H",
-        "_score": null,
-        "_source": {
-          "@timestamp": "2020-12-06T11:04:05.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
-          "event": {
-            "category": "process",
-            "id": "edwCRnyD",
-            "sequence": 1
-          },
-          "process": {
-            "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
-          }
-        }
-      },
-      {
-        "_index": "sec_logs",
-        "_id": "xLkCaj4EujzdNSxfYLbO",
-        "_score": null,
-        "_source": {
-          "@timestamp": "2020-12-07T11:06:07.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
-          "event": {
-            "category": "process",
-            "id": "cMyt5SZ2",
-            "sequence": 3
-          },
-          "process": {
-            "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
-          }
-        }
-      }
-    ]
-  }
-}
-----
-// TESTRESPONSE[s/"took": 60/"took": $body.took/]
-// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
-// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
-
-[discrete]
-[[eql-search-sequence]]
-=== Search for a sequence of events
-
-Many query languages allow you to match single events. However, EQL's
-<<eql-sequences,sequence syntax>> lets you match an ordered series of events.
-
-The following EQL search request matches a sequence that:
-
-. Starts with an event with:
-+
---
-* An `event.category` of `file`
-* A `file.name` of `cmd.exe`
---
-. Followed by an event with:
-+
---
-* An `event.category` of `process`
-* A `process.name` that contains the substring `regsvr32`
---
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    sequence
-      [ file where file.name == "cmd.exe" ]
-      [ process where stringContains(process.name, "regsvr32") ]
-  """
-}
-----
-
-The API returns the following response. Matching events in
-the `hits.sequences.events` property are sorted by
-<<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
-the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
-
-[source,console-result]
-----
-{
-  "is_partial": false,
-  "is_running": false,
-  "took": 60,
-  "timed_out": false,
-  "hits": {
-    "total": {
-      "value": 1,
-      "relation": "eq"
-    },
-    "sequences": [
-      {
-        "events": [
-          {
-            "_index": "sec_logs",
-            "_id": "AtOJ4UjUBAAx3XR5kcCM",
-            "_version" : 1,
-            "_seq_no" : 3,
-            "_primary_term" : 1,
-            "_score": null,
-            "_source": {
-              "@timestamp": "2020-12-07T11:07:08.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
-              "event": {
-                "category": "file",
-                "id": "bYA7gPay",
-                "sequence": 4
-              },
-              "file": {
-                "accessed": "2020-12-07T11:07:08.000Z",
-                "name": "cmd.exe",
-                "path": "C:\\Windows\\System32\\cmd.exe",
-                "type": "file",
-                "size": 16384
-              },
-              "process": {
-                "name": "cmd.exe",
-                "executable": "C:\\Windows\\System32\\cmd.exe"
-              }
-            }
-          },
-          {
-            "_index": "sec_logs",
-            "_id": "yDwnGIJouOYGBzP0ZE9n",
-            "_version" : 1,
-            "_seq_no" : 4,
-            "_primary_term" : 1,
-            "_score": null,
-            "_source": {
-              "@timestamp": "2020-12-07T11:07:09.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
-              "event": {
-                "category": "process",
-                "id": "aR3NWVOs",
-                "sequence": 5
-              },
-              "process": {
-                "name": "regsvr32.exe",
-                "executable": "C:\\Windows\\System32\\regsvr32.exe"
-              }
-            }
-          }
-        ]
-      }
-    ]
-  }
-}
-----
-// TESTRESPONSE[s/"took": 60/"took": $body.took/]
-// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
-// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
-
-You can use the <<eql-with-maxspan-keywords,`with maxspan` keywords>> to
-constrain a sequence to a specified timespan.
-
-The following EQL search request adds `with maxspan=1h` to the previous query.
-This ensures all events in a matching sequence occur within one hour (`1h`) of
-the first event's timestamp.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    sequence with maxspan=1h
-      [ file where file.name == "cmd.exe" ]
-      [ process where stringContains(process.name, "regsvr32") ]
-  """
-}
-----
-
-You can further constrain matching event sequences using the
-<<eql-by-keyword,`by` keyword>>.
-
-The following EQL search request adds `by agent.id` to each event item. This
-ensures events matching the sequence share the same `agent.id` field value.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    sequence with maxspan=1h
-      [ file where file.name == "cmd.exe" ] by agent.id
-      [ process where stringContains(process.name, "regsvr32") ] by agent.id
-  """
-}
-----
-
-Because the `agent.id` field is shared across all events in the sequence, it
-can be included using `sequence by`. The following query is equivalent to the
-prior one.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    sequence by agent.id with maxspan=1h
-      [ file where file.name == "cmd.exe" ]
-      [ process where stringContains(process.name, "regsvr32") ]
-  """
-}
-----
-
-The API returns the following response. The `hits.sequences.join_keys` property
-contains the shared `agent.id` value for each matching event.
-
-[source,console-result]
-----
-{
-  "is_partial": false,
-  "is_running": false,
-  "took": 60,
-  "timed_out": false,
-  "hits": {
-    "total": {
-      "value": 1,
-      "relation": "eq"
-    },
-    "sequences": [
-      {
-        "join_keys": [
-          "8a4f500d"
-        ],
-        "events": [
-          {
-            "_index": "sec_logs",
-            "_id": "AtOJ4UjUBAAx3XR5kcCM",
-            "_version": 1,
-            "_seq_no": 3,
-            "_primary_term": 1,
-            "_score": null,
-            "_source": {
-              "@timestamp": "2020-12-07T11:07:08.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
-              "event": {
-                "category": "file",
-                "id": "bYA7gPay",
-                "sequence": 4
-              },
-              "file": {
-                "accessed": "2020-12-07T11:07:08.000Z",
-                "name": "cmd.exe",
-                "path": "C:\\Windows\\System32\\cmd.exe",
-                "type": "file",
-                "size": 16384
-              },
-              "process": {
-                "name": "cmd.exe",
-                "executable": "C:\\Windows\\System32\\cmd.exe"
-              }
-            }
-          },
-          {
-            "_index": "sec_logs",
-            "_id": "yDwnGIJouOYGBzP0ZE9n",
-            "_version": 1,
-            "_seq_no": 4,
-            "_primary_term": 1,
-            "_score": null,
-            "_source": {
-              "@timestamp": "2020-12-07T11:07:09.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
-              "event": {
-                "category": "process",
-                "id": "aR3NWVOs",
-                "sequence": 5
-              },
-              "process": {
-                "name": "regsvr32.exe",
-                "executable": "C:\\Windows\\System32\\regsvr32.exe"
-              }
-            }
-          }
-        ]
-      }
-    ]
-  }
-}
-----
-// TESTRESPONSE[s/"took": 60/"took": $body.took/]
-// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
-// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
-
-You can use the <<eql-until-keyword,`until` keyword>> to specify an expiration
-event for sequences. Matching sequences must end before this event.
-
-The following request adds
-`until [ process where event.type == "termination" ]` to the previous EQL query.
-This ensures matching sequences end before a process termination event.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "query": """
-    sequence by agent.id with maxspan=1h
-      [ file where file.name == "cmd.exe" ]
-      [ process where stringContains(process.name, "regsvr32") ]
-    until [ process where event.type == "termination" ]
-  """
-}
-----
-
-[discrete]
-[[eql-search-specify-event-category-field]]
-=== Specify an event category field
-
-By default, the EQL search API uses `event.category` as the
-<<eql-required-fields,event category field>>. You can use the
-`event_category_field` parameter to specify another event category field.
-
-The following request specifies `file.type` as the event category
-field.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "event_category_field": "file.type",
-  "query": """
-    file where agent.id == "8a4f500d"
-  """
-}
-----
-
-[discrete]
-[[eql-search-specify-timestamp-field]]
-=== Specify a timestamp field
-
-By default, EQL searches use `@timestamp` as the <<eql-required-fields,event
-timestamp field>>. You can use the EQL search API's `timestamp_field` parameter
-to specify another timestamp field.
-
-The following request specifies `file.accessed` as the event
-timestamp field.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "timestamp_field": "file.accessed",
-  "query": """
-    file where (file.size > 1 and file.type == "file")
-  """
-}
-----
-
-[discrete]
-[[eql-search-specify-a-sort-tiebreaker]]
-=== Specify a sort tiebreaker
-
-By default, the EQL search API sorts matching events in the search response by
-timestamp. However, if two or more events share the same timestamp, a tiebreaker
-field is used to sort the events in ascending, lexicographic order.
-
-The EQL search API uses `event.sequence` as the default tiebreaker field. You
-can use the `tiebreaker_field` parameter to specify another field.
-
-The following request specifies `event.start` as the tiebreaker field.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "tiebreaker_field": "event.id",
-  "query": """
-    process where process.name == "cmd.exe" and stringContains(process.executable, "System32")
-  """
-}
-----
-// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
-
-The API returns the following response.
-
-[source,console-result]
-----
-{
-  "is_partial": false,
-  "is_running": false,
-  "took": 34,
-  "timed_out": false,
-  "hits": {
-    "total": {
-      "value": 2,
-      "relation": "eq"
-    },
-    "events": [
-      {
-        "_index": "sec_logs",
-        "_id": "OQmfCaduce8zoHT93o4H",
-        "_score": null,
-        "_source": {
-          "@timestamp": "2020-12-06T11:04:05.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
-          "event": {
-            "category": "process",
-            "id": "edwCRnyD",
-            "sequence": 1
-          },
-          "process": {
-            "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
-          }
-        }
-      },
-      {
-        "_index": "sec_logs",
-        "_id": "xLkCaj4EujzdNSxfYLbO",
-        "_score": null,
-        "_source": {
-          "@timestamp": "2020-12-07T11:06:07.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
-          "event": {
-            "category": "process",
-            "id": "cMyt5SZ2",
-            "sequence": 3
-          },
-          "process": {
-            "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
-          }
-        }
-      }
-    ]
-  }
-}
-----
-// TESTRESPONSE[s/"took": 34/"took": $body.took/]
-// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
-// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
-
-
-[discrete]
-[[eql-search-filter-query-dsl]]
-=== Filter using query DSL
-
-You can use the EQL search API's `filter` parameter to specify an additional
-query using <<query-dsl,query DSL>>. This query filters the documents on which
-the EQL query runs.
-
-The following request uses a `range` query to filter the `sec_logs`
-index down to only documents with a `file.size` value greater than `1` but less
-than `1000000` bytes. The EQL query in `query` parameter then runs on these
-filtered documents.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "filter": {
-    "range" : {
-      "file.size" : {
-        "gte" : 1,
-        "lte" : 1000000
-      }
-    }
-  },
-  "query": """
-    file where (file.type == "file" and file.name == "cmd.exe")
-  """
-}
-----
-
-[discrete]
-[[eql-search-async]]
-=== Run an async EQL search
-
-EQL searches in {es} are designed to run on large volumes of data quickly,
-often returning results in milliseconds. Because of this, the EQL search API
-runs _synchronous_ searches by default. This means the search request waits for
-complete results before returning a response.
-
-However, complete results can take longer for searches across:
-
-* <<frozen-indices,Frozen indices>>
-* <<modules-cross-cluster-search,Multiple clusters>>
-* Many shards
-
-To avoid long waits, you can use the EQL search API's
-`wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_,
-search.
-
-Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait
-for complete search results. If the search request does not finish within this
-period, the search becomes an async search. The EQL search
-API returns a response that includes:
-
-* A search ID, which can be used to monitor the progress of the async search and
-  retrieve complete results when it finishes.
-* An `is_partial` value of `true`, indicating the response does not contain
-  complete search results.
-* An `is_running` value of `true`, indicating the search is async and ongoing.
-
-The async search continues to run in the background without blocking
-other requests.
-
-The following request searches the `frozen_sec_logs` index, which has been
-<<frozen-indices,frozen>> for storage and is rarely searched.
-
-Because searches on frozen indices are expected to take longer to complete, the
-request contains a `wait_for_completion_timeout` parameter value of `2s`
-(two seconds).
-
-If the request does not return complete results in two seconds, the search
-becomes an async search and a search ID is returned.
-
-[source,console]
-----
-GET /frozen_sec_logs/_eql/search
-{
-  "wait_for_completion_timeout": "2s",
-  "query": """
-    process where process.name == "cmd.exe"
-  """
-}
-----
-// TEST[s/frozen_sec_logs/sec_logs/]
-
-After two seconds, the request returns the following response. Note the
-`is_partial` and `is_running` properties are `true`, indicating an ongoing async
-search.
-
-[source,console-result]
-----
-{
-  "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
-  "is_partial": true,
-  "is_running": true,
-  "took": 2000,
-  "timed_out": false,
-  "hits": ...
-}
-----
-// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
-// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/]
-// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
-// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
-// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
-
-You can use the the returned search ID and the <<get-async-eql-search-api,get
-async EQL search API>> to check the progress of an ongoing async search.
-
-The get async EQL search API also accepts a `wait_for_completion_timeout` query
-parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd
-like to wait for complete search results. If the request does not complete
-during this period, the response returns an `is_partial` value of `true` and no
-search results.
-
-The following get async EQL search API request checks the progress of the
-previous async EQL search. The request specifies a `wait_for_completion_timeout`
-query parameter value of `2s` (two seconds).
-
-[source,console]
-----
-GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s
-----
-// TEST[skip: no access to search ID]
-
-The request returns the following response. Note the `is_partial` and
-`is_running` properties are `false`, indicating the async EQL search has
-finished and the search results in the `hits` property are complete.
-
-[source,console-result]
-----
-{
-  "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
-  "is_partial": false,
-  "is_running": false,
-  "took": 2000,
-  "timed_out": false,
-  "hits": ...
-}
-----
-// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
-// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
-// TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/]
-// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
-
-[discrete]
-[[eql-search-store-async-eql-search]]
-=== Change the search retention period
-
-By default, the EQL search API only stores async searches and their results for
-five days. After this period, any ongoing searches or saved results are deleted.
-
-You can use the EQL search API's `keep_alive` parameter to change the duration
-of this period.
-
-In the following EQL search API request, the `keep_alive` parameter is `2d` (two
-days). This means that if the search becomes async, its results
-are stored on the cluster for two days. After two days, the async
-search and its results are deleted, even if it's still ongoing.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "keep_alive": "2d",
-  "wait_for_completion_timeout": "2s",
-  "query": """
-    process where process.name == "cmd.exe"
-  """
-}
-----
-
-You can use the <<get-async-eql-search-api,get async EQL search API>>'s
-`keep_alive` query parameter to later change the retention period. The new
-retention period starts after the get async EQL search API request executes.
-
-The following get async EQL search API request sets the `keep_alive` query
-parameter to `5d` (five days). The async search and its results are deleted five
-days after the get async EQL search API request executes.
-
-[source,console]
-----
-GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
-----
-// TEST[skip: no access to search ID]
-
-You can use the <<delete-async-eql-search-api,delete async EQL search API>> to
-manually delete an async EQL search before the `keep_alive` period ends. If the
-search is still ongoing, this cancels the search request.
-
-The following delete async EQL search API request deletes an async EQL search
-and its results.
-
-[source,console]
-----
-DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
-----
-// TEST[skip: no access to search ID]
-
-[discrete]
-[[eql-search-store-sync-eql-search]]
-=== Store synchronous EQL searches
-
-By default, the EQL search API only stores async searches that cannot be
-completed within the period set by the `wait_for_completion_timeout` parameter.
-
-To save the results of searches that complete during this period, set the
-`keep_on_completion` parameter to `true`.
-
-In the following EQL search API request, the `keep_on_completion` parameter is
-`true`. This means the search results are stored on the cluster, even if
-the search completes within the `2s` (two-second) period set by the
-`wait_for_completion_timeout` parameter.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "keep_on_completion": true,
-  "wait_for_completion_timeout": "2s",
-  "query": """
-    process where process.name == "cmd.exe"
-  """
-}
-----
-
-The API returns the following response. Note that a search ID is provided in the
-`id` property. The `is_partial` and `is_running` properties are `false`,
-indicating the EQL search was synchronous and returned complete search results.
-
-[source,console-result]
-----
-{
-  "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=",
-  "is_partial": false,
-  "is_running": false,
-  "took": 52,
-  "timed_out": false,
-  "hits": ...
-}
-----
-// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/]
-// TESTRESPONSE[s/"took": 52/"took": $body.took/]
-// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
-
-You can use the search ID and the <<get-async-eql-search-api,get async EQL
-search API>> to retrieve the same results later.
-
-[source,console]
-----
-GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=
-----
-// TEST[skip: no access to search ID]
-
-Saved synchronous searches are still subject to the storage retention period set
-by the `keep_alive` parameter. After this period, the search and its saved
-results are deleted.
-
-You can also manually delete saved synchronous searches using the
-<<delete-async-eql-search-api,delete async EQL search API>>.
-
-[discrete]
-[[eql-search-case-sensitive]]
-=== Run a case-sensitive EQL search
-
-By default, matching for EQL queries is case-insensitive. You can use the EQL
-search API's `case_sensitive` parameter to toggle case sensitivity on or off.
-
-The following search request contains a query that matches `process` events
-with a `process.executable` containing `System32`.
-
-Because the `case_sensitive` parameter is `true`, this query only matches
-`process.executable` values containing `System32` with the exact same capitalization.
-A `process.executable` value containing `system32` or `SYSTEM32` would not match this
-query.
-
-[source,console]
-----
-GET /sec_logs/_eql/search
-{
-  "keep_on_completion": true,
-  "case_sensitive": true,
-  "query": """
-    process where stringContains(process.executable, "System32")
-  """
-}
-----

docs/reference/eql/syntax.asciidoc

@@ -8,10 +8,7 @@
 
 experimental::[]
 
-[IMPORTANT]
-====
-{es} supports a subset of EQL syntax. See <<eql-limitations>>.
-====
+[IMPORTANT: {es} supports a subset of EQL syntax. See <<eql-syntax-limitations>.
 
 [discrete]
 [[eql-basic-syntax]]
@@ -683,3 +680,43 @@ You can pass the output of a pipe to another pipe. This lets you use multiple
 pipes with a single query.
 
 For a list of supported pipes, see <<eql-pipe-ref>>.
+
+[discrete]
+[[eql-syntax-limitations]]
+=== Limitations
+
+{es} EQL does not support the following features and syntax.
+
+[discrete]
+[[eql-nested-fields]]
+==== EQL search on nested fields
+
+You cannot use EQL to search the values of a <<nested,`nested`>> field or the
+sub-fields of a `nested` field. However, data streams and indices containing
+`nested` field mappings are otherwise supported.
+
+[discrete]
+[[eql-unsupported-syntax]]
+==== Unsupported syntax
+
+{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL
+queries that contain:
+
+* Array functions:
+** {eql-ref}/functions.html#arrayContains[`arrayContains`]
+** {eql-ref}/functions.html#arrayCount[`arrayCount`]
+** {eql-ref}/functions.html#arraySearch[`arraySearch`]
+
+* {eql-ref}/joins.html[Joins]
+
+* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]:
+** `child of`
+** `descendant of`
+** `event of`
+
+* The following {eql-ref}/pipes.html[pipes]:
+** {eql-ref}/pipes.html#count[`count`]
+** {eql-ref}/pipes.html#filter[`filter`]
+** {eql-ref}/pipes.html#sort[`sort`]
+** {eql-ref}/pipes.html#unique[`unique`]
+** {eql-ref}/pipes.html#unique-count[`unique_count`]

docs/reference/index.asciidoc

@@ -28,7 +28,7 @@ include::search/search-your-data.asciidoc[]
 
 include::query-dsl.asciidoc[]
 
-include::eql/index.asciidoc[]
+include::eql/eql.asciidoc[]
 
 include::sql/index.asciidoc[]
 

docs/reference/redirects.asciidoc

@@ -966,6 +966,21 @@ See <<run-an-es-search>>.
 
 See <<how-es-highlighters-work-internally>>.
 
+[role="exclude",id="eql-search"]
+=== Run an EQL search
+
+See <<run-an-eql-search>>.
+
+[role="exclude",id="eql-limitations"]
+=== EQL limitations
+
+See <<eql-syntax-limitations>>.
+
+[role="exclude",id="eql-requirements"]
+=== EQL requirements
+
+See <<eql-required-fields>>.
+
 ////
 [role="exclude",id="search-request-body"]
 === Request body search