search.asciidoc 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876
  1. [role="xpack"]
  2. [testenv="basic"]
  3. [[eql-search]]
  4. == Run an EQL search
  5. dev::[]
  6. To start using EQL in {es}, first ensure your event data meets
  7. <<eql-requirements,EQL requirements>>. You can then use the <<eql-search-api,EQL
  8. search API>> to search event data stored in one or more {es} indices.
  9. .*Example*
  10. [%collapsible]
  11. ====
  12. To get started, ingest or add the data to an {es} index.
  13. The following <<docs-bulk,bulk API>> request adds some example log data to the
  14. `sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema
  15. (ECS)].
  16. [source,console]
  17. ----
  18. PUT /sec_logs/_bulk?refresh
  19. {"index":{"_index" : "sec_logs", "_id" : "1"}}
  20. { "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD","sequence": 1 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } }
  21. {"index":{"_index" : "sec_logs", "_id" : "2"}}
  22. { "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } }
  23. {"index":{"_index" : "sec_logs", "_id" : "3"}}
  24. { "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } }
  25. {"index":{"_index" : "sec_logs", "_id" : "4"}}
  26. { "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } }
  27. {"index":{"_index" : "sec_logs", "_id" : "5"}}
  28. { "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } }
  29. ----
  30. // TESTSETUP
  31. [TIP]
  32. =====
  33. You also can set up {beats-ref}/getting-started.html[{beats}], such as
  34. {auditbeat-ref}/auditbeat-installation-configuration.html[{auditbeat}] or
  35. {winlogbeat-ref}/winlogbeat-installation-configuration.html[{winlogbeat}], to automatically
  36. send and index your event data in {es}. See
  37. {beats-ref}/getting-started.html[Getting started with {beats}].
  38. =====
  39. You can now use the EQL search API to search this index using an EQL query.
  40. The following request searches the `sec_logs` index using the EQL query
  41. specified in the `query` parameter. The EQL query matches events with an
  42. `event.category` of `process` that have a `process.name` of `cmd.exe`.
  43. [source,console]
  44. ----
  45. GET /sec_logs/_eql/search
  46. {
  47. "query": """
  48. process where process.name == "cmd.exe"
  49. """
  50. }
  51. ----
  52. // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
  53. Because the `sec_log` index follows the ECS, you don't need to specify the
  54. required <<eql-required-fields,event category or timestamp>> fields. The request
  55. uses the `event.category` and `@timestamp` fields by default.
  56. The API returns the following response containing the matching events. Events
  57. in the response are sorted by timestamp, converted to milliseconds since the
  58. https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
  59. [source,console-result]
  60. ----
  61. {
  62. "is_partial": false,
  63. "is_running": false,
  64. "took": 60,
  65. "timed_out": false,
  66. "hits": {
  67. "total": {
  68. "value": 2,
  69. "relation": "eq"
  70. },
  71. "events": [
  72. {
  73. "_index": "sec_logs",
  74. "_id": "1",
  75. "_score": null,
  76. "_source": {
  77. "@timestamp": "2020-12-06T11:04:05.000Z",
  78. "agent": {
  79. "id": "8a4f500d"
  80. },
  81. "event": {
  82. "category": "process",
  83. "id": "edwCRnyD",
  84. "sequence": 1
  85. },
  86. "process": {
  87. "name": "cmd.exe",
  88. "path": "C:\\Windows\\System32\\cmd.exe"
  89. }
  90. },
  91. "sort": [
  92. 1607252645000
  93. ]
  94. },
  95. {
  96. "_index": "sec_logs",
  97. "_id": "3",
  98. "_score": null,
  99. "_source": {
  100. "@timestamp": "2020-12-07T11:06:07.000Z",
  101. "agent": {
  102. "id": "8a4f500d"
  103. },
  104. "event": {
  105. "category": "process",
  106. "id": "cMyt5SZ2",
  107. "sequence": 3
  108. },
  109. "process": {
  110. "name": "cmd.exe",
  111. "path": "C:\\Windows\\System32\\cmd.exe"
  112. }
  113. },
  114. "sort": [
  115. 1607339167000
  116. ]
  117. }
  118. ]
  119. }
  120. }
  121. ----
  122. // TESTRESPONSE[s/"took": 60/"took": $body.took/]
  123. ====
  124. [discrete]
  125. [[eql-search-sequence]]
  126. === Search for a sequence of events
  127. Many query languages allow you to match single events. However, EQL's
  128. <<eql-sequences,sequence syntax>> lets you match an ordered series of events.
  129. .*Example*
  130. [%collapsible]
  131. ====
  132. The following EQL search request matches a sequence that:
  133. . Starts with an event with:
  134. +
  135. --
  136. * An `event.category` of `file`
  137. * A `file.name` of `cmd.exe`
  138. --
  139. . Followed by an event with:
  140. +
  141. --
  142. * An `event.category` of `process`
  143. * A `process.name` that contains the substring `regsvr32`
  144. --
  145. [source,console]
  146. ----
  147. GET /sec_logs/_eql/search
  148. {
  149. "query": """
  150. sequence
  151. [ file where file.name == "cmd.exe" ]
  152. [ process where stringContains(process.name, "regsvr32") ]
  153. """
  154. }
  155. ----
  156. // TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/]
  157. The API returns the following response. Matching events in
  158. the `hits.sequences.events` property are sorted by
  159. <<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
  160. the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
  161. [source,console-result]
  162. ----
  163. {
  164. "is_partial": false,
  165. "is_running": false,
  166. "took": 60,
  167. "timed_out": false,
  168. "hits": {
  169. "total": {
  170. "value": 1,
  171. "relation": "eq"
  172. },
  173. "sequences": [
  174. {
  175. "events": [
  176. {
  177. "_index": "sec_logs",
  178. "_id": "4",
  179. "_score": null,
  180. "_source": {
  181. "@timestamp": "2020-12-07T11:07:08.000Z",
  182. "agent": {
  183. "id": "8a4f500d"
  184. },
  185. "event": {
  186. "category": "file",
  187. "id": "bYA7gPay",
  188. "sequence": 4
  189. },
  190. "file": {
  191. "accessed": "2020-12-07T11:07:08.000Z",
  192. "name": "cmd.exe",
  193. "path": "C:\\Windows\\System32\\cmd.exe",
  194. "type": "file",
  195. "size": 16384
  196. },
  197. "process": {
  198. "name": "cmd.exe",
  199. "path": "C:\\Windows\\System32\\cmd.exe"
  200. }
  201. },
  202. "sort": [
  203. 1607339228000
  204. ]
  205. },
  206. {
  207. "_index": "sec_logs",
  208. "_id": "5",
  209. "_score": null,
  210. "_source": {
  211. "@timestamp": "2020-12-07T11:07:09.000Z",
  212. "agent": {
  213. "id": "8a4f500d"
  214. },
  215. "event": {
  216. "category": "process",
  217. "id": "aR3NWVOs",
  218. "sequence": 5
  219. },
  220. "process": {
  221. "name": "regsvr32.exe",
  222. "path": "C:\\Windows\\System32\\regsvr32.exe"
  223. }
  224. },
  225. "sort": [
  226. 1607339229000
  227. ]
  228. }
  229. ]
  230. }
  231. ]
  232. }
  233. }
  234. ----
  235. // TESTRESPONSE[s/"took": 60/"took": $body.took/]
  236. You can use the <<eql-with-maxspan-keywords,`with maxspan` keywords>> to
  237. constrain a sequence to a specified timespan.
  238. The following EQL search request adds `with maxspan=1h` to the previous query.
  239. This ensures all events in a matching sequence occur within one hour (`1h`) of
  240. the first event's timestamp.
  241. [source,console]
  242. ----
  243. GET /sec_logs/_eql/search
  244. {
  245. "query": """
  246. sequence with maxspan=1h
  247. [ file where file.name == "cmd.exe" ]
  248. [ process where stringContains(process.name, "regsvr32") ]
  249. """
  250. }
  251. ----
  252. // TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/]
  253. You can further constrain matching event sequences using the
  254. <<eql-by-keyword,`by` keyword>>.
  255. The following EQL search request adds `by agent.id` to each event item. This
  256. ensures events matching the sequence share the same `agent.id` field value.
  257. [source,console]
  258. ----
  259. GET /sec_logs/_eql/search
  260. {
  261. "query": """
  262. sequence with maxspan=1h
  263. [ file where file.name == "cmd.exe" ] by agent.id
  264. [ process where stringContains(process.name, "regsvr32") ] by agent.id
  265. """
  266. }
  267. ----
  268. Because the `agent.id` field is shared across all events in the sequence, it
  269. can be included using `sequence by`. The following query is equivalent to the
  270. prior one.
  271. [source,console]
  272. ----
  273. GET /sec_logs/_eql/search
  274. {
  275. "query": """
  276. sequence by agent.id with maxspan=1h
  277. [ file where file.name == "cmd.exe" ]
  278. [ process where stringContains(process.name, "regsvr32") ]
  279. """
  280. }
  281. ----
  282. // TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/]
  283. The API returns the following response. The `hits.sequences.join_keys` property
  284. contains the shared `agent.id` value for each matching event.
  285. [source,console-result]
  286. ----
  287. {
  288. "is_partial": false,
  289. "is_running": false,
  290. "took": 60,
  291. "timed_out": false,
  292. "hits": {
  293. "total": {
  294. "value": 1,
  295. "relation": "eq"
  296. },
  297. "sequences": [
  298. {
  299. "join_keys": [
  300. "8a4f500d"
  301. ],
  302. "events": [
  303. {
  304. "_index": "sec_logs",
  305. "_id": "4",
  306. "_score": null,
  307. "_source": {
  308. "@timestamp": "2020-12-07T11:07:08.000Z",
  309. "agent": {
  310. "id": "8a4f500d"
  311. },
  312. "event": {
  313. "category": "file",
  314. "id": "bYA7gPay",
  315. "sequence": 4
  316. },
  317. "file": {
  318. "accessed": "2020-12-07T11:07:08.000Z",
  319. "name": "cmd.exe",
  320. "path": "C:\\Windows\\System32\\cmd.exe",
  321. "type": "file",
  322. "size": 16384
  323. },
  324. "process": {
  325. "name": "cmd.exe",
  326. "path": "C:\\Windows\\System32\\cmd.exe"
  327. }
  328. },
  329. "sort": [
  330. 1607339228000
  331. ]
  332. },
  333. {
  334. "_index": "sec_logs",
  335. "_id": "5",
  336. "_score": null,
  337. "_source": {
  338. "@timestamp": "2020-12-07T11:07:09.000Z",
  339. "agent": {
  340. "id": "8a4f500d"
  341. },
  342. "event": {
  343. "category": "process",
  344. "id": "aR3NWVOs",
  345. "sequence": 5
  346. },
  347. "process": {
  348. "name": "regsvr32.exe",
  349. "path": "C:\\Windows\\System32\\regsvr32.exe"
  350. }
  351. },
  352. "sort": [
  353. 1607339229000
  354. ]
  355. }
  356. ]
  357. }
  358. ]
  359. }
  360. }
  361. ----
  362. // TESTRESPONSE[s/"took": 60/"took": $body.took/]
  363. ====
  364. [discrete]
  365. [[eql-search-specify-event-category-field]]
  366. === Specify an event category field
  367. The EQL search API uses `event.category` as the required
  368. <<eql-required-fields,event category field>> by default. You can use the
  369. `event_category_field` parameter to specify another event category field.
  370. .*Example*
  371. [%collapsible]
  372. ====
  373. The following request specifies `file.type` as the event category
  374. field.
  375. [source,console]
  376. ----
  377. GET /sec_logs/_eql/search
  378. {
  379. "event_category_field": "file.type",
  380. "query": """
  381. file where agent.id == "8a4f500d"
  382. """
  383. }
  384. ----
  385. ====
  386. [discrete]
  387. [[eql-search-specify-timestamp-field]]
  388. === Specify a timestamp field
  389. The EQL search API uses `@timestamp` as the required <<eql-required-fields,event
  390. timestamp field>> by default. You can use the `timestamp_field` parameter to
  391. specify another timestamp field.
  392. .*Example*
  393. [%collapsible]
  394. ====
  395. The following request specifies `file.accessed` as the event
  396. timestamp field.
  397. [source,console]
  398. ----
  399. GET /sec_logs/_eql/search
  400. {
  401. "timestamp_field": "file.accessed",
  402. "query": """
  403. file where (file.size > 1 and file.type == "file")
  404. """
  405. }
  406. ----
  407. ====
  408. [discrete]
  409. [[eql-search-specify-a-sort-tiebreaker]]
  410. === Specify a sort tiebreaker
  411. By default, the EQL search API sorts matching events in the search response by
  412. timestamp. However, if two or more events share the same timestamp, a tiebreaker
  413. field is used to sort the events in ascending, lexicographic order.
  414. The EQL search API uses `event.sequence` as the default tiebreaker field. You
  415. can use the `tiebreaker_field` parameter to specify another field.
  416. .*Example*
  417. [%collapsible]
  418. ====
  419. The following request specifies `event.start` as the tiebreaker field.
  420. [source,console]
  421. ----
  422. GET /sec_logs/_eql/search
  423. {
  424. "tiebreaker_field": "event.id",
  425. "query": """
  426. process where process.name == "cmd.exe" and stringContains(process.path, "System32")
  427. """
  428. }
  429. ----
  430. // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
  431. The API returns the following response. Note the `sort` property of each
  432. matching event contains an array of two items:
  433. * The first item is the event's <<eql-search-api-timestamp-field,timestamp>>,
  434. converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix
  435. epoch].
  436. * The second item is the event's `event.id` value. This value is used as a sort
  437. tiebreaker for events with the same timestamp.
  438. [source,console-result]
  439. ----
  440. {
  441. "is_partial": false,
  442. "is_running": false,
  443. "took": 34,
  444. "timed_out": false,
  445. "hits": {
  446. "total": {
  447. "value": 2,
  448. "relation": "eq"
  449. },
  450. "events": [
  451. {
  452. "_index": "sec_logs",
  453. "_id": "1",
  454. "_score": null,
  455. "_source": {
  456. "@timestamp": "2020-12-06T11:04:05.000Z",
  457. "agent": {
  458. "id": "8a4f500d"
  459. },
  460. "event": {
  461. "category": "process",
  462. "id": "edwCRnyD",
  463. "sequence": 1
  464. },
  465. "process": {
  466. "name": "cmd.exe",
  467. "path": "C:\\Windows\\System32\\cmd.exe"
  468. }
  469. },
  470. "sort": [
  471. 1607252645000, <1>
  472. "edwCRnyD" <2>
  473. ]
  474. },
  475. {
  476. "_index": "sec_logs",
  477. "_id": "3",
  478. "_score": null,
  479. "_source": {
  480. "@timestamp": "2020-12-07T11:06:07.000Z",
  481. "agent": {
  482. "id": "8a4f500d"
  483. },
  484. "event": {
  485. "category": "process",
  486. "id": "cMyt5SZ2",
  487. "sequence": 3
  488. },
  489. "process": {
  490. "name": "cmd.exe",
  491. "path": "C:\\Windows\\System32\\cmd.exe"
  492. }
  493. },
  494. "sort": [
  495. 1607339167000, <1>
  496. "cMyt5SZ2" <2>
  497. ]
  498. }
  499. ]
  500. }
  501. }
  502. ----
  503. // TESTRESPONSE[s/"took": 34/"took": $body.took/]
  504. <1> The event's <<eql-search-api-timestamp-field,timestamp>>, converted to
  505. milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix
  506. epoch]
  507. <2> The event's `event.id` value.
  508. ====
  509. [discrete]
  510. [[eql-search-filter-query-dsl]]
  511. === Filter using query DSL
  512. You can use the EQL search API's `filter` parameter to specify an additional
  513. query using <<query-dsl,query DSL>>. This query filters the documents on which
  514. the EQL query runs.
  515. .*Example*
  516. [%collapsible]
  517. ====
  518. The following request uses a `range` query to filter the `sec_logs`
  519. index down to only documents with a `file.size` value greater than `1` but less
  520. than `1000000` bytes. The EQL query in `query` parameter then runs on these
  521. filtered documents.
  522. [source,console]
  523. ----
  524. GET /sec_logs/_eql/search
  525. {
  526. "filter": {
  527. "range" : {
  528. "file.size" : {
  529. "gte" : 1,
  530. "lte" : 1000000
  531. }
  532. }
  533. },
  534. "query": """
  535. file where (file.type == "file" and file.name == "cmd.exe")
  536. """
  537. }
  538. ----
  539. ====
  540. [discrete]
  541. [[eql-search-async]]
  542. === Run an async EQL search
  543. EQL searches in {es} are designed to run on large volumes of data quickly,
  544. often returning results in milliseconds. Because of this, the EQL search API
  545. runs _synchronous_ searches by default. This means the search request waits for
  546. complete results before returning a response.
  547. However, complete results can take longer for searches across:
  548. * <<frozen-indices,Frozen indices>>
  549. * <<modules-cross-cluster-search,Multiple clusters>>
  550. * Many shards
  551. To avoid long waits, you can use the EQL search API's
  552. `wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_,
  553. search.
  554. Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait
  555. for complete search results. If the search request does not finish within this
  556. period, the search becomes an async search. The EQL search
  557. API returns a response that includes:
  558. * A search ID, which can be used to monitor the progress of the async search and
  559. retrieve complete results when it finishes.
  560. * An `is_partial` value of `true`, indicating the response does not contain
  561. complete search results.
  562. * An `is_running` value of `true`, indicating the search is async and ongoing.
  563. The async search continues to run in the background without blocking
  564. other requests.
  565. [%collapsible]
  566. .*Example*
  567. ====
  568. The following request searches the `frozen_sec_logs` index, which has been
  569. <<frozen-indices,frozen>> for storage and is rarely searched.
  570. Because searches on frozen indices are expected to take longer to complete, the
  571. request contains a `wait_for_completion_timeout` parameter value of `2s`
  572. (two seconds).
  573. If the request does not return complete results in two seconds, the search
  574. becomes an async search and a search ID is returned.
  575. [source,console]
  576. ----
  577. GET /frozen_sec_logs/_eql/search
  578. {
  579. "wait_for_completion_timeout": "2s",
  580. "query": """
  581. process where process.name == "cmd.exe"
  582. """
  583. }
  584. ----
  585. // TEST[s/frozen_sec_logs/sec_logs/]
  586. After two seconds, the request returns the following response. Note the
  587. `is_partial` and `is_running` properties are `true`, indicating an ongoing async
  588. search.
  589. [source,console-result]
  590. ----
  591. {
  592. "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
  593. "is_partial": true,
  594. "is_running": true,
  595. "took": 2000,
  596. "timed_out": false,
  597. "hits": ...
  598. }
  599. ----
  600. // TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
  601. // TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/]
  602. // TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
  603. // TESTRESPONSE[s/"took": 2000/"took": $body.took/]
  604. // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
  605. ====
  606. You can use the the returned search ID and the <<get-async-eql-search-api,get
  607. async EQL search API>> to check the progress of an ongoing async search.
  608. The get async EQL search API also accepts a `wait_for_completion_timeout` query
  609. parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd
  610. like to wait for complete search results. If the request does not complete
  611. during this period, the response returns an `is_partial` value of `true` and no
  612. search results.
  613. [%collapsible]
  614. .*Example*
  615. ====
  616. The following get async EQL search API request checks the progress of the
  617. previous async EQL search. The request specifies a `wait_for_completion_timeout`
  618. query parameter value of `2s` (two seconds).
  619. [source,console]
  620. ----
  621. GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s
  622. ----
  623. // TEST[skip: no access to search ID]
  624. The request returns the following response. Note the `is_partial` and
  625. `is_running` properties are `false`, indicating the async EQL search has
  626. finished and the search results in the `hits` property are complete.
  627. [source,console-result]
  628. ----
  629. {
  630. "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
  631. "is_partial": false,
  632. "is_running": false,
  633. "took": 2000,
  634. "timed_out": false,
  635. "hits": ...
  636. }
  637. ----
  638. // TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
  639. // TESTRESPONSE[s/"took": 2000/"took": $body.took/]
  640. // TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/]
  641. // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
  642. ====
  643. [discrete]
  644. [[eql-search-store-async-eql-search]]
  645. === Change the search retention period
  646. By default, the EQL search API only stores async searches and their results for
  647. five days. After this period, any ongoing searches or saved results are deleted.
  648. You can use the EQL search API's `keep_alive` parameter to change the duration
  649. of this period.
  650. .*Example*
  651. [%collapsible]
  652. ====
  653. In the following EQL search API request, the `keep_alive` parameter is `2d` (two
  654. days). This means that if the search becomes async, its results
  655. are stored on the cluster for two days. After two days, the async
  656. search and its results are deleted, even if it's still ongoing.
  657. [source,console]
  658. ----
  659. GET /sec_logs/_eql/search
  660. {
  661. "keep_alive": "2d",
  662. "wait_for_completion_timeout": "2s",
  663. "query": """
  664. process where process.name == "cmd.exe"
  665. """
  666. }
  667. ----
  668. ====
  669. You can use the <<get-async-eql-search-api,get async EQL search API>>'s
  670. `keep_alive` query parameter to later change the retention period. The new
  671. retention period starts after the get async EQL search API request executes.
  672. .*Example*
  673. [%collapsible]
  674. ====
  675. The following get async EQL search API request sets the `keep_alive` query
  676. parameter to `5d` (five days). The async search and its results are deleted five
  677. days after the get async EQL search API request executes.
  678. [source,console]
  679. ----
  680. GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
  681. ----
  682. // TEST[skip: no access to search ID]
  683. ====
  684. You can use the <<delete-async-eql-search-api,delete async EQL search API>> to
  685. manually delete an async EQL search before the `keep_alive` period ends. If the
  686. search is still ongoing, this cancels the search request.
  687. .*Example*
  688. [%collapsible]
  689. ====
  690. The following delete async EQL search API request deletes an async EQL search
  691. and its results.
  692. [source,console]
  693. ----
  694. DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
  695. ----
  696. // TEST[skip: no access to search ID]
  697. ====
  698. [discrete]
  699. [[eql-search-store-sync-eql-search]]
  700. === Store synchronous EQL searches
  701. By default, the EQL search API only stores async searches that cannot be
  702. completed within the period set by the `wait_for_completion_timeout` parameter.
  703. To save the results of searches that complete during this period, set the
  704. `keep_on_completion` parameter to `true`.
  705. [%collapsible]
  706. .*Example*
  707. ====
  708. In the following EQL search API request, the `keep_on_completion` parameter is
  709. `true`. This means the search results are stored on the cluster, even if
  710. the search completes within the `2s` (two-second) period set by the
  711. `wait_for_completion_timeout` parameter.
  712. [source,console]
  713. ----
  714. GET /sec_logs/_eql/search
  715. {
  716. "keep_on_completion": true,
  717. "wait_for_completion_timeout": "2s",
  718. "query": """
  719. process where process.name == "cmd.exe"
  720. """
  721. }
  722. ----
  723. The API returns the following response. Note that a search ID is provided in the
  724. `id` property. The `is_partial` and `is_running` properties are `false`,
  725. indicating the EQL search was synchronous and returned complete search results.
  726. [source,console-result]
  727. ----
  728. {
  729. "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=",
  730. "is_partial": false,
  731. "is_running": false,
  732. "took": 52,
  733. "timed_out": false,
  734. "hits": ...
  735. }
  736. ----
  737. // TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/]
  738. // TESTRESPONSE[s/"took": 52/"took": $body.took/]
  739. // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
  740. You can use the search ID and the <<get-async-eql-search-api,get async EQL
  741. search API>> to retrieve the same results later.
  742. [source,console]
  743. ----
  744. GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=
  745. ----
  746. // TEST[skip: no access to search ID]
  747. ====
  748. Saved synchronous searches are still subject to the storage retention period set
  749. by the `keep_alive` parameter. After this period, the search and its saved
  750. results are deleted.
  751. You can also manually delete saved synchronous searches using the
  752. <<delete-async-eql-search-api,delete async EQL search API>>.
  753. [discrete]
  754. [[eql-search-case-sensitive]]
  755. === Run a case-sensitive EQL search
  756. By default, matching for EQL queries is case-insensitive. You can use the EQL
  757. search API's `case_sensitive` parameter to toggle case sensitivity on or off.
  758. .*Example*
  759. [%collapsible]
  760. ====
  761. The following search request contains a query that matches `process` events
  762. with a `process.path` containing `System32`.
  763. Because the `case_sensitive` parameter is `true`, this query only matches
  764. `process.path` values containing `System32` with the exact same capitalization.
  765. A `process.path` value containing `system32` or `SYSTEM32` would not match this
  766. query.
  767. [source,console]
  768. ----
  769. GET /sec_logs/_eql/search
  770. {
  771. "keep_on_completion": true,
  772. "case_sensitive": true,
  773. "query": """
  774. process where stringContains(process.path, "System32")
  775. """
  776. }
  777. ----
  778. ====