terms-aggregation.asciidoc 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859
  1. [[search-aggregations-bucket-terms-aggregation]]
  2. === Terms aggregation
  3. ++++
  4. <titleabbrev>Terms</titleabbrev>
  5. ++++
  6. A multi-bucket value source based aggregation where buckets are dynamically built - one per unique value.
  7. //////////////////////////
  8. [source,console]
  9. --------------------------------------------------
  10. PUT /products
  11. {
  12. "mappings": {
  13. "properties": {
  14. "genre": {
  15. "type": "keyword"
  16. },
  17. "product": {
  18. "type": "keyword"
  19. }
  20. }
  21. }
  22. }
  23. POST /products/_bulk?refresh
  24. {"index":{"_id":0}}
  25. {"genre": "rock", "product": "Product A"}
  26. {"index":{"_id":1}}
  27. {"genre": "rock", "product": "Product B"}
  28. {"index":{"_id":2}}
  29. {"genre": "rock", "product": "Product C"}
  30. {"index":{"_id":3}}
  31. {"genre": "jazz", "product": "Product D"}
  32. {"index":{"_id":4}}
  33. {"genre": "jazz", "product": "Product E"}
  34. {"index":{"_id":5}}
  35. {"genre": "electronic", "product": "Anthology A"}
  36. {"index":{"_id":6}}
  37. {"genre": "electronic", "product": "Anthology A"}
  38. {"index":{"_id":7}}
  39. {"genre": "electronic", "product": "Product F"}
  40. {"index":{"_id":8}}
  41. {"genre": "electronic", "product": "Product G"}
  42. {"index":{"_id":9}}
  43. {"genre": "electronic", "product": "Product H"}
  44. {"index":{"_id":10}}
  45. {"genre": "electronic", "product": "Product I"}
  46. -------------------------------------------------
  47. // TESTSETUP
  48. //////////////////////////
  49. Example:
  50. [source,console,id=terms-aggregation-example]
  51. --------------------------------------------------
  52. GET /_search
  53. {
  54. "aggs": {
  55. "genres": {
  56. "terms": { "field": "genre" }
  57. }
  58. }
  59. }
  60. --------------------------------------------------
  61. // TEST[s/_search/_search\?filter_path=aggregations/]
  62. Response:
  63. [source,console-result]
  64. --------------------------------------------------
  65. {
  66. ...
  67. "aggregations": {
  68. "genres": {
  69. "doc_count_error_upper_bound": 0, <1>
  70. "sum_other_doc_count": 0, <2>
  71. "buckets": [ <3>
  72. {
  73. "key": "electronic",
  74. "doc_count": 6
  75. },
  76. {
  77. "key": "rock",
  78. "doc_count": 3
  79. },
  80. {
  81. "key": "jazz",
  82. "doc_count": 2
  83. }
  84. ]
  85. }
  86. }
  87. }
  88. --------------------------------------------------
  89. // TESTRESPONSE[s/\.\.\.//]
  90. <1> an upper bound of the error on the document counts for each term, see <<terms-agg-doc-count-error,below>>
  91. <2> when there are lots of unique terms, Elasticsearch only returns the top terms; this number is the sum of the document counts for all buckets that are not part of the response
  92. <3> the list of the top buckets, the meaning of `top` being defined by the <<search-aggregations-bucket-terms-aggregation-order,order>>
  93. [[search-aggregations-bucket-terms-aggregation-types]]
  94. The `field` can be <<keyword>>, <<number>>, <<ip, `ip`>>, <<boolean, `boolean`>>,
  95. or <<binary, `binary`>>.
  96. NOTE: By default, you cannot run a `terms` aggregation on a `text` field. Use a
  97. `keyword` <<multi-fields,sub-field>> instead. Alternatively, you can enable
  98. <<fielddata-mapping-param,`fielddata`>> on the `text` field to create buckets for the field's
  99. <<analysis,analyzed>> terms. Enabling `fielddata` can significantly increase
  100. memory usage.
  101. [[search-aggregations-bucket-terms-aggregation-size]]
  102. ==== Size
  103. By default, the `terms` aggregation returns the top ten terms with the most
  104. documents. Use the `size` parameter to return more terms, up to the
  105. <<search-settings-max-buckets,search.max_buckets>> limit.
  106. If your data contains 100 or 1000 unique terms, you can increase the `size` of
  107. the `terms` aggregation to return them all. If you have more unique terms and
  108. you need them all, use the
  109. <<search-aggregations-bucket-composite-aggregation,composite aggregation>>
  110. instead.
  111. Larger values of `size` use more memory to compute and, push the whole
  112. aggregation close to the `max_buckets` limit. You'll know you've gone too large
  113. if the request fails with a message about `max_buckets`.
  114. [[search-aggregations-bucket-terms-aggregation-shard-size]]
  115. ==== Shard size
  116. To get more accurate results, the `terms` agg fetches more than
  117. the top `size` terms from each shard. It fetches the top `shard_size` terms,
  118. which defaults to `size * 1.5 + 10`.
  119. This is to handle the case when one term has many documents on one shard but is
  120. just below the `size` threshold on all other shards. If each shard only
  121. returned `size` terms, the aggregation would return an partial doc count for
  122. the term. So `terms` returns more terms in an attempt to catch the missing
  123. terms. This helps, but it's still quite possible to return a partial doc
  124. count for a term. It just takes a term with more disparate per-shard doc counts.
  125. You can increase `shard_size` to better account for these disparate doc counts
  126. and improve the accuracy of the selection of top terms. It is much cheaper to increase
  127. the `shard_size` than to increase the `size`. However, it still takes more
  128. bytes over the wire and waiting in memory on the coordinating node.
  129. IMPORTANT: This guidance only applies if you're using the `terms` aggregation's
  130. default sort `order`. If you're sorting by anything other than document count in
  131. descending order, see <<search-aggregations-bucket-terms-aggregation-order>>.
  132. NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will
  133. override it and reset it to be equal to `size`.
  134. [[terms-agg-doc-count-error]]
  135. ==== Document count error
  136. Even with a larger `shard_size` value, `doc_count` values for a `terms`
  137. aggregation may be approximate. As a result, any sub-aggregations on the `terms`
  138. aggregation may also be approximate.
  139. `sum_other_doc_count` is the number of documents that didn't make it into the
  140. the top `size` terms. If this is greater than `0`, you can be sure that the
  141. `terms` agg had to throw away some buckets, either because they didn't fit into
  142. `size` on the coordinating node or they didn't fit into `shard_size` on the
  143. data node.
  144. ==== Per bucket document count error
  145. If you set the `show_term_doc_count_error` parameter to `true`, the `terms`
  146. aggregation will include `doc_count_error_upper_bound`, which is an upper bound
  147. to the error on the `doc_count` returned by each shard. It's the
  148. sum of the size of the largest bucket on each shard that didn't fit into
  149. `shard_size`.
  150. In more concrete terms, imagine there is one bucket that is very large on one
  151. shard and just outside the `shard_size` on all the other shards. In that case,
  152. the `terms` agg will return the bucket because it is large, but it'll be missing
  153. data from many documents on the shards where the term fell below the `shard_size` threshold.
  154. `doc_count_error_upper_bound` is the maximum number of those missing documents.
  155. [source,console,id=terms-aggregation-doc-count-error-example]
  156. --------------------------------------------------
  157. GET /_search
  158. {
  159. "aggs": {
  160. "products": {
  161. "terms": {
  162. "field": "product",
  163. "size": 5,
  164. "show_term_doc_count_error": true
  165. }
  166. }
  167. }
  168. }
  169. --------------------------------------------------
  170. // TEST[s/_search/_search\?filter_path=aggregations/]
  171. These errors can only be calculated in this way when the terms are ordered by descending document count. When the aggregation is
  172. ordered by the terms values themselves (either ascending or descending) there is no error in the document count since if a shard
  173. does not return a particular term which appears in the results from another shard, it must not have that term in its index. When the
  174. aggregation is either sorted by a sub aggregation or in order of ascending document count, the error in the document counts cannot be
  175. determined and is given a value of -1 to indicate this.
  176. [[search-aggregations-bucket-terms-aggregation-order]]
  177. ==== Order
  178. By default, the `terms` aggregation orders terms by descending document
  179. `_count`. This produces a bounded <<terms-agg-doc-count-error,document count>>
  180. error that {es} can report.
  181. You can use the `order` parameter to specify a different sort order, but we
  182. don't recommend it. It is extremely easy to create a terms ordering that will
  183. just return wrong results, and not obvious to see when you have done so.
  184. Change this only with caution.
  185. WARNING: Especially avoid using `"order": { "_count": "asc" }`. If you need to find rare
  186. terms, use the
  187. <<search-aggregations-bucket-rare-terms-aggregation,`rare_terms`>> aggregation
  188. instead. Due to the way the `terms` aggregation
  189. <<search-aggregations-bucket-terms-aggregation-shard-size,gets terms from
  190. shards>>, sorting by ascending doc count often produces inaccurate results.
  191. ===== Ordering by the term value
  192. In this case, the buckets are ordered by the actual term values, such as
  193. lexicographic order for keywords or numerically for numbers. This sorting is
  194. safe in both ascending and descending directions, and produces accurate
  195. results.
  196. Example of ordering the buckets alphabetically by their terms in an ascending manner:
  197. [source,console,id=terms-aggregation-asc-example]
  198. --------------------------------------------------
  199. GET /_search
  200. {
  201. "aggs": {
  202. "genres": {
  203. "terms": {
  204. "field": "genre",
  205. "order": { "_key": "asc" }
  206. }
  207. }
  208. }
  209. }
  210. --------------------------------------------------
  211. ===== Ordering by a sub aggregation
  212. WARNING: Sorting by a sub aggregation generally produces incorrect ordering, due to the way the `terms` aggregation
  213. <<search-aggregations-bucket-terms-aggregation-shard-size,gets results from
  214. shards>>.
  215. There are two cases when sub-aggregation ordering is safe and returns correct
  216. results: sorting by a maximum in descending order, or sorting by a minimum in
  217. ascending order. These approaches work because they align with the behavior of
  218. sub aggregations. That is, if you're looking for the largest maximum or the
  219. smallest minimum, the global answer (from combined shards) must be included in
  220. one of the local shard answers. Conversely, the smallest maximum and largest
  221. minimum wouldn't be accurately computed.
  222. Note also that in these cases, the ordering is correct but the doc counts and
  223. non-ordering sub aggregations may still have errors (and {es} does not calculate a
  224. bound for those errors).
  225. Ordering the buckets by single value metrics sub-aggregation (identified by the aggregation name):
  226. [source,console,id=terms-aggregation-subaggregation-example]
  227. --------------------------------------------------
  228. GET /_search
  229. {
  230. "aggs": {
  231. "genres": {
  232. "terms": {
  233. "field": "genre",
  234. "order": { "max_play_count": "desc" }
  235. },
  236. "aggs": {
  237. "max_play_count": { "max": { "field": "play_count" } }
  238. }
  239. }
  240. }
  241. }
  242. --------------------------------------------------
  243. Ordering the buckets by multi value metrics sub-aggregation (identified by the aggregation name):
  244. [source,console,id=terms-aggregation-multivalue-subaggregation-example]
  245. --------------------------------------------------
  246. GET /_search
  247. {
  248. "aggs": {
  249. "genres": {
  250. "terms": {
  251. "field": "genre",
  252. "order": { "playback_stats.max": "desc" }
  253. },
  254. "aggs": {
  255. "playback_stats": { "stats": { "field": "play_count" } }
  256. }
  257. }
  258. }
  259. }
  260. --------------------------------------------------
  261. [NOTE]
  262. .Pipeline aggs cannot be used for sorting
  263. =======================================
  264. <<search-aggregations-pipeline,Pipeline aggregations>> are run during the
  265. reduce phase after all other aggregations have already completed. For this
  266. reason, they cannot be used for ordering.
  267. =======================================
  268. It is also possible to order the buckets based on a "deeper" aggregation in the hierarchy. This is supported as long
  269. as the aggregations path are of a single-bucket type, where the last aggregation in the path may either be a single-bucket
  270. one or a metrics one. If it's a single-bucket type, the order will be defined by the number of docs in the bucket (i.e. `doc_count`),
  271. in case it's a metrics one, the same rules as above apply (where the path must indicate the metric name to sort by in case of
  272. a multi-value metrics aggregation, and in case of a single-value metrics aggregation the sort will be applied on that value).
  273. The path must be defined in the following form:
  274. // {wikipedia}/Extended_Backus%E2%80%93Naur_Form
  275. [source,ebnf]
  276. --------------------------------------------------
  277. AGG_SEPARATOR = '>' ;
  278. METRIC_SEPARATOR = '.' ;
  279. AGG_NAME = <the name of the aggregation> ;
  280. METRIC = <the name of the metric (in case of multi-value metrics aggregation)> ;
  281. PATH = <AGG_NAME> [ <AGG_SEPARATOR>, <AGG_NAME> ]* [ <METRIC_SEPARATOR>, <METRIC> ] ;
  282. --------------------------------------------------
  283. [source,console,id=terms-aggregation-hierarchy-example]
  284. --------------------------------------------------
  285. GET /_search
  286. {
  287. "aggs": {
  288. "countries": {
  289. "terms": {
  290. "field": "artist.country",
  291. "order": { "rock>playback_stats.avg": "desc" }
  292. },
  293. "aggs": {
  294. "rock": {
  295. "filter": { "term": { "genre": "rock" } },
  296. "aggs": {
  297. "playback_stats": { "stats": { "field": "play_count" } }
  298. }
  299. }
  300. }
  301. }
  302. }
  303. }
  304. --------------------------------------------------
  305. The above will sort the artist's countries buckets based on the average play count among the rock songs.
  306. Multiple criteria can be used to order the buckets by providing an array of order criteria such as the following:
  307. [source,console,id=terms-aggregation-multicriteria-example]
  308. --------------------------------------------------
  309. GET /_search
  310. {
  311. "aggs": {
  312. "countries": {
  313. "terms": {
  314. "field": "artist.country",
  315. "order": [ { "rock>playback_stats.avg": "desc" }, { "_count": "desc" } ]
  316. },
  317. "aggs": {
  318. "rock": {
  319. "filter": { "term": { "genre": "rock" } },
  320. "aggs": {
  321. "playback_stats": { "stats": { "field": "play_count" } }
  322. }
  323. }
  324. }
  325. }
  326. }
  327. }
  328. --------------------------------------------------
  329. The above will sort the artist's countries buckets based on the average play count among the rock songs and then by
  330. their `doc_count` in descending order.
  331. NOTE: In the event that two buckets share the same values for all order criteria the bucket's term value is used as a
  332. tie-breaker in ascending alphabetical order to prevent non-deterministic ordering of buckets.
  333. ===== Ordering by count ascending
  334. Ordering terms by ascending document `_count` produces an unbounded error that
  335. {es} can't accurately report. We therefore strongly recommend against using
  336. `"order": { "_count": "asc" }` as shown in the following example:
  337. [source,console,id=terms-aggregation-count-example]
  338. --------------------------------------------------
  339. GET /_search
  340. {
  341. "aggs": {
  342. "genres": {
  343. "terms": {
  344. "field": "genre",
  345. "order": { "_count": "asc" }
  346. }
  347. }
  348. }
  349. }
  350. --------------------------------------------------
  351. ==== Minimum document count
  352. It is possible to only return terms that match more than a configured number of hits using the `min_doc_count` option:
  353. [source,console,id=terms-aggregation-min-doc-count-example]
  354. --------------------------------------------------
  355. GET /_search
  356. {
  357. "aggs": {
  358. "tags": {
  359. "terms": {
  360. "field": "tags",
  361. "min_doc_count": 10
  362. }
  363. }
  364. }
  365. }
  366. --------------------------------------------------
  367. The above aggregation would only return tags which have been found in 10 hits or more. Default value is `1`.
  368. Terms are collected and ordered on a shard level and merged with the terms collected from other shards in a second step. However, the shard does not have the information about the global document count available. The decision if a term is added to a candidate list depends only on the order computed on the shard using local shard frequencies. The `min_doc_count` criterion is only applied after merging local terms statistics of all shards. In a way the decision to add the term as a candidate is made without being very _certain_ about if the term will actually reach the required `min_doc_count`. This might cause many (globally) high frequent terms to be missing in the final result if low frequent terms populated the candidate lists. To avoid this, the `shard_size` parameter can be increased to allow more candidate terms on the shards. However, this increases memory consumption and network traffic.
  369. [[search-aggregations-bucket-terms-shard-min-doc-count]]
  370. ===== `shard_min_doc_count`
  371. // tag::min-doc-count[]
  372. The parameter `shard_min_doc_count` regulates the _certainty_ a shard has if the term should actually be added to the candidate list or not with respect to the `min_doc_count`. Terms will only be considered if their local shard frequency within the set is higher than the `shard_min_doc_count`. If your dictionary contains many low frequent terms and you are not interested in those (for example misspellings), then you can set the `shard_min_doc_count` parameter to filter out candidate terms on a shard level that will with a reasonable certainty not reach the required `min_doc_count` even after merging the local counts. `shard_min_doc_count` is set to `0` per default and has no effect unless you explicitly set it.
  373. // end::min-doc-count[]
  374. NOTE: Setting `min_doc_count`=`0` will also return buckets for terms that didn't match any hit. However, some of
  375. the returned terms which have a document count of zero might only belong to deleted documents or documents
  376. from other types, so there is no warranty that a `match_all` query would find a positive document count for
  377. those terms.
  378. WARNING: When NOT sorting on `doc_count` descending, high values of `min_doc_count` may return a number of buckets
  379. which is less than `size` because not enough data was gathered from the shards. Missing buckets can be
  380. back by increasing `shard_size`.
  381. Setting `shard_min_doc_count` too high will cause terms to be filtered out on a shard level. This value should be set much lower than `min_doc_count/#shards`.
  382. [[search-aggregations-bucket-terms-aggregation-script]]
  383. ==== Script
  384. Use a <<runtime,runtime field>> if the data in your documents doesn't
  385. exactly match what you'd like to aggregate. If, for example, "anthologies"
  386. need to be in a special category then you could run this:
  387. [source,console,id=terms-aggregation-script-example]
  388. --------------------------------------------------
  389. GET /_search
  390. {
  391. "size": 0,
  392. "runtime_mappings": {
  393. "normalized_genre": {
  394. "type": "keyword",
  395. "script": """
  396. String genre = doc['genre'].value;
  397. if (doc['product'].value.startsWith('Anthology')) {
  398. emit(genre + ' anthology');
  399. } else {
  400. emit(genre);
  401. }
  402. """
  403. }
  404. },
  405. "aggs": {
  406. "genres": {
  407. "terms": {
  408. "field": "normalized_genre"
  409. }
  410. }
  411. }
  412. }
  413. --------------------------------------------------
  414. Which will look like:
  415. [source,console-result]
  416. --------------------------------------------------
  417. {
  418. "aggregations": {
  419. "genres": {
  420. "doc_count_error_upper_bound": 0,
  421. "sum_other_doc_count": 0,
  422. "buckets": [
  423. {
  424. "key": "electronic",
  425. "doc_count": 4
  426. },
  427. {
  428. "key": "rock",
  429. "doc_count": 3
  430. },
  431. {
  432. "key": "electronic anthology",
  433. "doc_count": 2
  434. },
  435. {
  436. "key": "jazz",
  437. "doc_count": 2
  438. }
  439. ]
  440. }
  441. },
  442. ...
  443. }
  444. --------------------------------------------------
  445. // TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
  446. This is a little slower because the runtime field has to access two fields
  447. instead of one and because there are some optimizations that work on
  448. non-runtime `keyword` fields that we have to give up for for runtime
  449. `keyword` fields. If you need the speed, you can index the
  450. `normalized_genre` field.
  451. // TODO when we have calculated fields we can link to them here.
  452. ==== Filtering Values
  453. It is possible to filter the values for which buckets will be created. This can be done using the `include` and
  454. `exclude` parameters which are based on regular expression strings or arrays of exact values. Additionally,
  455. `include` clauses can filter using `partition` expressions.
  456. ===== Filtering Values with regular expressions
  457. [source,console,id=terms-aggregation-regex-example]
  458. --------------------------------------------------
  459. GET /_search
  460. {
  461. "aggs": {
  462. "tags": {
  463. "terms": {
  464. "field": "tags",
  465. "include": ".*sport.*",
  466. "exclude": "water_.*"
  467. }
  468. }
  469. }
  470. }
  471. --------------------------------------------------
  472. In the above example, buckets will be created for all the tags that has the word `sport` in them, except those starting
  473. with `water_` (so the tag `water_sports` will not be aggregated). The `include` regular expression will determine what
  474. values are "allowed" to be aggregated, while the `exclude` determines the values that should not be aggregated. When
  475. both are defined, the `exclude` has precedence, meaning, the `include` is evaluated first and only then the `exclude`.
  476. The syntax is the same as <<regexp-syntax,regexp queries>>.
  477. ===== Filtering Values with exact values
  478. For matching based on exact values the `include` and `exclude` parameters can simply take an array of
  479. strings that represent the terms as they are found in the index:
  480. [source,console,id=terms-aggregation-exact-example]
  481. --------------------------------------------------
  482. GET /_search
  483. {
  484. "aggs": {
  485. "JapaneseCars": {
  486. "terms": {
  487. "field": "make",
  488. "include": [ "mazda", "honda" ]
  489. }
  490. },
  491. "ActiveCarManufacturers": {
  492. "terms": {
  493. "field": "make",
  494. "exclude": [ "rover", "jensen" ]
  495. }
  496. }
  497. }
  498. }
  499. --------------------------------------------------
  500. ===== Filtering Values with partitions
  501. Sometimes there are too many unique terms to process in a single request/response pair so
  502. it can be useful to break the analysis up into multiple requests.
  503. This can be achieved by grouping the field's values into a number of partitions at query-time and processing
  504. only one partition in each request.
  505. Consider this request which is looking for accounts that have not logged any access recently:
  506. [source,console,id=terms-aggregation-partitions-example]
  507. --------------------------------------------------
  508. GET /_search
  509. {
  510. "size": 0,
  511. "aggs": {
  512. "expired_sessions": {
  513. "terms": {
  514. "field": "account_id",
  515. "include": {
  516. "partition": 0,
  517. "num_partitions": 20
  518. },
  519. "size": 10000,
  520. "order": {
  521. "last_access": "asc"
  522. }
  523. },
  524. "aggs": {
  525. "last_access": {
  526. "max": {
  527. "field": "access_date"
  528. }
  529. }
  530. }
  531. }
  532. }
  533. }
  534. --------------------------------------------------
  535. This request is finding the last logged access date for a subset of customer accounts because we
  536. might want to expire some customer accounts who haven't been seen for a long while.
  537. The `num_partitions` setting has requested that the unique account_ids are organized evenly into twenty
  538. partitions (0 to 19). and the `partition` setting in this request filters to only consider account_ids falling
  539. into partition 0. Subsequent requests should ask for partitions 1 then 2 etc to complete the expired-account analysis.
  540. Note that the `size` setting for the number of results returned needs to be tuned with the `num_partitions`.
  541. For this particular account-expiration example the process for balancing values for `size` and `num_partitions` would be as follows:
  542. 1. Use the `cardinality` aggregation to estimate the total number of unique account_id values
  543. 2. Pick a value for `num_partitions` to break the number from 1) up into more manageable chunks
  544. 3. Pick a `size` value for the number of responses we want from each partition
  545. 4. Run a test request
  546. If we have a circuit-breaker error we are trying to do too much in one request and must increase `num_partitions`.
  547. If the request was successful but the last account ID in the date-sorted test response was still an account we might want to
  548. expire then we may be missing accounts of interest and have set our numbers too low. We must either
  549. * increase the `size` parameter to return more results per partition (could be heavy on memory) or
  550. * increase the `num_partitions` to consider less accounts per request (could increase overall processing time as we need to make more requests)
  551. Ultimately this is a balancing act between managing the Elasticsearch resources required to process a single request and the volume
  552. of requests that the client application must issue to complete a task.
  553. WARNING: Partitions cannot be used together with an `exclude` parameter.
  554. ==== Multi-field terms aggregation
  555. The `terms` aggregation does not support collecting terms from multiple fields
  556. in the same document. The reason is that the `terms` agg doesn't collect the
  557. string term values themselves, but rather uses
  558. <<search-aggregations-bucket-terms-aggregation-execution-hint,global ordinals>>
  559. to produce a list of all of the unique values in the field. Global ordinals
  560. results in an important performance boost which would not be possible across
  561. multiple fields.
  562. There are three approaches that you can use to perform a `terms` agg across
  563. multiple fields:
  564. <<search-aggregations-bucket-terms-aggregation-script,Script>>::
  565. Use a script to retrieve terms from multiple fields. This disables the global
  566. ordinals optimization and will be slower than collecting terms from a single
  567. field, but it gives you the flexibility to implement this option at search
  568. time.
  569. <<copy-to,`copy_to` field>>::
  570. If you know ahead of time that you want to collect the terms from two or more
  571. fields, then use `copy_to` in your mapping to create a new dedicated field at
  572. index time which contains the values from both fields. You can aggregate on
  573. this single field, which will benefit from the global ordinals optimization.
  574. <<search-aggregations-bucket-multi-terms-aggregation, `multi_terms` aggregation>>::
  575. Use multi_terms aggregation to combine terms from multiple fields into a compound key. This
  576. also disables the global ordinals and will be slower than collecting terms from a single field.
  577. It is faster but less flexible than using a script.
  578. [[search-aggregations-bucket-terms-aggregation-collect]]
  579. ==== Collect mode
  580. Deferring calculation of child aggregations
  581. For fields with many unique terms and a small number of required results it can be more efficient to delay the calculation
  582. of child aggregations until the top parent-level aggs have been pruned. Ordinarily, all branches of the aggregation tree
  583. are expanded in one depth-first pass and only then any pruning occurs.
  584. In some scenarios this can be very wasteful and can hit memory constraints.
  585. An example problem scenario is querying a movie database for the 10 most popular actors and their 5 most common co-stars:
  586. [source,console,id=terms-aggregation-collect-mode-example]
  587. --------------------------------------------------
  588. GET /_search
  589. {
  590. "aggs": {
  591. "actors": {
  592. "terms": {
  593. "field": "actors",
  594. "size": 10
  595. },
  596. "aggs": {
  597. "costars": {
  598. "terms": {
  599. "field": "actors",
  600. "size": 5
  601. }
  602. }
  603. }
  604. }
  605. }
  606. }
  607. --------------------------------------------------
  608. Even though the number of actors may be comparatively small and we want only 50 result buckets there is a combinatorial explosion of buckets
  609. during calculation - a single actor can produce n² buckets where n is the number of actors. The sane option would be to first determine
  610. the 10 most popular actors and only then examine the top co-stars for these 10 actors. This alternative strategy is what we call the `breadth_first` collection
  611. mode as opposed to the `depth_first` mode.
  612. NOTE: The `breadth_first` is the default mode for fields with a cardinality bigger than the requested size or when the cardinality is unknown (numeric fields or scripts for instance).
  613. It is possible to override the default heuristic and to provide a collect mode directly in the request:
  614. [source,console,id=terms-aggregation-breadth-first-example]
  615. --------------------------------------------------
  616. GET /_search
  617. {
  618. "aggs": {
  619. "actors": {
  620. "terms": {
  621. "field": "actors",
  622. "size": 10,
  623. "collect_mode": "breadth_first" <1>
  624. },
  625. "aggs": {
  626. "costars": {
  627. "terms": {
  628. "field": "actors",
  629. "size": 5
  630. }
  631. }
  632. }
  633. }
  634. }
  635. }
  636. --------------------------------------------------
  637. <1> the possible values are `breadth_first` and `depth_first`
  638. When using `breadth_first` mode the set of documents that fall into the uppermost buckets are
  639. cached for subsequent replay so there is a memory overhead in doing this which is linear with the number of matching documents.
  640. Note that the `order` parameter can still be used to refer to data from a child aggregation when using the `breadth_first` setting - the parent
  641. aggregation understands that this child aggregation will need to be called first before any of the other child aggregations.
  642. WARNING: Nested aggregations such as `top_hits` which require access to score information under an aggregation that uses the `breadth_first`
  643. collection mode need to replay the query on the second pass but only for the documents belonging to the top buckets.
  644. [[search-aggregations-bucket-terms-aggregation-execution-hint]]
  645. ==== Execution hint
  646. There are different mechanisms by which terms aggregations can be executed:
  647. - by using field values directly in order to aggregate data per-bucket (`map`)
  648. - by using global ordinals of the field and allocating one bucket per global ordinal (`global_ordinals`)
  649. Elasticsearch tries to have sensible defaults so this is something that generally doesn't need to be configured.
  650. `global_ordinals` is the default option for `keyword` field, it uses global ordinals to allocates buckets dynamically
  651. so memory usage is linear to the number of values of the documents that are part of the aggregation scope.
  652. `map` should only be considered when very few documents match a query. Otherwise the ordinals-based execution mode
  653. is significantly faster. By default, `map` is only used when running an aggregation on scripts, since they don't have
  654. ordinals.
  655. [source,console,id=terms-aggregation-execution-hint-example]
  656. --------------------------------------------------
  657. GET /_search
  658. {
  659. "aggs": {
  660. "tags": {
  661. "terms": {
  662. "field": "tags",
  663. "execution_hint": "map" <1>
  664. }
  665. }
  666. }
  667. }
  668. --------------------------------------------------
  669. <1> The possible values are `map`, `global_ordinals`
  670. Please note that Elasticsearch will ignore this execution hint if it is not applicable and that there is no backward compatibility guarantee on these hints.
  671. ==== Missing value
  672. The `missing` parameter defines how documents that are missing a value should be treated.
  673. By default they will be ignored but it is also possible to treat them as if they
  674. had a value.
  675. [source,console,id=terms-aggregation-missing-example]
  676. --------------------------------------------------
  677. GET /_search
  678. {
  679. "aggs": {
  680. "tags": {
  681. "terms": {
  682. "field": "tags",
  683. "missing": "N/A" <1>
  684. }
  685. }
  686. }
  687. }
  688. --------------------------------------------------
  689. <1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`.
  690. ==== Mixing field types
  691. WARNING: When aggregating on multiple indices the type of the aggregated field may not be the same in all indices.
  692. Some types are compatible with each other (`integer` and `long` or `float` and `double`) but when the types are a mix
  693. of decimal and non-decimal number the terms aggregation will promote the non-decimal numbers to decimal numbers.
  694. This can result in a loss of precision in the bucket values.
  695. [discrete]
  696. [[search-aggregations-bucket-terms-aggregation-troubleshooting]]
  697. ==== Troubleshooting
  698. ===== Failed Trying to Format Bytes
  699. When running a terms aggregation (or other aggregation, but in practice usually
  700. terms) over multiple indices, you may get an error that starts with "Failed
  701. trying to format bytes...". This is usually caused by two of the indices not
  702. having the same mapping type for the field being aggregated.
  703. **Use an explicit `value_type`**
  704. Although it's best to correct the mappings, you can work around this issue if
  705. the field is unmapped in one of the indices. Setting the `value_type` parameter
  706. can resolve the issue by coercing the unmapped field into the correct type.
  707. [source,console,id=terms-aggregation-value_type-example]
  708. ----
  709. GET /_search
  710. {
  711. "aggs": {
  712. "ip_addresses": {
  713. "terms": {
  714. "field": "destination_ip",
  715. "missing": "0.0.0.0",
  716. "value_type": "ip"
  717. }
  718. }
  719. }
  720. }
  721. ----