build.gradle 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983
  1. import org.elasticsearch.gradle.Version
  2. import org.elasticsearch.gradle.internal.info.BuildParams
  3. import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
  4. /*
  5. * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  6. * or more contributor license agreements. Licensed under the Elastic License
  7. * 2.0 and the Server Side Public License, v 1; you may not use this file except
  8. * in compliance with, at your election, the Elastic License 2.0 or the Server
  9. * Side Public License, v 1.
  10. */
  11. apply plugin: 'elasticsearch.docs-test'
  12. apply plugin: 'elasticsearch.rest-resources'
  13. ext.docsFileTree = fileTree(projectDir) {
  14. include '**/*.asciidoc'
  15. // That is where the snippets go, not where they come from!
  16. exclude 'build/**'
  17. exclude 'build-idea/**'
  18. exclude 'build-eclipse/**'
  19. // Just syntax examples
  20. exclude 'README.asciidoc'
  21. // Broken code snippet tests
  22. exclude 'reference/graph/explore.asciidoc'
  23. // These files simply don't pass yet. We should figure out how to fix them.
  24. exclude 'reference/watcher/reference/actions.asciidoc'
  25. exclude 'reference/rest-api/security/ssl.asciidoc'
  26. if (BuildParams.inFipsJvm) {
  27. // We don't support this component in FIPS 140
  28. exclude 'reference/ingest/processors/attachment.asciidoc'
  29. // We can't conditionally control output, this would be missing the ingest-attachment component
  30. exclude 'reference/cat/plugins.asciidoc'
  31. }
  32. }
  33. /* List of files that have snippets that will not work until platinum tests can occur ... */
  34. tasks.named("buildRestTests").configure {
  35. expectedUnconvertedCandidates = [
  36. 'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc',
  37. 'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc',
  38. 'reference/ml/anomaly-detection/apis/get-bucket.asciidoc',
  39. 'reference/ml/anomaly-detection/apis/get-category.asciidoc',
  40. 'reference/ml/anomaly-detection/apis/get-influencer.asciidoc',
  41. 'reference/ml/anomaly-detection/apis/get-job-stats.asciidoc',
  42. 'reference/ml/anomaly-detection/apis/get-job.asciidoc',
  43. 'reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc',
  44. 'reference/ml/anomaly-detection/apis/get-record.asciidoc',
  45. 'reference/ml/anomaly-detection/apis/get-snapshot.asciidoc',
  46. 'reference/ml/anomaly-detection/apis/post-data.asciidoc',
  47. 'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc',
  48. 'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc',
  49. 'reference/ml/anomaly-detection/apis/update-job.asciidoc',
  50. 'reference/security/authentication/user-cache.asciidoc',
  51. 'reference/security/authorization/run-as-privilege.asciidoc',
  52. 'reference/security/ccs-clients-integrations/http.asciidoc',
  53. 'reference/rest-api/watcher/put-watch.asciidoc',
  54. 'reference/rest-api/watcher/stats.asciidoc',
  55. 'reference/watcher/example-watches/watching-time-series-data.asciidoc'
  56. ]
  57. }
  58. restResources {
  59. restApi {
  60. include '*'
  61. }
  62. }
  63. testClusters.matching { it.name == "yamlRestTest"}.configureEach {
  64. if (singleNode().testDistribution == DEFAULT) {
  65. setting 'xpack.license.self_generated.type', 'trial'
  66. setting 'indices.lifecycle.history_index_enabled', 'false'
  67. keystorePassword 'keystore-password'
  68. if (BuildParams.isSnapshotBuild() == false) {
  69. requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0)
  70. }
  71. }
  72. // debug ccr test failures:
  73. // https://github.com/elastic/elasticsearch/issues/95678
  74. // https://github.com/elastic/elasticsearch/issues/94359
  75. // https://github.com/elastic/elasticsearch/issues/96561
  76. setting 'logger.org.elasticsearch.transport.SniffConnectionStrategy', 'DEBUG'
  77. setting 'logger.org.elasticsearch.transport.RemoteClusterService', 'DEBUG'
  78. // enable regexes in painless so our tests don't complain about example snippets that use them
  79. setting 'script.painless.regex.enabled', 'true'
  80. setting 'path.repo', "${buildDir}/cluster/shared/repo"
  81. Closure configFile = {
  82. extraConfigFile it, file("src/test/cluster/config/$it")
  83. }
  84. configFile 'analysis/example_word_list.txt'
  85. configFile 'analysis/hyphenation_patterns.xml'
  86. configFile 'analysis/synonym.txt'
  87. configFile 'analysis/stemmer_override.txt'
  88. configFile 'userdict_ja.txt'
  89. configFile 'userdict_ko.txt'
  90. configFile 'KeywordTokenizer.rbbi'
  91. extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff')
  92. extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic')
  93. extraConfigFile 'httpCa.p12', file("./httpCa.p12")
  94. extraConfigFile 'transport.p12', file("./transport.p12")
  95. // Whitelist reindexing from the local node so we can test it.
  96. setting 'reindex.remote.whitelist', '127.0.0.1:*'
  97. extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("src/yamlRestTest/resources/GeoLite2-City.mmdb")
  98. extraConfigFile 'ingest-geoip/GeoLite2-Country.mmdb', file("src/yamlRestTest/resources/GeoLite2-Country.mmdb")
  99. // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
  100. systemProperty 'es.transport.cname_in_publish_address', 'true'
  101. requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
  102. requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
  103. // TODO Rene: clean up this kind of cross project file references
  104. extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json")
  105. extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
  106. extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt")
  107. setting 'xpack.security.enabled', 'true'
  108. setting 'xpack.security.authc.api_key.enabled', 'true'
  109. setting 'xpack.security.authc.token.enabled', 'true'
  110. // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
  111. setting 'indices.lifecycle.history_index_enabled', 'false'
  112. setting 'xpack.license.self_generated.type', 'trial'
  113. setting 'xpack.security.authc.realms.file.file.order', '0'
  114. setting 'xpack.security.authc.realms.native.native.order', '1'
  115. setting 'xpack.security.authc.realms.oidc.oidc1.order', '2'
  116. setting 'xpack.security.authc.realms.oidc.oidc1.op.issuer', 'http://127.0.0.1:8080'
  117. setting 'xpack.security.authc.realms.oidc.oidc1.op.authorization_endpoint', "http://127.0.0.1:8080/c2id-login"
  118. setting 'xpack.security.authc.realms.oidc.oidc1.op.token_endpoint', "http://127.0.0.1:8080/c2id/token"
  119. setting 'xpack.security.authc.realms.oidc.oidc1.op.jwkset_path', 'op-jwks.json'
  120. setting 'xpack.security.authc.realms.oidc.oidc1.rp.redirect_uri', 'https://my.fantastic.rp/cb'
  121. setting 'xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'elasticsearch-rp'
  122. keystore 'xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
  123. setting 'xpack.security.authc.realms.oidc.oidc1.rp.response_type', 'id_token'
  124. setting 'xpack.security.authc.realms.oidc.oidc1.claims.principal', 'sub'
  125. setting 'xpack.security.authc.realms.pki.pki1.order', '3'
  126. setting 'xpack.security.authc.realms.pki.pki1.certificate_authorities', '[ "testClient.crt" ]'
  127. setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
  128. setting 'xpack.security.authc.realms.saml.saml1.order', '4'
  129. setting 'xpack.security.authc.realms.saml.saml1.sp.logout', 'https://kibana.org/logout'
  130. setting 'xpack.security.authc.realms.saml.saml1.idp.entity_id', 'https://my-idp.org'
  131. setting 'xpack.security.authc.realms.saml.saml1.idp.metadata.path', 'idp-docs-metadata.xml'
  132. setting 'xpack.security.authc.realms.saml.saml1.sp.entity_id', 'https://kibana.org'
  133. setting 'xpack.security.authc.realms.saml.saml1.sp.acs', 'https://kibana.org/api/security/saml/callback'
  134. setting 'xpack.security.authc.realms.saml.saml1.attributes.principal', 'uid'
  135. setting 'xpack.security.authc.realms.saml.saml1.attributes.name', 'urn:oid:2.5.4.3'
  136. user username: 'test_admin'
  137. user username: 'test_user'
  138. // build the cluster with all plugins
  139. project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
  140. /* Skip repositories. We just aren't going to be able to test them so it
  141. * doesn't make sense to waste time installing them.
  142. */
  143. if (subproj.path.startsWith(':plugins:repository-')) {
  144. return
  145. }
  146. // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported
  147. if (subproj.path.startsWith(':modules:ingest-attachment') && BuildParams.inFipsJvm) {
  148. return
  149. }
  150. plugin subproj.path
  151. }
  152. }
  153. tasks.named("yamlRestTest").configure {
  154. doFirst {
  155. delete("${buildDir}/cluster/shared/repo")
  156. }
  157. }
  158. tasks.named("forbiddenPatterns").configure {
  159. exclude '**/*.mmdb'
  160. }
  161. tasks.named("buildRestTests").configure {
  162. docs = docsFileTree
  163. }
  164. tasks.named("listSnippets").configure {
  165. docs = docsFileTree
  166. }
  167. tasks.named("listConsoleCandidates").configure {
  168. docs = docsFileTree
  169. }
  170. Closure setupMyIndex = { String name, int count ->
  171. tasks.named("buildRestTests").configure { buildRestTests ->
  172. buildRestTests.setups[name] = '''
  173. - do:
  174. indices.create:
  175. index: my-index-000001
  176. body:
  177. settings:
  178. number_of_shards: 1
  179. number_of_replicas: 1
  180. mappings:
  181. properties:
  182. "@timestamp":
  183. type: date
  184. http:
  185. properties:
  186. request:
  187. properties:
  188. method:
  189. type: keyword
  190. message:
  191. type: text
  192. fields:
  193. keyword:
  194. type: keyword
  195. user:
  196. properties:
  197. id:
  198. type: keyword
  199. doc_values: true
  200. - do:
  201. bulk:
  202. index: my-index-000001
  203. refresh: true
  204. body: |'''
  205. for (int i = 0; i < count; i++) {
  206. String ip, user_id
  207. if (i == 0) {
  208. ip = '127.0.0.1'
  209. user_id = 'kimchy'
  210. } else {
  211. ip = '10.42.42.42'
  212. user_id = 'elkbee'
  213. }
  214. buildRestTests.setups[name] += """
  215. { "index":{"_id": "$i"} }
  216. { "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "$ip" }, "user": { "id": "$user_id" } }"""
  217. }
  218. }
  219. }
  220. setupMyIndex('my_index', 5)
  221. setupMyIndex('my_index_big', 120)
  222. setupMyIndex('my_index_huge', 1200)
  223. tasks.named("buildRestTests").configure {buildRestTests ->
  224. setups['my_data_stream_template'] = '''
  225. - do:
  226. indices.put_index_template:
  227. name: my-data-stream-template
  228. body: |
  229. {
  230. "index_patterns": [ "my-data-stream*" ],
  231. "data_stream": { },
  232. "priority": 500
  233. }
  234. '''
  235. setups['my_data_stream'] = setups['my_data_stream_template'] + '''
  236. - do:
  237. raw:
  238. method: PUT
  239. path: '_data_stream/my-data-stream'
  240. '''
  241. teardowns['data_stream_cleanup'] = '''
  242. - do:
  243. raw:
  244. method: DELETE
  245. path: '_data_stream/*'
  246. - is_true: acknowledged
  247. - do:
  248. raw:
  249. method: DELETE
  250. path: '_index_template/*'
  251. - is_true: acknowledged
  252. '''
  253. // Used for TSDS examples
  254. setups['tsds_template'] = '''
  255. - do:
  256. indices.put_index_template:
  257. name: my-weather-sensor-index-template
  258. body: |
  259. {
  260. "index_patterns": [ "metrics-weather_sensors-*" ],
  261. "data_stream": { },
  262. "template": {
  263. "settings": {
  264. "index.mode": "time_series",
  265. "index.routing_path": [ "sensor_id", "location" ]
  266. },
  267. "mappings": {
  268. "properties": {
  269. "sensor_id": {
  270. "type": "keyword",
  271. "time_series_dimension": true
  272. },
  273. "location": {
  274. "type": "keyword",
  275. "time_series_dimension": true
  276. }
  277. }
  278. }
  279. },
  280. "priority": 500
  281. }
  282. '''
  283. setups['tsds'] = setups['tsds_template'] + '''
  284. - do:
  285. raw:
  286. method: PUT
  287. path: '_data_stream/metrics-weather_sensors-dev'
  288. '''
  289. teardowns['tsds_cleanup'] = '''
  290. - do:
  291. raw:
  292. method: DELETE
  293. path: '_data_stream/*'
  294. - is_true: acknowledged
  295. - do:
  296. raw:
  297. method: DELETE
  298. path: '_index_template/*'
  299. - is_true: acknowledged
  300. '''
  301. // Used for several full-text search and agg examples
  302. buildRestTests.setups['messages'] = '''
  303. - do:
  304. indices.create:
  305. index: my-index-000001
  306. body:
  307. settings:
  308. number_of_shards: 1
  309. number_of_replicas: 1
  310. - do:
  311. bulk:
  312. index: my-index-000001
  313. refresh: true
  314. body: |
  315. {"index":{"_id": "0"}}
  316. {"message": "trying out Elasticsearch", "context": "foo"}
  317. {"index":{"_id": "1"}}
  318. {"message": "some message with the number 1", "context": "bar"}
  319. {"index":{"_id": "2"}}
  320. {"message": "some message with the number 2", "context": "bar"}
  321. {"index":{"_id": "3"}}
  322. {"message": "some message with the number 3", "context": "bar"}
  323. {"index":{"_id": "4"}}
  324. {"message": "some message with the number 4", "context": "bar"}
  325. '''
  326. // Used for EQL
  327. setups['sec_logs'] = setups['my_data_stream'] + '''
  328. - do:
  329. bulk:
  330. index: my-data-stream
  331. refresh: true
  332. body: |
  333. {"create":{}}
  334. {"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
  335. {"create":{}}
  336. {"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
  337. {"create":{}}
  338. {"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
  339. {"create":{}}
  340. {"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}
  341. {"create":{}}
  342. {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }}
  343. {"create":{}}
  344. {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}'''
  345. buildRestTests.setups['host'] = '''
  346. # Fetch the http host. We use the host of the master because we know there will always be a master.
  347. - do:
  348. cluster.state: {}
  349. - set: { master_node: master }
  350. - do:
  351. nodes.info:
  352. metric: [ http, transport ]
  353. - set: {nodes.$master.http.publish_address: host}
  354. - set: {nodes.$master.transport.publish_address: transport_host}
  355. '''
  356. buildRestTests.setups['node'] = '''
  357. # Fetch the node name. We use the host of the master because we know there will always be a master.
  358. - do:
  359. cluster.state: {}
  360. - is_true: master_node
  361. - set: { master_node: node_name }
  362. '''
  363. // Used by scripted metric docs
  364. buildRestTests.setups['ledger'] = '''
  365. - do:
  366. indices.create:
  367. index: ledger
  368. body:
  369. settings:
  370. number_of_shards: 2
  371. number_of_replicas: 1
  372. mappings:
  373. properties:
  374. type:
  375. type: keyword
  376. amount:
  377. type: double
  378. - do:
  379. bulk:
  380. index: ledger
  381. refresh: true
  382. body: |
  383. {"index":{}}
  384. {"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"}
  385. {"index":{}}
  386. {"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "description": "another thing"}
  387. {"index":{}}
  388. {"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"}
  389. {"index":{}}
  390. {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"}
  391. {"index":{}}
  392. {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}'''
  393. // Used by aggregation docs
  394. buildRestTests.setups['sales'] = '''
  395. - do:
  396. indices.create:
  397. index: sales
  398. body:
  399. settings:
  400. number_of_shards: 2
  401. number_of_replicas: 1
  402. mappings:
  403. properties:
  404. type:
  405. type: keyword
  406. - do:
  407. bulk:
  408. index: sales
  409. refresh: true
  410. body: |
  411. {"index":{}}
  412. {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
  413. {"index":{}}
  414. {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"}
  415. {"index":{}}
  416. {"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"}
  417. {"index":{}}
  418. {"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"}
  419. {"index":{}}
  420. {"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"}
  421. {"index":{}}
  422. {"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
  423. {"index":{}}
  424. {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}'''
  425. // Used by cumulative cardinality aggregation docs
  426. buildRestTests.setups['user_hits'] = '''
  427. - do:
  428. indices.create:
  429. index: user_hits
  430. body:
  431. settings:
  432. number_of_shards: 1
  433. number_of_replicas: 0
  434. mappings:
  435. properties:
  436. user_id:
  437. type: keyword
  438. timestamp:
  439. type: date
  440. - do:
  441. bulk:
  442. index: user_hits
  443. refresh: true
  444. body: |
  445. {"index":{}}
  446. {"timestamp": "2019-01-01T13:00:00", "user_id": "1"}
  447. {"index":{}}
  448. {"timestamp": "2019-01-01T13:00:00", "user_id": "2"}
  449. {"index":{}}
  450. {"timestamp": "2019-01-02T13:00:00", "user_id": "1"}
  451. {"index":{}}
  452. {"timestamp": "2019-01-02T13:00:00", "user_id": "3"}
  453. {"index":{}}
  454. {"timestamp": "2019-01-03T13:00:00", "user_id": "1"}
  455. {"index":{}}
  456. {"timestamp": "2019-01-03T13:00:00", "user_id": "2"}
  457. {"index":{}}
  458. {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}'''
  459. // Used by sampler and diversified-sampler aggregation docs
  460. buildRestTests.setups['stackoverflow'] = '''
  461. - do:
  462. indices.create:
  463. index: stackoverflow
  464. body:
  465. settings:
  466. number_of_shards: 1
  467. number_of_replicas: 1
  468. mappings:
  469. properties:
  470. author:
  471. type: keyword
  472. tags:
  473. type: keyword
  474. - do:
  475. bulk:
  476. index: stackoverflow
  477. refresh: true
  478. body: |'''
  479. // Make Kibana strongly connected to elasticsearch and logstash
  480. // Make Kibana rarer (and therefore higher-ranking) than JavaScript
  481. // Make JavaScript strongly connected to jquery and angular
  482. // Make Cabana strongly connected to elasticsearch but only as a result of a single author
  483. for (int i = 0; i < 150; i++) {
  484. buildRestTests.setups['stackoverflow'] += """
  485. {"index":{}}
  486. {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}"""
  487. }
  488. for (int i = 0; i < 50; i++) {
  489. buildRestTests.setups['stackoverflow'] += """
  490. {"index":{}}
  491. {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}"""
  492. }
  493. for (int i = 0; i < 200; i++) {
  494. buildRestTests.setups['stackoverflow'] += """
  495. {"index":{}}
  496. {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}"""
  497. }
  498. for (int i = 0; i < 200; i++) {
  499. buildRestTests.setups['stackoverflow'] += """
  500. {"index":{}}
  501. {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}"""
  502. }
  503. for (int i = 0; i < 50; i++) {
  504. buildRestTests.setups['stackoverflow'] += """
  505. {"index":{}}
  506. {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}"""
  507. }
  508. buildRestTests.setups['stackoverflow'] += """
  509. """
  510. // Used by significant_text aggregation docs
  511. buildRestTests.setups['news'] = '''
  512. - do:
  513. indices.create:
  514. index: news
  515. body:
  516. settings:
  517. number_of_shards: 1
  518. number_of_replicas: 1
  519. mappings:
  520. properties:
  521. source:
  522. type: keyword
  523. content:
  524. type: text
  525. copy_to: custom_all
  526. custom_all:
  527. type: text
  528. - do:
  529. bulk:
  530. index: news
  531. refresh: true
  532. body: |'''
  533. // Make h5n1 strongly connected to bird flu
  534. for (int i = 0; i < 100; i++) {
  535. buildRestTests.setups['news'] += """
  536. {"index":{}}
  537. {"source": "very_relevant_$i", "content": "bird flu h5n1"}"""
  538. }
  539. for (int i = 0; i < 100; i++) {
  540. buildRestTests.setups['news'] += """
  541. {"index":{}}
  542. {"source": "filler_$i", "content": "bird dupFiller "}"""
  543. }
  544. for (int i = 0; i < 100; i++) {
  545. buildRestTests.setups['news'] += """
  546. {"index":{}}
  547. {"source": "filler_$i", "content": "flu dupFiller "}"""
  548. }
  549. for (int i = 0; i < 20; i++) {
  550. buildRestTests.setups['news'] += """
  551. {"index":{}}
  552. {"source": "partially_relevant_$i", "content": "elasticsearch dupFiller dupFiller dupFiller dupFiller pozmantier"}"""
  553. }
  554. for (int i = 0; i < 10; i++) {
  555. buildRestTests.setups['news'] += """
  556. {"index":{}}
  557. {"source": "partially_relevant_$i", "content": "elasticsearch logstash kibana"}"""
  558. }
  559. buildRestTests.setups['news'] += """
  560. """
  561. // Used by some aggregations
  562. buildRestTests.setups['exams'] = '''
  563. - do:
  564. indices.create:
  565. index: exams
  566. body:
  567. settings:
  568. number_of_shards: 1
  569. number_of_replicas: 1
  570. mappings:
  571. properties:
  572. grade:
  573. type: byte
  574. - do:
  575. bulk:
  576. index: exams
  577. refresh: true
  578. body: |
  579. {"index":{}}
  580. {"grade": 100, "weight": 2}
  581. {"index":{}}
  582. {"grade": 50, "weight": 3}'''
  583. buildRestTests.setups['stored_scripted_metric_script'] = '''
  584. - do:
  585. put_script:
  586. id: "my_init_script"
  587. body: { "script": { "lang": "painless", "source": "state.transactions = []" } }
  588. - match: { acknowledged: true }
  589. - do:
  590. put_script:
  591. id: "my_map_script"
  592. body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } }
  593. - match: { acknowledged: true }
  594. - do:
  595. put_script:
  596. id: "my_combine_script"
  597. body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } }
  598. - match: { acknowledged: true }
  599. - do:
  600. put_script:
  601. id: "my_reduce_script"
  602. body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } }
  603. - match: { acknowledged: true }
  604. '''
  605. // Used by analyze api
  606. buildRestTests.setups['analyze_sample'] = '''
  607. - do:
  608. indices.create:
  609. index: analyze_sample
  610. body:
  611. settings:
  612. number_of_shards: 1
  613. number_of_replicas: 0
  614. analysis:
  615. normalizer:
  616. my_normalizer:
  617. type: custom
  618. filter: [lowercase]
  619. mappings:
  620. properties:
  621. obj1.field1:
  622. type: text'''
  623. // Used by percentile/percentile-rank aggregations
  624. buildRestTests.setups['latency'] = '''
  625. - do:
  626. indices.create:
  627. index: latency
  628. body:
  629. settings:
  630. number_of_shards: 1
  631. number_of_replicas: 1
  632. mappings:
  633. properties:
  634. load_time:
  635. type: long
  636. - do:
  637. bulk:
  638. index: latency
  639. refresh: true
  640. body: |'''
  641. for (int i = 0; i < 100; i++) {
  642. def value = i
  643. if (i % 10) {
  644. value = i * 10
  645. }
  646. buildRestTests.setups['latency'] += """
  647. {"index":{}}
  648. {"load_time": "$value"}"""
  649. }
  650. // Used by t_test aggregations
  651. buildRestTests.setups['node_upgrade'] = '''
  652. - do:
  653. indices.create:
  654. index: node_upgrade
  655. body:
  656. settings:
  657. number_of_shards: 1
  658. number_of_replicas: 1
  659. mappings:
  660. properties:
  661. group:
  662. type: keyword
  663. startup_time_before:
  664. type: long
  665. startup_time_after:
  666. type: long
  667. - do:
  668. bulk:
  669. index: node_upgrade
  670. refresh: true
  671. body: |
  672. {"index":{}}
  673. {"group": "A", "startup_time_before": 102, "startup_time_after": 89}
  674. {"index":{}}
  675. {"group": "A", "startup_time_before": 99, "startup_time_after": 93}
  676. {"index":{}}
  677. {"group": "A", "startup_time_before": 111, "startup_time_after": 72}
  678. {"index":{}}
  679. {"group": "B", "startup_time_before": 97, "startup_time_after": 98}
  680. {"index":{}}
  681. {"group": "B", "startup_time_before": 101, "startup_time_after": 102}
  682. {"index":{}}
  683. {"group": "B", "startup_time_before": 99, "startup_time_after": 98}'''
  684. // Used by iprange agg
  685. buildRestTests.setups['iprange'] = '''
  686. - do:
  687. indices.create:
  688. index: ip_addresses
  689. body:
  690. settings:
  691. number_of_shards: 1
  692. number_of_replicas: 1
  693. mappings:
  694. properties:
  695. ip:
  696. type: ip
  697. - do:
  698. bulk:
  699. index: ip_addresses
  700. refresh: true
  701. body: |'''
  702. for (int i = 0; i < 255; i++) {
  703. buildRestTests.setups['iprange'] += """
  704. {"index":{}}
  705. {"ip": "10.0.0.$i"}"""
  706. }
  707. for (int i = 0; i < 5; i++) {
  708. buildRestTests.setups['iprange'] += """
  709. {"index":{}}
  710. {"ip": "9.0.0.$i"}"""
  711. buildRestTests.setups['iprange'] += """
  712. {"index":{}}
  713. {"ip": "11.0.0.$i"}"""
  714. buildRestTests.setups['iprange'] += """
  715. {"index":{}}
  716. {"ip": "12.0.0.$i"}"""
  717. }
  718. // Used by SQL because it looks SQL-ish
  719. buildRestTests.setups['library'] = '''
  720. - do:
  721. indices.create:
  722. index: library
  723. body:
  724. settings:
  725. number_of_shards: 1
  726. number_of_replicas: 1
  727. mappings:
  728. properties:
  729. name:
  730. type: text
  731. fields:
  732. keyword:
  733. type: keyword
  734. author:
  735. type: text
  736. fields:
  737. keyword:
  738. type: keyword
  739. release_date:
  740. type: date
  741. page_count:
  742. type: short
  743. - do:
  744. bulk:
  745. index: library
  746. refresh: true
  747. body: |
  748. {"index":{"_id": "Leviathan Wakes"}}
  749. {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
  750. {"index":{"_id": "Hyperion"}}
  751. {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
  752. {"index":{"_id": "Dune"}}
  753. {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
  754. {"index":{"_id": "Dune Messiah"}}
  755. {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331}
  756. {"index":{"_id": "Children of Dune"}}
  757. {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408}
  758. {"index":{"_id": "God Emperor of Dune"}}
  759. {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454}
  760. {"index":{"_id": "Consider Phlebas"}}
  761. {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471}
  762. {"index":{"_id": "Pandora's Star"}}
  763. {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768}
  764. {"index":{"_id": "Revelation Space"}}
  765. {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585}
  766. {"index":{"_id": "A Fire Upon the Deep"}}
  767. {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613}
  768. {"index":{"_id": "Ender's Game"}}
  769. {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324}
  770. {"index":{"_id": "1984"}}
  771. {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328}
  772. {"index":{"_id": "Fahrenheit 451"}}
  773. {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227}
  774. {"index":{"_id": "Brave New World"}}
  775. {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268}
  776. {"index":{"_id": "Foundation"}}
  777. {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224}
  778. {"index":{"_id": "The Giver"}}
  779. {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208}
  780. {"index":{"_id": "Slaughterhouse-Five"}}
  781. {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275}
  782. {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}}
  783. {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180}
  784. {"index":{"_id": "Snow Crash"}}
  785. {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470}
  786. {"index":{"_id": "Neuromancer"}}
  787. {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271}
  788. {"index":{"_id": "The Handmaid's Tale"}}
  789. {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}
  790. {"index":{"_id": "Starship Troopers"}}
  791. {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335}
  792. {"index":{"_id": "The Left Hand of Darkness"}}
  793. {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304}
  794. {"index":{"_id": "The Moon is a Harsh Mistress"}}
  795. {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288}
  796. '''
  797. buildRestTests.setups['sensor_rollup_job'] = '''
  798. - do:
  799. indices.create:
  800. index: sensor-1
  801. body:
  802. settings:
  803. number_of_shards: 1
  804. number_of_replicas: 0
  805. mappings:
  806. properties:
  807. timestamp:
  808. type: date
  809. temperature:
  810. type: long
  811. voltage:
  812. type: float
  813. node:
  814. type: keyword
  815. - do:
  816. raw:
  817. method: PUT
  818. path: _rollup/job/sensor
  819. body: >
  820. {
  821. "index_pattern": "sensor-*",
  822. "rollup_index": "sensor_rollup",
  823. "cron": "*/30 * * * * ?",
  824. "page_size" :1000,
  825. "groups" : {
  826. "date_histogram": {
  827. "field": "timestamp",
  828. "fixed_interval": "1h",
  829. "delay": "7d"
  830. },
  831. "terms": {
  832. "fields": ["node"]
  833. }
  834. },
  835. "metrics": [
  836. {
  837. "field": "temperature",
  838. "metrics": ["min", "max", "sum"]
  839. },
  840. {
  841. "field": "voltage",
  842. "metrics": ["avg"]
  843. }
  844. ]
  845. }
  846. '''
  847. buildRestTests.setups['sensor_started_rollup_job'] = '''
  848. - do:
  849. indices.create:
  850. index: sensor-1
  851. body:
  852. settings:
  853. number_of_shards: 1
  854. number_of_replicas: 0
  855. mappings:
  856. properties:
  857. timestamp:
  858. type: date
  859. temperature:
  860. type: long
  861. voltage:
  862. type: float
  863. node:
  864. type: keyword
  865. - do:
  866. bulk:
  867. index: sensor-1
  868. refresh: true
  869. body: |
  870. {"index":{}}
  871. {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
  872. {"index":{}}
  873. {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
  874. {"index":{}}
  875. {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
  876. {"index":{}}
  877. {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
  878. {"index":{}}
  879. {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
  880. {"index":{}}
  881. {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
  882. - do:
  883. raw:
  884. method: PUT
  885. path: _rollup/job/sensor
  886. body: >
  887. {
  888. "index_pattern": "sensor-*",
  889. "rollup_index": "sensor_rollup",
  890. "cron": "* * * * * ?",
  891. "page_size" :1000,
  892. "groups" : {
  893. "date_histogram": {
  894. "field": "timestamp",
  895. "fixed_interval": "1h",
  896. "delay": "7d"
  897. },
  898. "terms": {
  899. "fields": ["node"]
  900. }
  901. },
  902. "metrics": [
  903. {
  904. "field": "temperature",
  905. "metrics": ["min", "max", "sum"]
  906. },
  907. {
  908. "field": "voltage",
  909. "metrics": ["avg"]
  910. }
  911. ]
  912. }
  913. - do:
  914. raw:
  915. method: POST
  916. path: _rollup/job/sensor/_start
  917. '''
  918. buildRestTests.setups['sensor_index'] = '''
  919. - do:
  920. indices.create:
  921. index: sensor-1
  922. body:
  923. settings:
  924. number_of_shards: 1
  925. number_of_replicas: 0
  926. mappings:
  927. properties:
  928. timestamp:
  929. type: date
  930. temperature:
  931. type: long
  932. voltage:
  933. type: float
  934. node:
  935. type: keyword
  936. load:
  937. type: double
  938. net_in:
  939. type: long
  940. net_out:
  941. type: long
  942. hostname:
  943. type: keyword
  944. datacenter:
  945. type: keyword
  946. '''
  947. buildRestTests.setups['sensor_prefab_data'] = '''
  948. - do:
  949. indices.create:
  950. index: sensor-1
  951. body:
  952. settings:
  953. number_of_shards: 1
  954. number_of_replicas: 0
  955. mappings:
  956. properties:
  957. timestamp:
  958. type: date
  959. temperature:
  960. type: long
  961. voltage:
  962. type: float
  963. node:
  964. type: keyword
  965. - do:
  966. indices.create:
  967. index: sensor_rollup
  968. body:
  969. settings:
  970. number_of_shards: 1
  971. number_of_replicas: 0
  972. mappings:
  973. properties:
  974. node.terms.value:
  975. type: keyword
  976. temperature.sum.value:
  977. type: double
  978. temperature.max.value:
  979. type: double
  980. temperature.min.value:
  981. type: double
  982. timestamp.date_histogram.time_zone:
  983. type: keyword
  984. timestamp.date_histogram.interval:
  985. type: keyword
  986. timestamp.date_histogram.timestamp:
  987. type: date
  988. timestamp.date_histogram._count:
  989. type: long
  990. voltage.avg.value:
  991. type: double
  992. voltage.avg._count:
  993. type: long
  994. _rollup.id:
  995. type: keyword
  996. _rollup.version:
  997. type: long
  998. _meta:
  999. _rollup:
  1000. sensor:
  1001. cron: "* * * * * ?"
  1002. rollup_index: "sensor_rollup"
  1003. index_pattern: "sensor-*"
  1004. timeout: "20s"
  1005. page_size: 1000
  1006. groups:
  1007. date_histogram:
  1008. delay: "7d"
  1009. field: "timestamp"
  1010. fixed_interval: "60m"
  1011. time_zone: "UTC"
  1012. terms:
  1013. fields:
  1014. - "node"
  1015. id: sensor
  1016. metrics:
  1017. - field: "temperature"
  1018. metrics:
  1019. - min
  1020. - max
  1021. - sum
  1022. - field: "voltage"
  1023. metrics:
  1024. - avg
  1025. - do:
  1026. bulk:
  1027. index: sensor_rollup
  1028. refresh: true
  1029. body: |
  1030. {"index":{}}
  1031. {"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1032. {"index":{}}
  1033. {"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1034. {"index":{}}
  1035. {"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1036. {"index":{}}
  1037. {"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1038. {"index":{}}
  1039. {"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1040. {"index":{}}
  1041. {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
  1042. '''
  1043. buildRestTests.setups['sample_job'] = '''
  1044. - do:
  1045. ml.put_job:
  1046. job_id: "sample_job"
  1047. body: >
  1048. {
  1049. "description" : "Very basic job",
  1050. "analysis_config" : {
  1051. "bucket_span":"10m",
  1052. "detectors" :[
  1053. {
  1054. "function": "count"
  1055. }
  1056. ]},
  1057. "data_description" : {
  1058. "time_field":"timestamp",
  1059. "time_format": "epoch_ms"
  1060. }
  1061. }
  1062. '''
  1063. buildRestTests.setups['farequote_index'] = '''
  1064. - do:
  1065. indices.create:
  1066. index: farequote
  1067. body:
  1068. settings:
  1069. number_of_shards: 1
  1070. number_of_replicas: 0
  1071. mappings:
  1072. metric:
  1073. properties:
  1074. time:
  1075. type: date
  1076. responsetime:
  1077. type: float
  1078. airline:
  1079. type: keyword
  1080. doc_count:
  1081. type: integer
  1082. '''
  1083. buildRestTests.setups['farequote_data'] = buildRestTests.setups['farequote_index'] + '''
  1084. - do:
  1085. bulk:
  1086. index: farequote
  1087. refresh: true
  1088. body: |
  1089. {"index": {"_id":"1"}}
  1090. {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5}
  1091. {"index": {"_id":"2"}}
  1092. {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23}
  1093. {"index": {"_id":"3"}}
  1094. {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42}
  1095. '''
  1096. buildRestTests.setups['farequote_job'] = buildRestTests.setups['farequote_data'] + '''
  1097. - do:
  1098. ml.put_job:
  1099. job_id: "farequote"
  1100. body: >
  1101. {
  1102. "analysis_config": {
  1103. "bucket_span": "60m",
  1104. "detectors": [{
  1105. "function": "mean",
  1106. "field_name": "responsetime",
  1107. "by_field_name": "airline"
  1108. }],
  1109. "summary_count_field_name": "doc_count"
  1110. },
  1111. "data_description": {
  1112. "time_field": "time"
  1113. }
  1114. }
  1115. '''
  1116. buildRestTests.setups['farequote_datafeed'] = buildRestTests.setups['farequote_job'] + '''
  1117. - do:
  1118. ml.put_datafeed:
  1119. datafeed_id: "datafeed-farequote"
  1120. body: >
  1121. {
  1122. "job_id":"farequote",
  1123. "indexes":"farequote"
  1124. }
  1125. '''
  1126. buildRestTests.setups['categorize_text'] = '''
  1127. - do:
  1128. indices.create:
  1129. index: log-messages
  1130. body:
  1131. settings:
  1132. number_of_shards: 1
  1133. number_of_replicas: 0
  1134. mappings:
  1135. properties:
  1136. time:
  1137. type: date
  1138. message:
  1139. type: text
  1140. - do:
  1141. bulk:
  1142. index: log-messages
  1143. refresh: true
  1144. body: |
  1145. {"index": {"_id":"1"}}
  1146. {"time":"2016-02-07T00:01:00+0000", "message": "2016-02-07T00:00:00+0000 Node 3 shutting down"}
  1147. {"index": {"_id":"2"}}
  1148. {"time":"2016-02-07T00:02:00+0000", "message": "2016-02-07T00:00:00+0000 Node 5 starting up"}
  1149. {"index": {"_id":"3"}}
  1150. {"time":"2016-02-07T00:03:00+0000", "message": "2016-02-07T00:00:00+0000 Node 4 shutting down"}
  1151. {"index": {"_id":"4"}}
  1152. {"time":"2016-02-08T00:01:00+0000", "message": "2016-02-08T00:00:00+0000 Node 5 shutting down"}
  1153. {"index": {"_id":"5"}}
  1154. {"time":"2016-02-08T00:02:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_325 logging on"}
  1155. {"index": {"_id":"6"}}
  1156. {"time":"2016-02-08T00:04:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_864 logged off"}
  1157. '''
  1158. buildRestTests.setups['server_metrics_index'] = '''
  1159. - do:
  1160. indices.create:
  1161. index: server-metrics
  1162. body:
  1163. settings:
  1164. number_of_shards: 1
  1165. number_of_replicas: 0
  1166. mappings:
  1167. properties:
  1168. timestamp:
  1169. type: date
  1170. total:
  1171. type: long
  1172. '''
  1173. buildRestTests.setups['server_metrics_data'] = buildRestTests.setups['server_metrics_index'] + '''
  1174. - do:
  1175. bulk:
  1176. index: server-metrics
  1177. refresh: true
  1178. body: |
  1179. {"index": {"_id":"1177"}}
  1180. {"timestamp":"2017-03-23T13:00:00","total":40476}
  1181. {"index": {"_id":"1178"}}
  1182. {"timestamp":"2017-03-23T13:00:00","total":15287}
  1183. {"index": {"_id":"1179"}}
  1184. {"timestamp":"2017-03-23T13:00:00","total":-776}
  1185. {"index": {"_id":"1180"}}
  1186. {"timestamp":"2017-03-23T13:00:00","total":11366}
  1187. {"index": {"_id":"1181"}}
  1188. {"timestamp":"2017-03-23T13:00:00","total":3606}
  1189. {"index": {"_id":"1182"}}
  1190. {"timestamp":"2017-03-23T13:00:00","total":19006}
  1191. {"index": {"_id":"1183"}}
  1192. {"timestamp":"2017-03-23T13:00:00","total":38613}
  1193. {"index": {"_id":"1184"}}
  1194. {"timestamp":"2017-03-23T13:00:00","total":19516}
  1195. {"index": {"_id":"1185"}}
  1196. {"timestamp":"2017-03-23T13:00:00","total":-258}
  1197. {"index": {"_id":"1186"}}
  1198. {"timestamp":"2017-03-23T13:00:00","total":9551}
  1199. {"index": {"_id":"1187"}}
  1200. {"timestamp":"2017-03-23T13:00:00","total":11217}
  1201. {"index": {"_id":"1188"}}
  1202. {"timestamp":"2017-03-23T13:00:00","total":22557}
  1203. {"index": {"_id":"1189"}}
  1204. {"timestamp":"2017-03-23T13:00:00","total":40508}
  1205. {"index": {"_id":"1190"}}
  1206. {"timestamp":"2017-03-23T13:00:00","total":11887}
  1207. {"index": {"_id":"1191"}}
  1208. {"timestamp":"2017-03-23T13:00:00","total":31659}
  1209. '''
  1210. buildRestTests.setups['server_metrics_job'] = buildRestTests.setups['server_metrics_data'] + '''
  1211. - do:
  1212. ml.put_job:
  1213. job_id: "total-requests"
  1214. body: >
  1215. {
  1216. "description" : "Total sum of requests",
  1217. "analysis_config" : {
  1218. "bucket_span":"10m",
  1219. "detectors" :[
  1220. {
  1221. "detector_description": "Sum of total",
  1222. "function": "sum",
  1223. "field_name": "total"
  1224. }
  1225. ]},
  1226. "data_description" : {
  1227. "time_field":"timestamp",
  1228. "time_format": "epoch_ms"
  1229. }
  1230. }
  1231. '''
  1232. buildRestTests.setups['server_metrics_job-raw'] = buildRestTests.setups['server_metrics_data'] + '''
  1233. - do:
  1234. raw:
  1235. method: PUT
  1236. path: _ml/anomaly_detectors/total-requests
  1237. body: >
  1238. {
  1239. "description" : "Total sum of requests",
  1240. "analysis_config" : {
  1241. "bucket_span":"10m",
  1242. "detectors" :[
  1243. {
  1244. "detector_description": "Sum of total",
  1245. "function": "sum",
  1246. "field_name": "total"
  1247. }
  1248. ]},
  1249. "data_description" : {
  1250. "time_field":"timestamp",
  1251. "time_format": "epoch_ms"
  1252. }
  1253. }
  1254. '''
  1255. buildRestTests.setups['server_metrics_datafeed'] = buildRestTests.setups['server_metrics_job'] + '''
  1256. - do:
  1257. ml.put_datafeed:
  1258. datafeed_id: "datafeed-total-requests"
  1259. body: >
  1260. {
  1261. "job_id":"total-requests",
  1262. "indexes":"server-metrics"
  1263. }
  1264. '''
  1265. buildRestTests.setups['server_metrics_datafeed-raw'] = buildRestTests.setups['server_metrics_job-raw'] + '''
  1266. - do:
  1267. raw:
  1268. method: PUT
  1269. path: _ml/datafeeds/datafeed-total-requests
  1270. body: >
  1271. {
  1272. "job_id":"total-requests",
  1273. "indexes":"server-metrics"
  1274. }
  1275. '''
  1276. buildRestTests.setups['server_metrics_openjob'] = buildRestTests.setups['server_metrics_datafeed'] + '''
  1277. - do:
  1278. ml.open_job:
  1279. job_id: "total-requests"
  1280. '''
  1281. buildRestTests.setups['server_metrics_openjob-raw'] = buildRestTests.setups['server_metrics_datafeed-raw'] + '''
  1282. - do:
  1283. raw:
  1284. method: POST
  1285. path: _ml/anomaly_detectors/total-requests/_open
  1286. '''
  1287. buildRestTests.setups['server_metrics_startdf'] = buildRestTests.setups['server_metrics_openjob'] + '''
  1288. - do:
  1289. ml.start_datafeed:
  1290. datafeed_id: "datafeed-total-requests"
  1291. '''
  1292. buildRestTests.setups['calendar_outages'] = '''
  1293. - do:
  1294. ml.put_calendar:
  1295. calendar_id: "planned-outages"
  1296. '''
  1297. buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['calendar_outages'] + '''
  1298. - do:
  1299. ml.post_calendar_events:
  1300. calendar_id: "planned-outages"
  1301. body: >
  1302. { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" }
  1303. '''
  1304. buildRestTests.setups['calendar_outages_openjob'] = buildRestTests.setups['server_metrics_openjob'] + '''
  1305. - do:
  1306. ml.put_calendar:
  1307. calendar_id: "planned-outages"
  1308. '''
  1309. buildRestTests.setups['calendar_outages_addjob'] = buildRestTests.setups['server_metrics_openjob'] + '''
  1310. - do:
  1311. ml.put_calendar:
  1312. calendar_id: "planned-outages"
  1313. body: >
  1314. {
  1315. "job_ids": ["total-requests"]
  1316. }
  1317. '''
  1318. setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + '''
  1319. - do:
  1320. ml.post_calendar_events:
  1321. calendar_id: "planned-outages"
  1322. body: >
  1323. { "events" : [
  1324. { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"},
  1325. { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"},
  1326. { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"}
  1327. ]}
  1328. '''
  1329. // used by median absolute deviation aggregation
  1330. setups['reviews'] = '''
  1331. - do:
  1332. indices.create:
  1333. index: reviews
  1334. body:
  1335. settings:
  1336. number_of_shards: 1
  1337. number_of_replicas: 0
  1338. mappings:
  1339. properties:
  1340. product:
  1341. type: keyword
  1342. rating:
  1343. type: long
  1344. - do:
  1345. bulk:
  1346. index: reviews
  1347. refresh: true
  1348. body: |
  1349. {"index": {"_id": "1"}}
  1350. {"product": "widget-foo", "rating": 1}
  1351. {"index": {"_id": "2"}}
  1352. {"product": "widget-foo", "rating": 5}
  1353. '''
  1354. setups['remote_cluster'] = setups['host'] + '''
  1355. - do:
  1356. cluster.put_settings:
  1357. body:
  1358. persistent:
  1359. cluster.remote.remote_cluster.seeds: $transport_host
  1360. '''
  1361. setups['remote_cluster_and_leader_index'] = setups['remote_cluster'] + '''
  1362. - do:
  1363. indices.create:
  1364. index: leader_index
  1365. body:
  1366. settings:
  1367. index.number_of_replicas: 0
  1368. index.number_of_shards: 1
  1369. index.soft_deletes.enabled: true
  1370. '''
  1371. setups['remote_cluster_and_leader_index_and_follower_index'] = setups['remote_cluster_and_leader_index'] + '''
  1372. - do:
  1373. raw:
  1374. method: PUT
  1375. path: 'follower_index/_ccr/follow'
  1376. wait_for_active_shards: 1
  1377. body: |
  1378. {
  1379. "remote_cluster" : "remote_cluster",
  1380. "leader_index" : "leader_index"
  1381. }
  1382. - is_true: follow_index_created
  1383. - is_true: follow_index_shards_acked
  1384. - is_true: index_following_started
  1385. '''
  1386. teardowns['pause_follow'] = '''
  1387. - do:
  1388. raw:
  1389. method: POST
  1390. path: 'follower_index/_ccr/pause_follow'
  1391. - is_true: acknowledged
  1392. '''
  1393. setups['seats'] = '''
  1394. - do:
  1395. indices.create:
  1396. index: seats
  1397. body:
  1398. settings:
  1399. number_of_shards: 1
  1400. number_of_replicas: 0
  1401. mappings:
  1402. properties:
  1403. theatre:
  1404. type: keyword
  1405. play:
  1406. type: keyword
  1407. actors:
  1408. type: keyword
  1409. date:
  1410. type: keyword
  1411. time:
  1412. type: keyword
  1413. cost:
  1414. type: long
  1415. row:
  1416. type: long
  1417. number:
  1418. type: long
  1419. sold:
  1420. type: boolean
  1421. datetime:
  1422. type: date
  1423. - do:
  1424. raw:
  1425. method: PUT
  1426. path: "_ingest/pipeline/seats"
  1427. body: |
  1428. {
  1429. "description": "update datetime for seats",
  1430. "processors": [
  1431. {
  1432. "script": {
  1433. "source": "String[] dateSplit = ctx.date.splitOnToken('-'); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = '0' + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = '0' + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals('PM'); String[] timeSplit = ctx.time.substring(0, ctx.time.length() - 2).splitOnToken(':'); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + '-' + month + '-' + day + 'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + (minutes < 10 ? '0' + minutes : '' + minutes) + ':00+08:00'; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;"
  1434. }
  1435. }
  1436. ]
  1437. }
  1438. - do:
  1439. bulk:
  1440. index: seats
  1441. pipeline: seats
  1442. refresh: true
  1443. body: |
  1444. {"index":{"_id": "1"}}
  1445. {"theatre": "Skyline", "play": "Rent", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 37, "row": 1, "number": 7, "sold": false}
  1446. {"index":{"_id": "2"}}
  1447. {"theatre": "Graye", "play": "Rent", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 30, "row": 3, "number": 5, "sold": false}
  1448. {"index":{"_id": "3"}}
  1449. {"theatre": "Graye", "play": "Rented", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 33, "row": 2, "number": 6, "sold": false}
  1450. {"index":{"_id": "4"}}
  1451. {"theatre": "Skyline", "play": "Rented", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 20, "row": 5, "number": 2, "sold": false}
  1452. {"index":{"_id": "5"}}
  1453. {"theatre": "Down Port", "play": "Pick It Up", "actors": ["Joel Madigan", "Jessica Brown", "Baz Knight", "Jo Hangum", "Rachel Grass", "Phoebe Miller"], "date": "2018-4-2", "time": "8:00PM", "cost": 27.5, "row": 3, "number": 2, "sold": false}
  1454. {"index":{"_id": "6"}}
  1455. {"theatre": "Down Port", "play": "Harriot", "actors": ["Phoebe Miller", "Sarah Notch", "Brayden Green", "Joshua Iller", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-8-7", "time": "8:00PM", "cost": 30.0, "row": 1, "number": 10, "sold": false}
  1456. {"index":{"_id": "7"}}
  1457. {"theatre": "Skyline", "play": "Auntie Jo", "actors": ["Jo Hangum", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue"], "date": "2018-10-2", "time": "5:40PM", "cost": 22.5, "row": 7, "number": 10, "sold": false}
  1458. {"index":{"_id": "8"}}
  1459. {"theatre": "Skyline", "play": "Test Run", "actors": ["Joe Muir", "Ryan Earns", "Joel Madigan", "Jessica Brown"], "date": "2018-8-5", "time": "7:30PM", "cost": 17.5, "row": 11, "number": 12, "sold": true}
  1460. {"index":{"_id": "9"}}
  1461. {"theatre": "Skyline", "play": "Sunnyside Down", "actors": ["Krissy Smith", "Joe Muir", "Ryan Earns", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-6-12", "time": "4:00PM", "cost": 21.25, "row": 8, "number": 15, "sold": true}
  1462. {"index":{"_id": "10"}}
  1463. {"theatre": "Graye", "play": "Line and Single", "actors": ["Nora Blue", "Mike Candlestick"], "date": "2018-6-5", "time": "2:00PM", "cost": 30.0, "row": 1, "number": 2, "sold": false}
  1464. {"index":{"_id":"11"}}
  1465. {"theatre": "Graye", "play": "Hamilton", "actors": ["Lin-Manuel Miranda", "Leslie Odom Jr."] ,"date":"2018-6-5", "time": "2:00PM", "cost": 5000.0, "row":1, "number": 20, "sold": true}
  1466. '''
  1467. setups['kibana_sample_data_ecommerce'] = '''
  1468. - do:
  1469. indices.create:
  1470. index: kibana_sample_data_ecommerce
  1471. body:
  1472. settings:
  1473. number_of_shards: 1
  1474. number_of_replicas: 0
  1475. mappings:
  1476. properties:
  1477. order_date:
  1478. type: date
  1479. '''
  1480. setups['add_timestamp_pipeline'] = '''
  1481. - do:
  1482. ingest.put_pipeline:
  1483. id: "add_timestamp_pipeline"
  1484. body: >
  1485. {
  1486. "processors": [
  1487. {
  1488. "set" : {
  1489. "field" : "@timestamp",
  1490. "value" : "{{_ingest.timestamp}}"
  1491. }
  1492. }
  1493. ]
  1494. }
  1495. '''
  1496. setups['simple_kibana_continuous_pivot'] = setups['kibana_sample_data_ecommerce'] + setups['add_timestamp_pipeline'] + '''
  1497. - do:
  1498. raw:
  1499. method: PUT
  1500. path: _transform/simple-kibana-ecomm-pivot
  1501. body: >
  1502. {
  1503. "source": {
  1504. "index": "kibana_sample_data_ecommerce",
  1505. "query": {
  1506. "term": {
  1507. "geoip.continent_name": {
  1508. "value": "Asia"
  1509. }
  1510. }
  1511. }
  1512. },
  1513. "pivot": {
  1514. "group_by": {
  1515. "customer_id": {
  1516. "terms": {
  1517. "field": "customer_id"
  1518. }
  1519. }
  1520. },
  1521. "aggregations": {
  1522. "max_price": {
  1523. "max": {
  1524. "field": "taxful_total_price"
  1525. }
  1526. }
  1527. }
  1528. },
  1529. "description": "Maximum priced ecommerce data",
  1530. "dest": {
  1531. "index": "kibana_sample_data_ecommerce_transform",
  1532. "pipeline": "add_timestamp_pipeline"
  1533. },
  1534. "frequency": "5m",
  1535. "sync": {
  1536. "time": {
  1537. "field": "order_date",
  1538. "delay": "60s"
  1539. }
  1540. }
  1541. }
  1542. '''
  1543. setups['setup_houseprices'] = '''
  1544. - do:
  1545. indices.create:
  1546. index: houses_sold_last_10_yrs
  1547. body:
  1548. settings:
  1549. number_of_shards: 1
  1550. number_of_replicas: 1
  1551. '''
  1552. setups['setup_logdata'] = '''
  1553. - do:
  1554. indices.create:
  1555. index: logdata
  1556. body:
  1557. settings:
  1558. number_of_shards: 1
  1559. number_of_replicas: 1
  1560. mappings:
  1561. properties:
  1562. grade:
  1563. type: byte
  1564. - do:
  1565. bulk:
  1566. index: logdata
  1567. refresh: true
  1568. body: |
  1569. {"index":{}}
  1570. {"grade": 100, "weight": 2}
  1571. {"index":{}}
  1572. {"grade": 50, "weight": 3}
  1573. '''
  1574. setups['logdata_job'] = setups['setup_logdata'] + '''
  1575. - do:
  1576. ml.put_data_frame_analytics:
  1577. id: "loganalytics"
  1578. body: >
  1579. {
  1580. "source": {
  1581. "index": "logdata"
  1582. },
  1583. "dest": {
  1584. "index": "logdata_out"
  1585. },
  1586. "analysis": {
  1587. "outlier_detection": {}
  1588. }
  1589. }
  1590. '''
  1591. // Used by snapshot lifecycle management docs
  1592. setups['setup-repository'] = '''
  1593. - do:
  1594. snapshot.create_repository:
  1595. repository: my_repository
  1596. body:
  1597. type: fs
  1598. settings:
  1599. location: buildDir/cluster/shared/repo
  1600. '''
  1601. // Used by snapshot restore docs
  1602. setups['setup-snapshots'] = setups['setup-repository'] + '''
  1603. - do:
  1604. bulk:
  1605. index: my-index
  1606. refresh: true
  1607. body: |
  1608. {"index":{"_id": "0"}}
  1609. {"message": "trying out Elasticsearch", "context": "foo"}
  1610. - do:
  1611. bulk:
  1612. index: logs-my_app-default
  1613. refresh: true
  1614. body: |
  1615. {"create":{"_id": "0"}}
  1616. {"message": "trying out Elasticsearch", "context": "foo"}
  1617. - do:
  1618. snapshot.create:
  1619. repository: my_repository
  1620. snapshot: my_snapshot_2099.05.06
  1621. wait_for_completion: true
  1622. '''
  1623. // Fake sec logs data used by EQL search
  1624. setups['atomic_red_regsvr32'] = setups['my_data_stream'] + '''
  1625. - do:
  1626. bulk:
  1627. index: my-data-stream
  1628. refresh: true
  1629. body: |
  1630. #atomic_red_data#
  1631. '''
  1632. setups['my_inactive_watch'] = '''
  1633. - do:
  1634. watcher.put_watch:
  1635. id: "my_watch"
  1636. active: false
  1637. body: >
  1638. {
  1639. "trigger": {
  1640. "schedule": {
  1641. "hourly": {
  1642. "minute": [ 0, 5 ]
  1643. }
  1644. }
  1645. },
  1646. "input": {
  1647. "simple": {
  1648. "payload": {
  1649. "send": "yes"
  1650. }
  1651. }
  1652. },
  1653. "condition": {
  1654. "always": {}
  1655. },
  1656. "actions": {
  1657. "test_index": {
  1658. "index": {
  1659. "index": "test"
  1660. }
  1661. }
  1662. }
  1663. }
  1664. - match: { _id: "my_watch" }
  1665. '''
  1666. setups['my_active_watch'] = setups['my_inactive_watch'].replace(
  1667. 'active: false', 'active: true')
  1668. setups['role_mapping'] = '''
  1669. - do:
  1670. security.put_role_mapping:
  1671. name: "mapping1"
  1672. body: >
  1673. {
  1674. "enabled": true,
  1675. "roles": [ "user" ],
  1676. "rules": { "field": { "username": "*" } }
  1677. }
  1678. '''
  1679. setups['admin_role'] = '''
  1680. - do:
  1681. security.put_role:
  1682. name: "my_admin_role"
  1683. body: >
  1684. {
  1685. "cluster": ["all"],
  1686. "indices": [
  1687. {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}}
  1688. ],
  1689. "run_as": [ "other_user" ],
  1690. "metadata" : {"version": 1}
  1691. }
  1692. '''
  1693. setups['jacknich_user'] = '''
  1694. - do:
  1695. security.put_user:
  1696. username: "jacknich"
  1697. body: >
  1698. {
  1699. "password" : "l0ng-r4nd0m-p@ssw0rd",
  1700. "roles" : [ "admin", "other_role1" ],
  1701. "full_name" : "Jack Nicholson",
  1702. "email" : "jacknich@example.com",
  1703. "metadata" : { "intelligence" : 7 }
  1704. }
  1705. - do:
  1706. security.activate_user_profile:
  1707. body: >
  1708. {
  1709. "grant_type": "password",
  1710. "username": "jacknich",
  1711. "password" : "l0ng-r4nd0m-p@ssw0rd"
  1712. }
  1713. '''
  1714. setups['sandrakn_user'] = '''
  1715. - do:
  1716. security.put_user:
  1717. username: "sandrakn"
  1718. body: >
  1719. {
  1720. "password" : "l0ng-r4nd0m-p@ssw0rd",
  1721. "roles" : [ "admin", "other_role1" ],
  1722. "full_name" : "Sandra Knight",
  1723. "email" : "sandrakn@example.com",
  1724. "metadata" : { "intelligence" : 7 }
  1725. }
  1726. - do:
  1727. security.activate_user_profile:
  1728. body: >
  1729. {
  1730. "grant_type": "password",
  1731. "username": "sandrakn",
  1732. "password" : "l0ng-r4nd0m-p@ssw0rd"
  1733. }
  1734. '''
  1735. setups['app0102_privileges'] = '''
  1736. - do:
  1737. security.put_privileges:
  1738. body: >
  1739. {
  1740. "myapp": {
  1741. "read": {
  1742. "application": "myapp",
  1743. "name": "read",
  1744. "actions": [
  1745. "data:read/*",
  1746. "action:login" ],
  1747. "metadata": {
  1748. "description": "Read access to myapp"
  1749. }
  1750. }
  1751. }
  1752. }
  1753. '''
  1754. setups['service_token42'] = '''
  1755. - do:
  1756. security.create_service_token:
  1757. namespace: elastic
  1758. service: fleet-server
  1759. name: token42
  1760. '''
  1761. setups['user_profiles'] = '''
  1762. - do:
  1763. security.put_user:
  1764. username: "jacknich"
  1765. body: >
  1766. {
  1767. "password" : "l0ng-r4nd0m-p@ssw0rd",
  1768. "roles" : [ "admin", "other_role1" ],
  1769. "full_name" : "Jack Nicholson",
  1770. "email" : "jacknich@example.com"
  1771. }
  1772. - do:
  1773. security.put_user:
  1774. username: "jackrea"
  1775. body: >
  1776. {
  1777. "password" : "l0ng-r4nd0m-p@ssw0rd",
  1778. "roles" : [ "admin" ],
  1779. "full_name" : "Jack Reacher",
  1780. "email" : "jackrea@example.com"
  1781. }
  1782. - do:
  1783. security.put_user:
  1784. username: "jackspa"
  1785. body: >
  1786. {
  1787. "password" : "l0ng-r4nd0m-p@ssw0rd",
  1788. "roles" : [ "user" ],
  1789. "full_name" : "Jack Sparrow",
  1790. "email" : "jackspa@example.com"
  1791. }
  1792. - do:
  1793. security.activate_user_profile:
  1794. body: >
  1795. {
  1796. "grant_type": "password",
  1797. "username": "jacknich",
  1798. "password" : "l0ng-r4nd0m-p@ssw0rd"
  1799. }
  1800. - do:
  1801. security.activate_user_profile:
  1802. body: >
  1803. {
  1804. "grant_type": "password",
  1805. "username": "jackrea",
  1806. "password" : "l0ng-r4nd0m-p@ssw0rd"
  1807. }
  1808. - do:
  1809. security.activate_user_profile:
  1810. body: >
  1811. {
  1812. "grant_type": "password",
  1813. "username": "jackspa",
  1814. "password" : "l0ng-r4nd0m-p@ssw0rd"
  1815. }
  1816. # jacknich
  1817. - do:
  1818. security.update_user_profile_data:
  1819. uid: "u_79HkWkwmnBH5gqFKwoxggWPjEBOur1zLPXQPEl1VBW0_0"
  1820. body: >
  1821. {
  1822. "labels": {
  1823. "direction": "north"
  1824. },
  1825. "data": {
  1826. "app1": {
  1827. "key1": "value1"
  1828. }
  1829. }
  1830. }
  1831. # jackrea
  1832. - do:
  1833. security.update_user_profile_data:
  1834. uid: "u_P_0BMHgaOK3p7k-PFWUCbw9dQ-UFjt01oWJ_Dp2PmPc_0"
  1835. body: >
  1836. {
  1837. "labels": {
  1838. "direction": "west"
  1839. }
  1840. }
  1841. # jackspa
  1842. - do:
  1843. security.update_user_profile_data:
  1844. uid: "u_8RKO7AKfEbSiIHZkZZ2LJy2MUSDPWDr3tMI_CkIGApU_0"
  1845. body: >
  1846. {
  1847. "labels": {
  1848. "direction": "south"
  1849. }
  1850. }
  1851. '''
  1852. // fake data used by the correlation bucket agg
  1853. buildRestTests.setups['correlate_latency'] = '''
  1854. - do:
  1855. indices.create:
  1856. index: correlate_latency
  1857. body:
  1858. settings:
  1859. number_of_shards: 1
  1860. number_of_replicas: 0
  1861. mappings:
  1862. properties:
  1863. latency:
  1864. type: double
  1865. version:
  1866. type: keyword
  1867. - do:
  1868. bulk:
  1869. index: correlate_latency
  1870. refresh: true
  1871. body: |'''
  1872. for (int i = 100; i < 200; i++) {
  1873. def value = i
  1874. if (i % 10) {
  1875. value = i * 10
  1876. }
  1877. buildRestTests.setups['correlate_latency'] += """
  1878. {"index":{}}
  1879. {"latency": "$value", "version": "1.0"}"""
  1880. }
  1881. for (int i = 0; i < 100; i++) {
  1882. def value = i
  1883. if (i % 10) {
  1884. value = i * 10
  1885. }
  1886. buildRestTests.setups['correlate_latency'] += """
  1887. {"index":{}}
  1888. {"latency": "$value", "version": "2.0"}"""
  1889. }
  1890. /* Load the actual events only if we're going to use them. */
  1891. File atomicRedRegsvr32File = new File("$projectDir/src/yamlRestTest/resources/normalized-T1117-AtomicRed-regsvr32.json")
  1892. inputs.file(atomicRedRegsvr32File)
  1893. doFirst {
  1894. String events = atomicRedRegsvr32File.getText('UTF-8')
  1895. // Indent like a yaml test needs
  1896. events = events.replaceAll('(?m)^', ' ')
  1897. setups['atomic_red_regsvr32'] =
  1898. setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events)
  1899. }
  1900. }