|
@@ -28,7 +28,7 @@ mappings and change <<index-modules-settings,dynamic index settings>>. See
|
|
|
////
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_ilm/policy/logs_policy
|
|
|
+PUT /_ilm/policy/my-data-stream-policy
|
|
|
{
|
|
|
"policy": {
|
|
|
"phases": {
|
|
@@ -49,23 +49,23 @@ PUT /_ilm/policy/logs_policy
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-PUT /_index_template/logs_data_stream
|
|
|
+PUT /_index_template/my-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "logs*" ],
|
|
|
+ "index_patterns": [ "my-data-stream*" ],
|
|
|
"data_stream": { }
|
|
|
}
|
|
|
|
|
|
-PUT /_index_template/new_logs_data_stream
|
|
|
+PUT /_index_template/new-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "new_logs*" ],
|
|
|
+ "index_patterns": [ "new-data-stream*" ],
|
|
|
"data_stream": { }
|
|
|
}
|
|
|
|
|
|
-PUT /_data_stream/logs
|
|
|
+PUT /_data_stream/my-data-stream
|
|
|
|
|
|
-POST /logs/_rollover/
|
|
|
+POST /my-data-stream/_rollover/
|
|
|
|
|
|
-PUT /_data_stream/new_logs
|
|
|
+PUT /_data_stream/new-data-stream
|
|
|
----
|
|
|
// TESTSETUP
|
|
|
|
|
@@ -75,7 +75,7 @@ DELETE /_data_stream/*
|
|
|
|
|
|
DELETE /_index_template/*
|
|
|
|
|
|
-DELETE /_ilm/policy/logs_policy
|
|
|
+DELETE /_ilm/policy/my-data-stream-policy
|
|
|
----
|
|
|
// TEARDOWN
|
|
|
////
|
|
@@ -90,17 +90,17 @@ To add a mapping for a new field to a data stream, following these steps:
|
|
|
field mapping is added to future backing indices created for the stream.
|
|
|
+
|
|
|
--
|
|
|
-For example, `logs_data_stream` is an existing index template used by the `logs`
|
|
|
-data stream.
|
|
|
+For example, `my-data-stream-template` is an existing index template used by
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
The following <<indices-templates,put index template>> request adds a mapping
|
|
|
for a new field, `message`, to the template.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_index_template/logs_data_stream
|
|
|
+PUT /_index_template/my-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "logs*" ],
|
|
|
+ "index_patterns": [ "my-data-stream*" ],
|
|
|
"data_stream": { },
|
|
|
"template": {
|
|
|
"mappings": {
|
|
@@ -122,11 +122,11 @@ backing indices, including the write index.
|
|
|
+
|
|
|
--
|
|
|
The following put mapping API request adds the new `message` field mapping to
|
|
|
-the `logs` data stream.
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /logs/_mapping
|
|
|
+PUT /my-data-stream/_mapping
|
|
|
{
|
|
|
"properties": {
|
|
|
"message": {
|
|
@@ -142,12 +142,12 @@ To add the mapping only to the stream's write index, set the put mapping API's
|
|
|
+
|
|
|
--
|
|
|
The following put mapping request adds the new `message` field mapping only to
|
|
|
-the `logs` stream's write index. The new field mapping is not added to the
|
|
|
-stream's other backing indices.
|
|
|
+`my-data-stream`'s write index. The new field mapping is not added to
|
|
|
+the stream's other backing indices.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /logs/_mapping?write_index_only=true
|
|
|
+PUT /my-data-stream/_mapping?write_index_only=true
|
|
|
{
|
|
|
"properties": {
|
|
|
"message": {
|
|
@@ -171,8 +171,8 @@ existing field, follow these steps:
|
|
|
field mapping is added to future backing indices created for the stream.
|
|
|
+
|
|
|
--
|
|
|
-For example, `logs_data_stream` is an existing index template used by the `logs`
|
|
|
-data stream.
|
|
|
+For example, `my-data-stream-template` is an existing index template used by
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
The following <<indices-templates,put index template>> request changes the
|
|
|
argument for the `host.ip` field's <<ignore-malformed,`ignore_malformed`>>
|
|
@@ -180,9 +180,9 @@ mapping parameter to `true`.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_index_template/logs_data_stream
|
|
|
+PUT /_index_template/my-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "logs*" ],
|
|
|
+ "index_patterns": [ "my-data-stream*" ],
|
|
|
"data_stream": { },
|
|
|
"template": {
|
|
|
"mappings": {
|
|
@@ -208,13 +208,13 @@ to the data stream. By default, this applies the changes to the stream's
|
|
|
existing backing indices, including the write index.
|
|
|
+
|
|
|
--
|
|
|
-The following <<indices-put-mapping,put mapping API>> request targets the `logs`
|
|
|
-data stream. The request changes the argument for the `host.ip` field's
|
|
|
-`ignore_malformed` mapping parameter to `true`.
|
|
|
+The following <<indices-put-mapping,put mapping API>> request targets
|
|
|
+`my-data-stream`. The request changes the argument for the `host.ip`
|
|
|
+field's `ignore_malformed` mapping parameter to `true`.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /logs/_mapping
|
|
|
+PUT /my-data-stream/_mapping
|
|
|
{
|
|
|
"properties": {
|
|
|
"host": {
|
|
@@ -230,17 +230,17 @@ PUT /logs/_mapping
|
|
|
----
|
|
|
--
|
|
|
+
|
|
|
-To apply the mapping changes only to the stream's write index, set the put mapping API's
|
|
|
-`write_index_only` query parameter to `true`.
|
|
|
+To apply the mapping changes only to the stream's write index, set the put
|
|
|
+mapping API's `write_index_only` query parameter to `true`.
|
|
|
+
|
|
|
--
|
|
|
The following put mapping request changes the `host.ip` field's mapping only for
|
|
|
-the `logs` stream's write index. The change is not applied to the stream's other
|
|
|
-backing indices.
|
|
|
+`my-data-stream`'s write index. The change is not applied to the
|
|
|
+stream's other backing indices.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /logs/_mapping?write_index_only=true
|
|
|
+PUT /my-data-stream/_mapping?write_index_only=true
|
|
|
{
|
|
|
"properties": {
|
|
|
"host": {
|
|
@@ -276,17 +276,17 @@ follow these steps:
|
|
|
applied to future backing indices created for the stream.
|
|
|
+
|
|
|
--
|
|
|
-For example, `logs_data_stream` is an existing index template used by the `logs`
|
|
|
-data stream.
|
|
|
+For example, `my-data-stream-template` is an existing index template used by
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
The following <<indices-templates,put index template>> request changes the
|
|
|
template's `index.refresh_interval` index setting to `30s` (30 seconds).
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_index_template/logs_data_stream
|
|
|
+PUT /_index_template/my-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "logs*" ],
|
|
|
+ "index_patterns": [ "my-data-stream*" ],
|
|
|
"data_stream": { },
|
|
|
"template": {
|
|
|
"settings": {
|
|
@@ -304,11 +304,11 @@ the stream's existing backing indices, including the write index.
|
|
|
+
|
|
|
--
|
|
|
The following update index settings API request updates the
|
|
|
-`index.refresh_interval` setting for the `logs` data stream.
|
|
|
+`index.refresh_interval` setting for `my-data-stream`.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /logs/_settings
|
|
|
+PUT /my-data-stream/_settings
|
|
|
{
|
|
|
"index": {
|
|
|
"refresh_interval": "30s"
|
|
@@ -329,17 +329,17 @@ To apply a new static setting to future backing indices, update the index
|
|
|
template used by the data stream. The setting is automatically applied to any
|
|
|
backing index created after the update.
|
|
|
|
|
|
-For example, `logs_data_stream` is an existing index template used by the `logs`
|
|
|
-data stream.
|
|
|
+For example, `my-data-stream-template` is an existing index template used by
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
The following <<indices-templates,put index template API>> requests adds new
|
|
|
`sort.field` and `sort.order index` settings to the template.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_index_template/logs_data_stream
|
|
|
+PUT /_index_template/my-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "logs*" ],
|
|
|
+ "index_patterns": [ "my-data-stream*" ],
|
|
|
"data_stream": { },
|
|
|
"template": {
|
|
|
"settings": {
|
|
@@ -386,12 +386,12 @@ existing indices, index aliases, or data streams. If so, you should consider
|
|
|
using another name or pattern.
|
|
|
--
|
|
|
The following resolve index API request checks for any existing indices, index
|
|
|
-aliases, or data streams that start with `new_logs`. If not, the `new_logs*`
|
|
|
-wildcard pattern can be used to create a new data stream.
|
|
|
+aliases, or data streams that start with `new-data-stream`. If not, the
|
|
|
+`new-data-stream*` wildcard pattern can be used to create a new data stream.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-GET /_resolve/index/new_logs*
|
|
|
+GET /_resolve/index/new-data-stream*
|
|
|
----
|
|
|
|
|
|
The API returns the following response, indicating no existing targets match
|
|
@@ -421,25 +421,26 @@ TIP: If you are only adding or changing a few things, we recommend you create a
|
|
|
new template by copying an existing one and modifying it as needed.
|
|
|
+
|
|
|
--
|
|
|
-For example, `logs_data_stream` is an existing index template used by the
|
|
|
-`logs` data stream.
|
|
|
+For example, `my-data-stream-template` is an existing index template used by
|
|
|
+`my-data-stream`.
|
|
|
|
|
|
-The following <<indices-templates,put index template API>> request creates
|
|
|
-a new index template, `new_logs_data_stream`. `new_logs_data_stream`
|
|
|
-uses the `logs_data_stream` template as its basis, with the following changes:
|
|
|
+The following <<indices-templates,put index template API>> request creates a new
|
|
|
+index template, `new-data-stream-template`. `new-data-stream-template`
|
|
|
+uses `my-data-stream-template` as its basis, with the following
|
|
|
+changes:
|
|
|
|
|
|
* The `index_patterns` wildcard pattern matches any index or data stream
|
|
|
- starting with `new_logs`.
|
|
|
+ starting with `new-data-stream`.
|
|
|
* The `@timestamp` field mapping uses the `date_nanos` field data type rather
|
|
|
than the `date` data type.
|
|
|
* The template includes `sort.field` and `sort.order` index settings, which were
|
|
|
- not in the original `logs_data_stream` template.
|
|
|
+ not in the original `my-data-stream-template` template.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_index_template/new_logs_data_stream
|
|
|
+PUT /_index_template/new-data-stream-template
|
|
|
{
|
|
|
- "index_patterns": [ "new_logs*" ],
|
|
|
+ "index_patterns": [ "new-data-stream*" ],
|
|
|
"data_stream": { },
|
|
|
"template": {
|
|
|
"mappings": {
|
|
@@ -481,16 +482,16 @@ to retain such a backing index until you are ready to delete its newest data.
|
|
|
====
|
|
|
+
|
|
|
--
|
|
|
-The following create data stream API request targets `new_logs`, which matches
|
|
|
-the wildcard pattern for the `new_logs_data_stream` template. Because no
|
|
|
-existing index or data stream uses this name, this request creates the
|
|
|
-`new_logs` data stream.
|
|
|
+The following create data stream API request targets `new-data-stream`, which
|
|
|
+matches the wildcard pattern for `new-data-stream-template`.
|
|
|
+Because no existing index or data stream uses this name, this request creates
|
|
|
+the `new-data-stream` data stream.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-PUT /_data_stream/new_logs
|
|
|
+PUT /_data_stream/new-data-stream
|
|
|
----
|
|
|
-// TEST[s/new_logs/new_logs_two/]
|
|
|
+// TEST[s/new-data-stream/new-data-stream-two/]
|
|
|
--
|
|
|
|
|
|
. If you do not want to mix new and old data in your new data stream, pause the
|
|
@@ -527,46 +528,46 @@ individual backing indices as the source. You can use the
|
|
|
indices.
|
|
|
+
|
|
|
--
|
|
|
-You plan to reindex data from the `logs` data stream into the newly created
|
|
|
-`new_logs` data stream. However, you want to submit a separate reindex request
|
|
|
-for each backing index in the `logs` data stream, starting with the oldest
|
|
|
-backing index. This preserves the order in which the data was originally
|
|
|
-indexed.
|
|
|
+For example, you plan to reindex data from `my-data-stream` into
|
|
|
+`new-data-stream`. However, you want to submit a separate reindex request for
|
|
|
+each backing index in `my-data-stream`, starting with the oldest backing index.
|
|
|
+This preserves the order in which the data was originally indexed.
|
|
|
|
|
|
-The following get data stream API request retrieves information about the `logs`
|
|
|
-data stream, including a list of its backing indices.
|
|
|
+The following get data stream API request retrieves information about
|
|
|
+`my-data-stream`, including a list of its backing indices.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-GET /_data_stream/logs
|
|
|
+GET /_data_stream/my-data-stream
|
|
|
----
|
|
|
|
|
|
The API returns the following response. Note the `indices` property contains an
|
|
|
array of the stream's current backing indices. The first item in the array
|
|
|
-contains information about the stream's oldest backing index, `.ds-logs-000001`.
|
|
|
+contains information about the stream's oldest backing index,
|
|
|
+`.ds-my-data-stream-000001`.
|
|
|
|
|
|
[source,console-result]
|
|
|
----
|
|
|
{
|
|
|
"data_streams": [
|
|
|
{
|
|
|
- "name": "logs",
|
|
|
+ "name": "my-data-stream",
|
|
|
"timestamp_field": {
|
|
|
"name": "@timestamp"
|
|
|
},
|
|
|
"indices": [
|
|
|
{
|
|
|
- "index_name": ".ds-logs-000001", <1>
|
|
|
+ "index_name": ".ds-my-data-stream-000001", <1>
|
|
|
"index_uuid": "Gpdiyq8sRuK9WuthvAdFbw"
|
|
|
},
|
|
|
{
|
|
|
- "index_name": ".ds-logs-000002",
|
|
|
+ "index_name": ".ds-my-data-stream-000002",
|
|
|
"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"
|
|
|
}
|
|
|
],
|
|
|
"generation": 2,
|
|
|
"status": "GREEN",
|
|
|
- "template": "logs_data_stream"
|
|
|
+ "template": "my-data-stream-template"
|
|
|
}
|
|
|
]
|
|
|
}
|
|
@@ -575,22 +576,23 @@ contains information about the stream's oldest backing index, `.ds-logs-000001`.
|
|
|
// TESTRESPONSE[s/"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
|
|
|
// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
|
|
|
|
|
|
-<1> First item in the `indices` array for the `logs` data stream. This item
|
|
|
-contains information about the stream's oldest backing index, `.ds-logs-000001`.
|
|
|
+<1> First item in the `indices` array for `my-data-stream`. This
|
|
|
+item contains information about the stream's oldest backing index,
|
|
|
+`.ds-my-data-stream-000001`.
|
|
|
|
|
|
The following <<docs-reindex,reindex API>> request copies documents from
|
|
|
-`.ds-logs-000001` to the `new_logs` data stream. Note the request's `op_type` is
|
|
|
-`create`.
|
|
|
+`.ds-my-data-stream-000001` to `new-data-stream`. Note the request's `op_type`
|
|
|
+is `create`.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
POST /_reindex
|
|
|
{
|
|
|
"source": {
|
|
|
- "index": ".ds-logs-000001"
|
|
|
+ "index": ".ds-my-data-stream-000001"
|
|
|
},
|
|
|
"dest": {
|
|
|
- "index": "new_logs",
|
|
|
+ "index": "new-data-stream",
|
|
|
"op_type": "create"
|
|
|
}
|
|
|
}
|
|
@@ -601,9 +603,9 @@ You can also use a query to reindex only a subset of documents with each
|
|
|
request.
|
|
|
+
|
|
|
--
|
|
|
-The following <<docs-reindex,reindex API>> request copies documents from the
|
|
|
-`logs` data stream to the `new_logs` data stream. The request uses a
|
|
|
-<<query-dsl-range-query,`range` query>> to only reindex documents with a
|
|
|
+The following <<docs-reindex,reindex API>> request copies documents from
|
|
|
+`my-data-stream` to `new-data-stream`. The request
|
|
|
+uses a <<query-dsl-range-query,`range` query>> to only reindex documents with a
|
|
|
timestamp within the last week. Note the request's `op_type` is `create`.
|
|
|
|
|
|
[source,console]
|
|
@@ -611,7 +613,7 @@ timestamp within the last week. Note the request's `op_type` is `create`.
|
|
|
POST /_reindex
|
|
|
{
|
|
|
"source": {
|
|
|
- "index": "logs",
|
|
|
+ "index": "my-data-stream",
|
|
|
"query": {
|
|
|
"range": {
|
|
|
"@timestamp": {
|
|
@@ -622,7 +624,7 @@ POST /_reindex
|
|
|
}
|
|
|
},
|
|
|
"dest": {
|
|
|
- "index": "new_logs",
|
|
|
+ "index": "new-data-stream",
|
|
|
"op_type": "create"
|
|
|
}
|
|
|
}
|
|
@@ -656,11 +658,11 @@ data stream, you can safely remove the old stream.
|
|
|
+
|
|
|
--
|
|
|
The following <<indices-delete-data-stream,delete data stream API>> request
|
|
|
-deletes the `logs` data stream. This request also deletes the stream's backing
|
|
|
-indices and any data they contain.
|
|
|
+deletes `my-data-stream`. This request also deletes the stream's
|
|
|
+backing indices and any data they contain.
|
|
|
|
|
|
[source,console]
|
|
|
----
|
|
|
-DELETE /_data_stream/logs
|
|
|
+DELETE /_data_stream/my-data-stream
|
|
|
----
|
|
|
--
|