|
|
@@ -53,11 +53,274 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=allow-no-jobs]
|
|
|
[[ml-get-job-stats-results]]
|
|
|
==== {api-response-body-title}
|
|
|
|
|
|
-The API returns the following information:
|
|
|
+The API returns the following information about the operational progress of a
|
|
|
+job:
|
|
|
|
|
|
-`jobs`::
|
|
|
-(array)
|
|
|
-include::{docdir}/ml/ml-shared.asciidoc[tag=jobs-stats-anomaly-detection]
|
|
|
+`assignment_explanation`::
|
|
|
+(string) For open jobs only, contains messages relating to the selection
|
|
|
+of a node to run the job.
|
|
|
+
|
|
|
+[[datacounts]]`data_counts`::
|
|
|
+(object) An object that describes the quantity of input to the job and any
|
|
|
+related error counts. The `data_count` values are cumulative for the lifetime of
|
|
|
+a job. If a model snapshot is reverted or old results are deleted, the job
|
|
|
+counts are not reset.
|
|
|
+
|
|
|
+`bucket_count`:::
|
|
|
+(long) The number of bucket results produced by the job.
|
|
|
+
|
|
|
+`earliest_record_timestamp`:::
|
|
|
+(date) The timestamp of the earliest chronologically input document.
|
|
|
+
|
|
|
+`empty_bucket_count`:::
|
|
|
+(long) The number of buckets which did not contain any data. If your data
|
|
|
+contains many empty buckets, consider increasing your `bucket_span` or using
|
|
|
+functions that are tolerant to gaps in data such as `mean`, `non_null_sum` or
|
|
|
+`non_zero_count`.
|
|
|
+
|
|
|
+`input_bytes`:::
|
|
|
+(long) The number of bytes of input data posted to the job.
|
|
|
+
|
|
|
+`input_field_count`:::
|
|
|
+(long) The total number of fields in input documents posted to the job. This
|
|
|
+count includes fields that are not used in the analysis. However, be aware that
|
|
|
+if you are using a {dfeed}, it extracts only the required fields from the
|
|
|
+documents it retrieves before posting them to the job.
|
|
|
+
|
|
|
+`input_record_count`:::
|
|
|
+(long) The number of input documents posted to the job.
|
|
|
+
|
|
|
+`invalid_date_count`:::
|
|
|
+(long) The number of input documents with either a missing date field or a date
|
|
|
+that could not be parsed.
|
|
|
+
|
|
|
+`job_id`:::
|
|
|
+(string)
|
|
|
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
|
|
|
+
|
|
|
+`last_data_time`:::
|
|
|
+(date) The timestamp at which data was last analyzed, according to server time.
|
|
|
+
|
|
|
+`latest_empty_bucket_timestamp`:::
|
|
|
+(date) The timestamp of the last bucket that did not contain any data.
|
|
|
+
|
|
|
+`latest_record_timestamp`:::
|
|
|
+(date) The timestamp of the latest chronologically input document.
|
|
|
+
|
|
|
+`latest_sparse_bucket_timestamp`:::
|
|
|
+(date) The timestamp of the last bucket that was considered sparse.
|
|
|
+
|
|
|
+`missing_field_count`:::
|
|
|
+(long) The number of input documents that are missing a field that the job is
|
|
|
+configured to analyze. Input documents with missing fields are still processed
|
|
|
+because it is possible that not all fields are missing. The value of
|
|
|
+`processed_record_count` includes this count.
|
|
|
++
|
|
|
+--
|
|
|
+NOTE: If you are using {dfeeds} or posting data to the job in JSON format, a
|
|
|
+high `missing_field_count` is often not an indication of data issues. It is not
|
|
|
+necessarily a cause for concern.
|
|
|
+
|
|
|
+--
|
|
|
+
|
|
|
+`out_of_order_timestamp_count`:::
|
|
|
+(long) The number of input documents that are out of time sequence and outside
|
|
|
+of the latency window. This information is applicable only when you provide data
|
|
|
+to the job by using the <<ml-post-data,post data API>>. These out of order
|
|
|
+documents are discarded, since jobs require time series data to be in ascending
|
|
|
+chronological order.
|
|
|
+
|
|
|
+`processed_field_count`:::
|
|
|
+(long) The total number of fields in all the documents that have been processed
|
|
|
+by the job. Only fields that are specified in the detector configuration object
|
|
|
+contribute to this count. The timestamp is not included in this count.
|
|
|
+
|
|
|
+`processed_record_count`:::
|
|
|
+(long) The number of input documents that have been processed by the job. This
|
|
|
+value includes documents with missing fields, since they are nonetheless
|
|
|
+analyzed. If you use {dfeeds} and have aggregations in your search query, the
|
|
|
+`processed_record_count` will be the number of aggregation results processed,
|
|
|
+not the number of {es} documents.
|
|
|
+
|
|
|
+`sparse_bucket_count`:::
|
|
|
+(long) The number of buckets that contained few data points compared to the
|
|
|
+expected number of data points. If your data contains many sparse buckets,
|
|
|
+consider using a longer `bucket_span`.
|
|
|
+
|
|
|
+[[forecastsstats]]`forecasts_stats`::
|
|
|
+(object) An object that provides statistical information about forecasts
|
|
|
+of this job. It has the following properties:
|
|
|
++
|
|
|
+--
|
|
|
+NOTE: Unless there is at least one forecast, `memory_bytes`, `records`,
|
|
|
+`processing_time_ms` and `status` properties are omitted.
|
|
|
+
|
|
|
+--
|
|
|
+
|
|
|
+`forecasted_jobs`:::
|
|
|
+(long) The number of jobs that have at least one forecast.
|
|
|
+
|
|
|
+`memory_bytes`:::
|
|
|
+(object) Statistics about the memory usage: minimum, maximum, average and total.
|
|
|
+
|
|
|
+`records`:::
|
|
|
+(object) Statistics about the number of forecast records: minimum, maximum,
|
|
|
+saverage and total.
|
|
|
+
|
|
|
+`processing_time_ms`:::
|
|
|
+(object) Statistics about the forecast runtime in milliseconds: minimum, maximum,
|
|
|
+average and total.
|
|
|
+
|
|
|
+`status`:::
|
|
|
+(object) Counts per forecast status. For example: `{"finished" : 2}`.
|
|
|
+
|
|
|
+`total`:::
|
|
|
+(long) The number of forecasts currently available for this model.
|
|
|
+
|
|
|
+`job_id`::
|
|
|
+(string)
|
|
|
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
|
|
|
+
|
|
|
+[[modelsizestats]]`model_size_stats`::
|
|
|
+(object) An object that provides information about the size and contents of the
|
|
|
+model. It has the following properties:
|
|
|
+
|
|
|
+`bucket_allocation_failures_count`:::
|
|
|
+(long) The number of buckets for which new entities in incoming data were not
|
|
|
+processed due to insufficient model memory. This situation is also signified
|
|
|
+by a `hard_limit: memory_status` property value.
|
|
|
+
|
|
|
+`job_id`:::
|
|
|
+(string)
|
|
|
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
|
|
|
+
|
|
|
+`log_time`:::
|
|
|
+(date) The timestamp of the `model_size_stats` according to server time.
|
|
|
+
|
|
|
+`memory_status`:::
|
|
|
+(string) The status of the mathematical models. This property can have one of
|
|
|
+the following values:
|
|
|
++
|
|
|
+--
|
|
|
+* `ok`: The models stayed below the configured value.
|
|
|
+* `soft_limit`: The models used more than 60% of the configured memory limit
|
|
|
+and older unused models will be pruned to free up space.
|
|
|
+* `hard_limit`: The models used more space than the configured memory limit.
|
|
|
+As a result, not all incoming data was processed.
|
|
|
+--
|
|
|
+
|
|
|
+`model_bytes`:::
|
|
|
+(long) The number of bytes of memory used by the models. This is the maximum
|
|
|
+value since the last time the model was persisted. If the job is closed,
|
|
|
+this value indicates the latest size.
|
|
|
+
|
|
|
+`model_bytes_exceeded`:::
|
|
|
+ (long) The number of bytes over the high limit for memory usage at the last
|
|
|
+ allocation failure.
|
|
|
+
|
|
|
+`model_bytes_memory_limit`:::
|
|
|
+(long) The upper limit for memory usage, checked on increasing values.
|
|
|
+
|
|
|
+`result_type`:::
|
|
|
+(string) For internal use. The type of result.
|
|
|
+
|
|
|
+`total_by_field_count`:::
|
|
|
+(long) The number of `by` field values that were analyzed by the models.
|
|
|
++
|
|
|
+--
|
|
|
+NOTE: The `by` field values are counted separately for each detector and
|
|
|
+partition.
|
|
|
+
|
|
|
+--
|
|
|
+
|
|
|
+`total_over_field_count`:::
|
|
|
+(long) The number of `over` field values that were analyzed by the models.
|
|
|
++
|
|
|
+--
|
|
|
+NOTE: The `over` field values are counted separately for each detector and
|
|
|
+partition.
|
|
|
+
|
|
|
+--
|
|
|
+
|
|
|
+`total_partition_field_count`:::
|
|
|
+(long) The number of `partition` field values that were analyzed by the models.
|
|
|
+
|
|
|
+`timestamp`:::
|
|
|
+(date) The timestamp of the `model_size_stats` according to the timestamp of the
|
|
|
+data.
|
|
|
+
|
|
|
+[[stats-node]]`node`::
|
|
|
+(object) Contains properties for the node that runs the job. This information is
|
|
|
+available only for open jobs.
|
|
|
+
|
|
|
+`attributes`:::
|
|
|
+(object) Lists node attributes. For example,
|
|
|
+`{"ml.machine_memory": "17179869184", "ml.max_open_jobs" : "20"}`.
|
|
|
+
|
|
|
+`ephemeral_id`:::
|
|
|
+(string) The ephemeral ID of the node.
|
|
|
+
|
|
|
+`id`:::
|
|
|
+(string) The unique identifier of the node.
|
|
|
+
|
|
|
+`name`:::
|
|
|
+(string) The node name.
|
|
|
+
|
|
|
+`transport_address`:::
|
|
|
+(string) The host and port where transport HTTP connections are accepted.
|
|
|
+
|
|
|
+`open_time`::
|
|
|
+(string) For open jobs only, the elapsed time for which the job has been open.
|
|
|
+For example, `28746386s`.
|
|
|
+
|
|
|
+`state`::
|
|
|
+(string) The status of the job, which can be one of the following values:
|
|
|
++
|
|
|
+--
|
|
|
+* `closed`: The job finished successfully with its model state persisted. The
|
|
|
+job must be opened before it can accept further data.
|
|
|
+* `closing`: The job close action is in progress and has not yet completed. A
|
|
|
+closing job cannot accept further data.
|
|
|
+* `failed`: The job did not finish successfully due to an error. This situation
|
|
|
+can occur due to invalid input data, a fatal error occurring during the analysis,
|
|
|
+or an external interaction such as the process being killed by the Linux out of
|
|
|
+memory (OOM) killer. If the job had irrevocably failed, it must be force closed
|
|
|
+and then deleted. If the {dfeed} can be corrected, the job can be closed and
|
|
|
+then re-opened.
|
|
|
+* `opened`: The job is available to receive and process data.
|
|
|
+* `opening`: The job open action is in progress and has not yet completed.
|
|
|
+--
|
|
|
+
|
|
|
+[[timingstats]]`timing_stats`::
|
|
|
+(object) An object that provides statistical information about timing aspect of
|
|
|
+this job. It has the following properties:
|
|
|
+
|
|
|
+`average_bucket_processing_time_ms`:::
|
|
|
+(double) Average of all bucket processing times in milliseconds.
|
|
|
+
|
|
|
+`bucket_count`:::
|
|
|
+(long) The number of buckets processed.
|
|
|
+
|
|
|
+`exponential_average_bucket_processing_time_ms`:::
|
|
|
+(double) Exponential moving average of all bucket processing times in
|
|
|
+milliseconds.
|
|
|
+
|
|
|
+`exponential_average_bucket_processing_time_per_hour_ms`:::
|
|
|
+(double) Exponentially-weighted moving average of bucket processing times
|
|
|
+calculated in a 1 hour time window.
|
|
|
+
|
|
|
+`job_id`:::
|
|
|
+(string)
|
|
|
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
|
|
|
+
|
|
|
+`maximum_bucket_processing_time_ms`:::
|
|
|
+(double) Maximum among all bucket processing times in milliseconds.
|
|
|
+
|
|
|
+`minimum_bucket_processing_time_ms`:::
|
|
|
+(double) Minimum among all bucket processing times in milliseconds.
|
|
|
+
|
|
|
+`total_bucket_processing_time_ms`:::
|
|
|
+(double) Sum of all bucket processing times in milliseconds.
|
|
|
|
|
|
[[ml-get-job-stats-response-codes]]
|
|
|
==== {api-response-codes-title}
|
|
|
@@ -69,59 +332,100 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=jobs-stats-anomaly-detection]
|
|
|
[[ml-get-job-stats-example]]
|
|
|
==== {api-examples-title}
|
|
|
|
|
|
-The following example gets usage information for the `farequote` job:
|
|
|
-
|
|
|
[source,console]
|
|
|
--------------------------------------------------
|
|
|
-GET _ml/anomaly_detectors/farequote/_stats
|
|
|
+GET _ml/anomaly_detectors/low_request_rate/_stats
|
|
|
--------------------------------------------------
|
|
|
-// TEST[skip:todo]
|
|
|
+// TEST[skip:Kibana sample data]
|
|
|
|
|
|
The API returns the following results:
|
|
|
[source,js]
|
|
|
----
|
|
|
{
|
|
|
- "count": 1,
|
|
|
- "jobs": [
|
|
|
+ "count" : 1,
|
|
|
+ "jobs" : [
|
|
|
{
|
|
|
- "job_id": "farequote",
|
|
|
- "data_counts": {
|
|
|
- "job_id": "farequote",
|
|
|
- "processed_record_count": 86275,
|
|
|
- "processed_field_count": 172550,
|
|
|
- "input_bytes": 6744714,
|
|
|
- "input_field_count": 172550,
|
|
|
- "invalid_date_count": 0,
|
|
|
- "missing_field_count": 0,
|
|
|
- "out_of_order_timestamp_count": 0,
|
|
|
- "empty_bucket_count": 0,
|
|
|
- "sparse_bucket_count": 15,
|
|
|
- "bucket_count": 1528,
|
|
|
- "earliest_record_timestamp": 1454803200000,
|
|
|
- "latest_record_timestamp": 1455235196000,
|
|
|
- "last_data_time": 1491948163685,
|
|
|
- "latest_sparse_bucket_timestamp": 1455174900000,
|
|
|
- "input_record_count": 86275
|
|
|
+ "job_id" : "low_request_rate",
|
|
|
+ "data_counts" : {
|
|
|
+ "job_id" : "low_request_rate",
|
|
|
+ "processed_record_count" : 1216,
|
|
|
+ "processed_field_count" : 1216,
|
|
|
+ "input_bytes" : 51678,
|
|
|
+ "input_field_count" : 1216,
|
|
|
+ "invalid_date_count" : 0,
|
|
|
+ "missing_field_count" : 0,
|
|
|
+ "out_of_order_timestamp_count" : 0,
|
|
|
+ "empty_bucket_count" : 242,
|
|
|
+ "sparse_bucket_count" : 0,
|
|
|
+ "bucket_count" : 1457,
|
|
|
+ "earliest_record_timestamp" : 1575172659612,
|
|
|
+ "latest_record_timestamp" : 1580417369440,
|
|
|
+ "last_data_time" : 1576017595046,
|
|
|
+ "latest_empty_bucket_timestamp" : 1580356800000,
|
|
|
+ "input_record_count" : 1216
|
|
|
+ },
|
|
|
+ "model_size_stats" : {
|
|
|
+ "job_id" : "low_request_rate",
|
|
|
+ "result_type" : "model_size_stats",
|
|
|
+ "model_bytes" : 41480,
|
|
|
+ "model_bytes_exceeded" : 0,
|
|
|
+ "model_bytes_memory_limit" : 10485760,
|
|
|
+ "total_by_field_count" : 3,
|
|
|
+ "total_over_field_count" : 0,
|
|
|
+ "total_partition_field_count" : 2,
|
|
|
+ "bucket_allocation_failures_count" : 0,
|
|
|
+ "memory_status" : "ok",
|
|
|
+ "log_time" : 1576017596000,
|
|
|
+ "timestamp" : 1580410800000
|
|
|
+ },
|
|
|
+ "forecasts_stats" : {
|
|
|
+ "total" : 1,
|
|
|
+ "forecasted_jobs" : 1,
|
|
|
+ "memory_bytes" : {
|
|
|
+ "total" : 9179.0,
|
|
|
+ "min" : 9179.0,
|
|
|
+ "avg" : 9179.0,
|
|
|
+ "max" : 9179.0
|
|
|
+ },
|
|
|
+ "records" : {
|
|
|
+ "total" : 168.0,
|
|
|
+ "min" : 168.0,
|
|
|
+ "avg" : 168.0,
|
|
|
+ "max" : 168.0
|
|
|
+ },
|
|
|
+ "processing_time_ms" : {
|
|
|
+ "total" : 40.0,
|
|
|
+ "min" : 40.0,
|
|
|
+ "avg" : 40.0,
|
|
|
+ "max" : 40.0
|
|
|
+ },
|
|
|
+ "status" : {
|
|
|
+ "finished" : 1
|
|
|
+ }
|
|
|
},
|
|
|
- "model_size_stats": {
|
|
|
- "job_id": "farequote",
|
|
|
- "result_type": "model_size_stats",
|
|
|
- "model_bytes": 387594,
|
|
|
- "total_by_field_count": 21,
|
|
|
- "total_over_field_count": 0,
|
|
|
- "total_partition_field_count": 20,
|
|
|
- "bucket_allocation_failures_count": 0,
|
|
|
- "memory_status": "ok",
|
|
|
- "log_time": 1491948163000,
|
|
|
- "timestamp": 1455234600000
|
|
|
+ "state" : "opened",
|
|
|
+ "node" : {
|
|
|
+ "id" : "7bmMXyWCRs-TuPfGJJ_yMw",
|
|
|
+ "name" : "node-0",
|
|
|
+ "ephemeral_id" : "hoXMLZB0RWKfR9UPPUCxXX",
|
|
|
+ "transport_address" : "127.0.0.1:9300",
|
|
|
+ "attributes" : {
|
|
|
+ "ml.machine_memory" : "17179869184",
|
|
|
+ "xpack.installed" : "true",
|
|
|
+ "ml.max_open_jobs" : "20"
|
|
|
+ }
|
|
|
},
|
|
|
- "state": "closed",
|
|
|
- "timing_stats": {
|
|
|
- "job_id": "farequote",
|
|
|
- "minimum_bucket_processing_time_ms": 0.0,
|
|
|
- "maximum_bucket_processing_time_ms": 15.0,
|
|
|
- "average_bucket_processing_time_ms": 8.75,
|
|
|
- "exponential_average_bucket_processing_time_ms": 6.1435899
|
|
|
+ "assignment_explanation" : "",
|
|
|
+ "open_time" : "13s",
|
|
|
+ "timing_stats" : {
|
|
|
+ "job_id" : "low_request_rate",
|
|
|
+ "bucket_count" : 1457,
|
|
|
+ "total_bucket_processing_time_ms" : 1094.000000000001,
|
|
|
+ "minimum_bucket_processing_time_ms" : 0.0,
|
|
|
+ "maximum_bucket_processing_time_ms" : 48.0,
|
|
|
+ "average_bucket_processing_time_ms" : 0.75085792724777,
|
|
|
+ "exponential_average_bucket_processing_time_ms" : 0.5571716855800993,
|
|
|
+ "exponential_average_bucket_processing_time_per_hour_ms" : 15.0
|
|
|
}
|
|
|
}
|
|
|
]
|
Lisa Cawley