[ML] Adding failed_category_count to model_size_stats (#55716)

The failed_category_count statistic records the number of times
categorization wanted to create a new category but couldn't
because the job had reached its model_memory_limit.

Relates elastic/ml-cpp#1130

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java

@@ -59,6 +59,7 @@ public class ModelSizeStats implements ToXContentObject {
     public static final ParseField FREQUENT_CATEGORY_COUNT_FIELD = new ParseField("frequent_category_count");
     public static final ParseField RARE_CATEGORY_COUNT_FIELD = new ParseField("rare_category_count");
     public static final ParseField DEAD_CATEGORY_COUNT_FIELD = new ParseField("dead_category_count");
+    public static final ParseField FAILED_CATEGORY_COUNT_FIELD = new ParseField("failed_category_count");
     public static final ParseField CATEGORIZATION_STATUS_FIELD = new ParseField("categorization_status");
     public static final ParseField LOG_TIME_FIELD = new ParseField("log_time");
     public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp");
@@ -81,6 +82,7 @@ public class ModelSizeStats implements ToXContentObject {
         PARSER.declareLong(Builder::setFrequentCategoryCount, FREQUENT_CATEGORY_COUNT_FIELD);
         PARSER.declareLong(Builder::setRareCategoryCount, RARE_CATEGORY_COUNT_FIELD);
         PARSER.declareLong(Builder::setDeadCategoryCount, DEAD_CATEGORY_COUNT_FIELD);
+        PARSER.declareLong(Builder::setFailedCategoryCount, FAILED_CATEGORY_COUNT_FIELD);
         PARSER.declareField(Builder::setCategorizationStatus,
             p -> CategorizationStatus.fromString(p.text()), CATEGORIZATION_STATUS_FIELD, ValueType.STRING);
         PARSER.declareField(Builder::setLogTime,
@@ -143,6 +145,7 @@ public class ModelSizeStats implements ToXContentObject {
     private final long frequentCategoryCount;
     private final long rareCategoryCount;
     private final long deadCategoryCount;
+    private final long failedCategoryCount;
     private final CategorizationStatus categorizationStatus;
     private final Date timestamp;
     private final Date logTime;
@@ -150,8 +153,8 @@ public class ModelSizeStats implements ToXContentObject {
     private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount,
                            long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount,
                            MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount,
-                           long rareCategoryCount, long deadCategoryCount, CategorizationStatus categorizationStatus,
-                           Date timestamp, Date logTime) {
+                           long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
+                           CategorizationStatus categorizationStatus, Date timestamp, Date logTime) {
         this.jobId = jobId;
         this.modelBytes = modelBytes;
         this.modelBytesExceeded = modelBytesExceeded;
@@ -166,6 +169,7 @@ public class ModelSizeStats implements ToXContentObject {
         this.frequentCategoryCount = frequentCategoryCount;
         this.rareCategoryCount = rareCategoryCount;
         this.deadCategoryCount = deadCategoryCount;
+        this.failedCategoryCount = failedCategoryCount;
         this.categorizationStatus = categorizationStatus;
         this.timestamp = timestamp;
         this.logTime = logTime;
@@ -194,6 +198,7 @@ public class ModelSizeStats implements ToXContentObject {
         builder.field(FREQUENT_CATEGORY_COUNT_FIELD.getPreferredName(), frequentCategoryCount);
         builder.field(RARE_CATEGORY_COUNT_FIELD.getPreferredName(), rareCategoryCount);
         builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount);
+        builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount);
         builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus);
         builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime());
         if (timestamp != null) {
@@ -260,6 +265,10 @@ public class ModelSizeStats implements ToXContentObject {
         return deadCategoryCount;
     }
 
+    public long getFailedCategoryCount() {
+        return failedCategoryCount;
+    }
+
     public CategorizationStatus getCategorizationStatus() {
         return categorizationStatus;
     }
@@ -286,7 +295,7 @@ public class ModelSizeStats implements ToXContentObject {
     public int hashCode() {
         return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
             totalPartitionFieldCount, this.bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
-            frequentCategoryCount, rareCategoryCount, deadCategoryCount, categorizationStatus, timestamp, logTime);
+            frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
     }
 
     /**
@@ -314,6 +323,7 @@ public class ModelSizeStats implements ToXContentObject {
             && this.frequentCategoryCount == that.frequentCategoryCount
             && this.rareCategoryCount == that.rareCategoryCount
             && this.deadCategoryCount == that.deadCategoryCount
+            && this.failedCategoryCount == that.failedCategoryCount
             && Objects.equals(this.categorizationStatus, that.categorizationStatus)
             && Objects.equals(this.timestamp, that.timestamp)
             && Objects.equals(this.logTime, that.logTime)
@@ -336,6 +346,7 @@ public class ModelSizeStats implements ToXContentObject {
         private long frequentCategoryCount;
         private long rareCategoryCount;
         private long deadCategoryCount;
+        private long failedCategoryCount;
         private CategorizationStatus categorizationStatus;
         private Date timestamp;
         private Date logTime;
@@ -362,6 +373,7 @@ public class ModelSizeStats implements ToXContentObject {
             this.frequentCategoryCount = modelSizeStats.frequentCategoryCount;
             this.rareCategoryCount = modelSizeStats.rareCategoryCount;
             this.deadCategoryCount = modelSizeStats.deadCategoryCount;
+            this.failedCategoryCount = modelSizeStats.failedCategoryCount;
             this.categorizationStatus = modelSizeStats.categorizationStatus;
             this.timestamp = modelSizeStats.timestamp;
             this.logTime = modelSizeStats.logTime;
@@ -433,6 +445,11 @@ public class ModelSizeStats implements ToXContentObject {
             return this;
         }
 
+        public Builder setFailedCategoryCount(long failedCategoryCount) {
+            this.failedCategoryCount = failedCategoryCount;
+            return this;
+        }
+
         public Builder setCategorizationStatus(CategorizationStatus categorizationStatus) {
             Objects.requireNonNull(categorizationStatus, "[" + CATEGORIZATION_STATUS_FIELD.getPreferredName() + "] must not be null");
             this.categorizationStatus = categorizationStatus;
@@ -452,7 +469,7 @@ public class ModelSizeStats implements ToXContentObject {
         public ModelSizeStats build() {
             return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
                 totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
-                frequentCategoryCount, rareCategoryCount, deadCategoryCount, categorizationStatus, timestamp, logTime);
+                frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java

@@ -44,6 +44,7 @@ public class ModelSizeStatsTests extends AbstractXContentTestCase<ModelSizeStats
         assertEquals(0, stats.getFrequentCategoryCount());
         assertEquals(0, stats.getRareCategoryCount());
         assertEquals(0, stats.getDeadCategoryCount());
+        assertEquals(0, stats.getFailedCategoryCount());
         assertEquals(CategorizationStatus.OK, stats.getCategorizationStatus());
     }
 
@@ -109,6 +110,9 @@ public class ModelSizeStatsTests extends AbstractXContentTestCase<ModelSizeStats
         if (randomBoolean()) {
             stats.setDeadCategoryCount(randomNonNegativeLong());
         }
+        if (randomBoolean()) {
+            stats.setFailedCategoryCount(randomNonNegativeLong());
+        }
         if (randomBoolean()) {
             stats.setCategorizationStatus(randomFrom(CategorizationStatus.values()));
         }

docs/reference/cat/anomaly-detectors.asciidoc

@@ -197,6 +197,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=categorized-doc-count]
 `model.dead_category_count`, `mdcc`, `modelDeadCategoryCount`:::
 include::{docdir}/ml/ml-shared.asciidoc[tag=dead-category-count]
 
+`model.failed_category_count`, `mdcc`, `modelFailedCategoryCount`:::
+include::{docdir}/ml/ml-shared.asciidoc[tag=failed-category-count]
+
 `model.frequent_category_count`, `mfcc`, `modelFrequentCategoryCount`:::
 include::{docdir}/ml/ml-shared.asciidoc[tag=frequent-category-count]
 

docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc

@@ -214,6 +214,10 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=categorization-status]
 (long)
 include::{docdir}/ml/ml-shared.asciidoc[tag=dead-category-count]
 
+`failed_category_count`:::
+(long)
+include::{docdir}/ml/ml-shared.asciidoc[tag=failed-category-count]
+
 `frequent_category_count`:::
 (long)
 include::{docdir}/ml/ml-shared.asciidoc[tag=frequent-category-count]
@@ -406,6 +410,7 @@ The API returns the following results:
         "frequent_category_count" : 0,
         "rare_category_count" : 0,
         "dead_category_count" : 0,
+        "failed_category_count" : 0,
         "categorization_status" : "ok",
         "log_time" : 1576017596000,
         "timestamp" : 1580410800000

docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc

@@ -119,6 +119,10 @@ never be assigned again because another category's definition
 makes it a superset of the dead category.  (Dead categories are a
 side effect of the way categorization has no prior training.)
 
+`failed_category_count`:::
+(long)
+include::{docdir}/ml/ml-shared.asciidoc[tag=failed-category-count]
+
 `frequent_category_count`:::
 (long) The number of categories that match more than 1% of categorized
 documents.

docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc

@@ -94,6 +94,7 @@ When the operation is complete, you receive the following results:
       "frequent_category_count" : 0,
       "rare_category_count" : 0,
       "dead_category_count" : 0,
+      "failed_category_count" : 0,
       "categorization_status" : "ok",
       "log_time" : 1575402237000,
       "timestamp" : 1576965600000

docs/reference/ml/ml-shared.asciidoc

@@ -603,6 +603,13 @@ If `true`, the output excludes interim results. By default, interim results are
 included.
 end::exclude-interim-results[]
 
+tag::failed-category-count[]
+The number of times that categorization wanted to create a new category but
+couldn't because the job had hit its `model_memory_limit`. This count does not
+track which specific categories failed to be created. Therefore you cannot use
+this value to determine the number of unique categories that were missed.
+end::failed-category-count[]
+
 tag::feature-bag-fraction[]
 Advanced configuration option. Defines the fraction of features that will be
 used when selecting a random bag for each candidate split. 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java

@@ -50,6 +50,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
     public static final ParseField FREQUENT_CATEGORY_COUNT_FIELD = new ParseField("frequent_category_count");
     public static final ParseField RARE_CATEGORY_COUNT_FIELD = new ParseField("rare_category_count");
     public static final ParseField DEAD_CATEGORY_COUNT_FIELD = new ParseField("dead_category_count");
+    public static final ParseField FAILED_CATEGORY_COUNT_FIELD = new ParseField("failed_category_count");
     public static final ParseField CATEGORIZATION_STATUS_FIELD = new ParseField("categorization_status");
     public static final ParseField LOG_TIME_FIELD = new ParseField("log_time");
     public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp");
@@ -76,6 +77,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         parser.declareLong(Builder::setFrequentCategoryCount, FREQUENT_CATEGORY_COUNT_FIELD);
         parser.declareLong(Builder::setRareCategoryCount, RARE_CATEGORY_COUNT_FIELD);
         parser.declareLong(Builder::setDeadCategoryCount, DEAD_CATEGORY_COUNT_FIELD);
+        parser.declareLong(Builder::setFailedCategoryCount, FAILED_CATEGORY_COUNT_FIELD);
         parser.declareField(Builder::setCategorizationStatus,
                 p -> CategorizationStatus.fromString(p.text()), CATEGORIZATION_STATUS_FIELD, ValueType.STRING);
         parser.declareField(Builder::setLogTime,
@@ -154,6 +156,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
     private final long frequentCategoryCount;
     private final long rareCategoryCount;
     private final long deadCategoryCount;
+    private final long failedCategoryCount;
     private final CategorizationStatus categorizationStatus;
     private final Date timestamp;
     private final Date logTime;
@@ -161,8 +164,8 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
     private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount,
                            long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount,
                            MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount,
-                           long rareCategoryCount, long deadCategoryCount, CategorizationStatus categorizationStatus,
-                           Date timestamp, Date logTime) {
+                           long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
+                           CategorizationStatus categorizationStatus, Date timestamp, Date logTime) {
         this.jobId = jobId;
         this.modelBytes = modelBytes;
         this.modelBytesExceeded = modelBytesExceeded;
@@ -177,6 +180,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         this.frequentCategoryCount = frequentCategoryCount;
         this.rareCategoryCount = rareCategoryCount;
         this.deadCategoryCount = deadCategoryCount;
+        this.failedCategoryCount = failedCategoryCount;
         this.categorizationStatus = categorizationStatus;
         this.timestamp = timestamp;
         this.logTime = logTime;
@@ -206,6 +210,12 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
             frequentCategoryCount = in.readVLong();
             rareCategoryCount = in.readVLong();
             deadCategoryCount = in.readVLong();
+            // TODO: change in backport
+            if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                failedCategoryCount = in.readVLong();
+            } else {
+                failedCategoryCount = 0;
+            }
             categorizationStatus = CategorizationStatus.readFromStream(in);
         } else {
             categorizedDocCount = 0;
@@ -213,6 +223,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
             frequentCategoryCount = 0;
             rareCategoryCount = 0;
             deadCategoryCount = 0;
+            failedCategoryCount = 0;
             categorizationStatus = CategorizationStatus.OK;
         }
         logTime = new Date(in.readVLong());
@@ -248,6 +259,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
             out.writeVLong(frequentCategoryCount);
             out.writeVLong(rareCategoryCount);
             out.writeVLong(deadCategoryCount);
+            // TODO: change in backport
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                out.writeVLong(failedCategoryCount);
+            }
             categorizationStatus.writeTo(out);
         }
         out.writeVLong(logTime.getTime());
@@ -286,6 +301,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         builder.field(FREQUENT_CATEGORY_COUNT_FIELD.getPreferredName(), frequentCategoryCount);
         builder.field(RARE_CATEGORY_COUNT_FIELD.getPreferredName(), rareCategoryCount);
         builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount);
+        builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount);
         builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus);
         builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime());
         if (timestamp != null) {
@@ -351,6 +367,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         return deadCategoryCount;
     }
 
+    public long getFailedCategoryCount() {
+        return deadCategoryCount;
+    }
+
     public CategorizationStatus getCategorizationStatus() {
         return categorizationStatus;
     }
@@ -376,7 +396,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         // this.id excluded here as it is generated by the datastore
         return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
                 totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
-                frequentCategoryCount, rareCategoryCount, deadCategoryCount, categorizationStatus, timestamp, logTime);
+                frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
     }
 
     /**
@@ -405,6 +425,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
                 && Objects.equals(this.frequentCategoryCount, that.frequentCategoryCount)
                 && Objects.equals(this.rareCategoryCount, that.rareCategoryCount)
                 && Objects.equals(this.deadCategoryCount, that.deadCategoryCount)
+                && Objects.equals(this.failedCategoryCount, that.failedCategoryCount)
                 && Objects.equals(this.categorizationStatus, that.categorizationStatus)
                 && Objects.equals(this.timestamp, that.timestamp)
                 && Objects.equals(this.logTime, that.logTime)
@@ -427,6 +448,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         private long frequentCategoryCount;
         private long rareCategoryCount;
         private long deadCategoryCount;
+        private long failedCategoryCount;
         private CategorizationStatus categorizationStatus;
         private Date timestamp;
         private Date logTime;
@@ -453,6 +475,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
             this.frequentCategoryCount = modelSizeStats.frequentCategoryCount;
             this.rareCategoryCount = modelSizeStats.rareCategoryCount;
             this.deadCategoryCount = modelSizeStats.deadCategoryCount;
+            this.failedCategoryCount = modelSizeStats.failedCategoryCount;
             this.categorizationStatus = modelSizeStats.categorizationStatus;
             this.timestamp = modelSizeStats.timestamp;
             this.logTime = modelSizeStats.logTime;
@@ -524,6 +547,11 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
             return this;
         }
 
+        public Builder setFailedCategoryCount(long failedCategoryCount) {
+            this.failedCategoryCount = failedCategoryCount;
+            return this;
+        }
+
         public Builder setCategorizationStatus(CategorizationStatus categorizationStatus) {
             Objects.requireNonNull(categorizationStatus, "[" + CATEGORIZATION_STATUS_FIELD.getPreferredName() + "] must not be null");
             this.categorizationStatus = categorizationStatus;
@@ -543,7 +571,8 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
         public ModelSizeStats build() {
             return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
                     totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
-                    frequentCategoryCount, rareCategoryCount, deadCategoryCount, categorizationStatus, timestamp, logTime);
+                    frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp,
+                    logTime);
         }
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java

@@ -35,6 +35,7 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase<ModelSizeSt
         assertEquals(0, stats.getFrequentCategoryCount());
         assertEquals(0, stats.getRareCategoryCount());
         assertEquals(0, stats.getDeadCategoryCount());
+        assertEquals(0, stats.getFailedCategoryCount());
         assertEquals(CategorizationStatus.OK, stats.getCategorizationStatus());
     }
 
@@ -106,6 +107,9 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase<ModelSizeSt
         if (randomBoolean()) {
             stats.setDeadCategoryCount(randomNonNegativeLong());
         }
+        if (randomBoolean()) {
+            stats.setFailedCategoryCount(randomNonNegativeLong());
+        }
         if (randomBoolean()) {
             stats.setCategorizationStatus(randomFrom(CategorizationStatus.values()));
         }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java

@@ -210,6 +210,10 @@ public class RestCatJobsAction extends AbstractCatAction {
             TableColumnAttributeBuilder.builder("count of dead categories", false)
                 .setAliases("mdcc", "modelDeadCategoryCount")
                 .build());
+        table.addCell("model.failed_category_count",
+            TableColumnAttributeBuilder.builder("count of failed categories", false)
+                .setAliases("mfcc", "modelFailedCategoryCount")
+                .build());
         table.addCell("model.log_time",
             TableColumnAttributeBuilder.builder("when the model stats were gathered", false)
                 .setAliases("mlt", "modelLogTime")
@@ -364,6 +368,7 @@ public class RestCatJobsAction extends AbstractCatAction {
             table.addCell(modelSizeStats == null ? null : modelSizeStats.getFrequentCategoryCount());
             table.addCell(modelSizeStats == null ? null : modelSizeStats.getRareCategoryCount());
             table.addCell(modelSizeStats == null ? null : modelSizeStats.getDeadCategoryCount());
+            table.addCell(modelSizeStats == null ? null : modelSizeStats.getFailedCategoryCount());
             table.addCell(modelSizeStats == null ? null : modelSizeStats.getLogTime());
             table.addCell(modelSizeStats == null ? null : modelSizeStats.getTimestamp());
 

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java

@@ -103,6 +103,7 @@ public class JobStatsMonitoringDocTests extends BaseMonitoringDocTestCase<JobSta
                                                             .setFrequentCategoryCount(4)
                                                             .setRareCategoryCount(2)
                                                             .setDeadCategoryCount(1)
+                                                            .setFailedCategoryCount(3)
                                                             .setCategorizationStatus(ModelSizeStats.CategorizationStatus.WARN)
                                                             .setTimestamp(date1)
                                                             .setLogTime(date2)
@@ -168,6 +169,7 @@ public class JobStatsMonitoringDocTests extends BaseMonitoringDocTestCase<JobSta
                 + "      \"frequent_category_count\": 4,"
                 + "      \"rare_category_count\": 2,"
                 + "      \"dead_category_count\": 1,"
+                + "      \"failed_category_count\": 3,"
                 + "      \"categorization_status\": \"warn\","
                 + "      \"log_time\": 1483315322002,"
                 + "      \"timestamp\": 1483228861001"

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml

@@ -103,6 +103,7 @@ setup:
                 "frequent_category_count" : 0,
                 "rare_category_count" : 0,
                 "dead_category_count" : 0,
+                "failed_category_count" : 0,
                 "categorization_status" : "ok",
                 "log_time" : 1495808248662,
                 "timestamp" : 1495808248662

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml

@@ -297,6 +297,7 @@ setup:
             frequent_category_count : 0,
             rare_category_count : 0,
             dead_category_count : 0,
+            failed_category_count : 0,
             categorization_status : ok,
             log_time : 1495808248662
           }