Add 'dataset' size to cat indices and cat shards (#98622)

* Add 'dataset' size to cat indices and cat shards

This adds the `dataset` computed size for the `/_cat/indices` and `/_cat/shards` APIs. This new
column is reported by default.

Resolves #95092

docs/changelog/98622.yaml

@@ -0,0 +1,6 @@
+pr: 98622
+summary: Add 'dataset' size to cat indices and cat shards
+area: CAT APIs
+type: enhancement
+issues:
+ - 95092

docs/reference/cat.asciidoc

@@ -129,9 +129,9 @@ The API returns the following response:
 
 [source,txt]
 ----
-health status index            uuid                   pri rep docs.count docs.deleted store.size pri.store.size
-yellow open   my-index-000001  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0      72171         72171
-green  open   my-index-000002  nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0        230          230
+health status index            uuid                   pri rep docs.count docs.deleted store.size pri.store.size dataset.size
+yellow open   my-index-000001  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0      72171         72171         72171
+green  open   my-index-000002  nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0        230          230            230
 ----
 // TESTRESPONSE[s/72171|230/\\d+/]
 // TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json]

docs/reference/cat/indices.asciidoc

@@ -116,9 +116,9 @@ The API returns the following response:
 
 [source,txt]
 --------------------------------------------------
-health status index            uuid                   pri rep docs.count docs.deleted store.size pri.store.size
-yellow open   my-index-000001  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb
-green  open   my-index-000002  nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0       260b           260b
+health status index            uuid                   pri rep docs.count docs.deleted store.size pri.store.size dataset.size
+yellow open   my-index-000001  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb       88.1kb
+green  open   my-index-000002  nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0       260b           260b         260b
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
 // TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json]

docs/reference/cat/shards.asciidoc

@@ -80,6 +80,9 @@ Valid columns are:
 `store`, `sto`::
 (Default) Disk space used by the shard, such as `5kb`.
 
+`dataset.size`::
+(Default) Disk space used by the shard's dataset, which may or may not be the size on disk, but includes space used by the shard on object storage. Reported as a size value such as `5kb`.
+
 `ip`::
 (Default) IP address of the node, such as `127.0.1.1`.
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml

@@ -9,6 +9,9 @@
                /^$/
 ---
 "Test cat indices output":
+  - skip:
+      version: " - 8.10.99"
+      reason:  dataset size was added in 8.11.0
 
   - do:
       indices.create:
@@ -32,6 +35,20 @@
                   0      \s+
                   0      \s+
                   (\d+|\d+[.]\d+)(kb|b) \s+
+                  (\d+|\d+[.]\d+)(kb|b) \s+
+                  (\d+|\d+[.]\d+)(kb|b) \s*
+                )
+                $/
+
+  - do:
+      cat.indices:
+        index: index1
+        v: false
+        h: [index, dataset.size]
+
+  - match:
+      $body: |
+               /^(index1 \s+
                   (\d+|\d+[.]\d+)(kb|b) \s*
                 )
                 $/

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml

@@ -1,8 +1,8 @@
 ---
 "Test cat indices output for hidden index":
   - skip:
-      version: "- 7.6.99"
-      reason: "hidden indices were added in 7.7.0"
+      version: "- 8.10.99"
+      reason: "dataset size was added in 8.11.0"
   - do:
       indices.create:
         index: index1
@@ -34,6 +34,7 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
@@ -41,8 +42,8 @@
 ---
 "Test cat indices output for dot-hidden index and dot-prefixed pattern":
   - skip:
-      version: "- 7.6.99"
-      reason: "hidden indices were added in 7.7.0"
+      version: "- 8.10.99"
+      reason: "dataset size was added in 8.11.0"
   - do:
       indices.create:
         index: .index1
@@ -72,6 +73,7 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
@@ -79,8 +81,8 @@
 ---
 "Test cat indices output with a hidden index with a visible alias":
   - skip:
-      version: "- 7.6.99"
-      reason: "hidden indices were added in 7.7.0"
+      version: "- 8.10.99"
+      reason: "dataset size was added in 8.11.0"
 
   - do:
       indices.create:
@@ -118,6 +120,7 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
@@ -136,14 +139,15 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
 ---
 "Test cat indices output with a hidden index with a hidden alias":
   - skip:
-      version: "- 7.6.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
+      version: "- 8.10.99"
+      reason: "dataset size was added in 8.11.0"
 
   - do:
       indices.create:
@@ -180,6 +184,7 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
@@ -198,14 +203,15 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
          )
          $/
 ---
 "Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
   - skip:
-      version: "- 7.6.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
+      version: "- 8.10.99"
+      reason: "dataset size was added in 8.11.0"
 
   - do:
       indices.create:
@@ -239,6 +245,7 @@
            0      \s+
            0      \s+
            (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
            (\d+|\d+[.]\d+)(kb|b) \s*
            .*           # Accounts for system indices that get automatically generated
          )

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -1,8 +1,8 @@
 ---
 "Help":
   - skip:
-      version: " - 8.9.99"
-      reason:  dense vector stats were added in 8.10.0
+      version: " - 8.10.99"
+      reason:  dataset size was added in 8.11.0
   - do:
       cat.shards:
         help: true
@@ -15,6 +15,7 @@
                     state                            .+   \n
                     docs                             .+   \n
                     store                            .+   \n
+                    dataset                          .+   \n
                     ip                               .+   \n
                     id                               .+   \n
                     node                             .+   \n
@@ -89,6 +90,9 @@
                $/
 ---
 "Test cat shards output":
+  - skip:
+      version: " - 8.10.99"
+      reason:  dataset size was added in 8.11.0
 
   - do:
       cat.shards:
@@ -110,7 +114,7 @@
 
   - match:
       $body: |
-               /^(index1 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+)  \n?){10}$/
+               /^(index1 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+)  \n?){10}$/
 
   - do:
       indices.create:
@@ -125,14 +129,14 @@
         index: i*
   - match:
       $body: |
-               /^(index(1|2) \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){15}$/
+               /^(index(1|2) \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){15}$/
 
   - do:
       cat.shards:
         index: index2
   - match:
       $body: |
-               /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/
+               /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/
 
 ---
 "Test cat shards using wildcards":
@@ -183,6 +187,10 @@
 
 ---
 "Test cat shards sort":
+  - skip:
+      version: " - 8.10.99"
+      reason:  dataset size was added in 8.11.0
+
   - do:
       indices.create:
         index: foo
@@ -218,6 +226,17 @@
                bar \s+ 1\n
             $/
 
+  - do:
+      cat.shards:
+        h: [index, dataset]
+        s: [docs]
+        index: "foo,bar"
+
+  - match:
+      $body: |
+            /^ foo \s+ (\d+|\d+[.]\d+)(kb|b)\n
+               bar \s+ (\d+|\d+[.]\d+)(kb|b)\n
+            $/
 
 ---
 "Test cat shards with hidden indices":

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.rest.RestRequest;
@@ -170,6 +171,12 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas");
         table.addCell("pri.store.size", "text-align:right;desc:store size of primaries");
 
+        if (request.getRestApiVersion() == RestApiVersion.V_7) {
+            table.addCell("dataset.size", "default:false;text-align:right;desc:total size of dataset");
+        } else {
+            table.addCell("dataset.size", "text-align:right;desc:total size of dataset");
+        }
+
         table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion");
         table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion");
 
@@ -585,6 +592,7 @@ public class RestIndicesAction extends AbstractCatAction {
 
             table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size());
             table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size());
+            table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().totalDataSetSize());
 
             table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize());
             table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize());

server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.bulk.stats.BulkStats;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -116,6 +117,12 @@ public class RestShardsAction extends AbstractCatAction {
             .addCell("state", "default:true;alias:st;desc:shard state")
             .addCell("docs", "alias:d,dc;text-align:right;desc:number of docs in shard")
             .addCell("store", "alias:sto;text-align:right;desc:store size of shard (how much disk it uses)")
+            .addCell(
+                "dataset",
+                request.getRestApiVersion() == RestApiVersion.V_7
+                    ? "default:false;text-align:right;desc:total size of dataset"
+                    : "text-align:right;desc:total size of dataset"
+            )
             .addCell("ip", "default:true;desc:ip of node where it lives")
             .addCell("id", "default:false;desc:unique id of node where it lives")
             .addCell("node", "default:true;alias:n;desc:name of node where it lives");
@@ -287,6 +294,7 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(shard.state());
             table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
             table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
+            table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getTotalDataSetSize));
             if (shard.assignedToNode()) {
                 String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
                 String nodeId = shard.currentNodeId();

server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java

@@ -95,9 +95,10 @@ public class RestShardsActionTests extends ESTestCase {
         assertThat(headers.get(3).value, equalTo("state"));
         assertThat(headers.get(4).value, equalTo("docs"));
         assertThat(headers.get(5).value, equalTo("store"));
-        assertThat(headers.get(6).value, equalTo("ip"));
-        assertThat(headers.get(7).value, equalTo("id"));
-        assertThat(headers.get(8).value, equalTo("node"));
+        assertThat(headers.get(6).value, equalTo("dataset"));
+        assertThat(headers.get(7).value, equalTo("ip"));
+        assertThat(headers.get(8).value, equalTo("id"));
+        assertThat(headers.get(9).value, equalTo("node"));
 
         final List<List<Table.Cell>> rows = table.getRows();
         assertThat(rows.size(), equalTo(numShards));
@@ -110,10 +111,10 @@ public class RestShardsActionTests extends ESTestCase {
             assertThat(row.get(1).value, equalTo(shardRouting.getId()));
             assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
             assertThat(row.get(3).value, equalTo(shardRouting.state()));
-            assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
-            assertThat(row.get(7).value, equalTo(localNode.getId()));
-            assertThat(row.get(69).value, equalTo(shardStats.getDataPath()));
-            assertThat(row.get(70).value, equalTo(shardStats.getStatePath()));
+            assertThat(row.get(7).value, equalTo(localNode.getHostAddress()));
+            assertThat(row.get(8).value, equalTo(localNode.getId()));
+            assertThat(row.get(70).value, equalTo(shardStats.getDataPath()));
+            assertThat(row.get(71).value, equalTo(shardStats.getStatePath()));
         }
     }
 }

x-pack/plugin/build.gradle

@@ -170,6 +170,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("service_accounts/10_basic/Test get service accounts", "new service accounts are added")
   task.skipTest("spatial/70_script_doc_values/diagonal length", "precision changed in 8.4.0")
   task.skipTest("spatial/70_script_doc_values/geoshape value", "error message changed in 8.9.0")
+  task.skipTest("security/authz/14_cat_indices/Test empty request while single authorized index", "not supported for compatibility")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml

@@ -88,6 +88,9 @@ teardown:
 
 ---
 "Test empty request while single authorized index":
+  - skip:
+      version: " - 8.10.99"
+      reason: cat indices was changed in 8.11.0
 
   - do:
       indices.create:
@@ -112,6 +115,7 @@ teardown:
                   0      \s+
                   0      \s+
                   (\d+|\d+[.]\d+)(kb|b) \s+
+                  (\d+|\d+[.]\d+)(kb|b) \s+
                   (\d+|\d+[.]\d+)(kb|b) \s*
                 )
                 $/