Allow dynamically changing the use_real_memory setting (#78288)

Allow changing use_real_memory setting for parent circuit breaker dynamically. 
Until now this setting could only be modified the .yml file followed by node restart.

server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java

@@ -403,4 +403,42 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
             assertEquals(ex.getByteLimit(), inFlightRequestsLimit.getBytes());
         }
     }
+
+    // Test the default value of TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING should
+    // change with the update of USE_REAL_MEMORY_USAGE_SETTING
+    // but should stay the same if it is overridden
+    public void testDynamicUseRealMemory() {
+        final Client client = client();
+        // use_real_memory is set to false for internalTestCluster
+        checkLimitSize(client, 0.7);
+        String useRealMemoryUsageSetting = HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey();
+        String totalCircuitBreakerLimitSettingKey = HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey();
+
+        Settings settings = Settings.builder().put(useRealMemoryUsageSetting, true).build();
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings).get();
+        checkLimitSize(client, 0.95);
+
+        Settings setTrueAndMemorySettings = Settings.builder()
+            .put(totalCircuitBreakerLimitSettingKey, "80%")
+            .put(useRealMemoryUsageSetting, true)
+            .build();
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(setTrueAndMemorySettings).get();
+        checkLimitSize(client, 0.8);
+
+        Settings setFalseSettings = Settings.builder().put(useRealMemoryUsageSetting, false).build();
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(setFalseSettings).get();
+        checkLimitSize(client, 0.8);
+
+        Settings resetSettings = Settings.builder().putNull(totalCircuitBreakerLimitSettingKey).putNull(useRealMemoryUsageSetting).build();
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(resetSettings).get();
+    }
+
+    private void checkLimitSize(Client client, double limitRatio) {
+        NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).setJvm(true).get();
+        for (NodeStats node : stats.getNodes()) {
+            long heapSize = node.getJvm().getMem().getHeapCommitted().getBytes();
+            long limitSize = node.getBreaker().getStats(CircuitBreaker.PARENT).getLimit();
+            assertEquals((long) (heapSize * limitRatio), limitSize);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java

@@ -60,6 +60,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
     public static final Setting<Boolean> USE_REAL_MEMORY_USAGE_SETTING = Setting.boolSetting(
         "indices.breaker.total.use_real_memory",
         true,
+        Property.Dynamic,
         Property.NodeScope
     );
 
@@ -136,13 +137,14 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         Property.NodeScope
     );
 
-    private final boolean trackRealMemoryUsage;
+    private volatile boolean trackRealMemoryUsage;
     private volatile BreakerSettings parentSettings;
 
     // Tripped count for when redistribution was attempted but wasn't successful
     private final AtomicLong parentTripCount = new AtomicLong(0);
 
-    private final OverLimitStrategy overLimitStrategy;
+    private final Function<Boolean, OverLimitStrategy> overLimitStrategyFactory;
+    private volatile OverLimitStrategy overLimitStrategy;
 
     public HierarchyCircuitBreakerService(Settings settings, List<BreakerSettings> customBreakers, ClusterSettings clusterSettings) {
         this(settings, customBreakers, clusterSettings, HierarchyCircuitBreakerService::createOverLimitStrategy);
@@ -240,7 +242,9 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
             (name, updatedValues) -> updateCircuitBreakerSettings(name, updatedValues.v1(), updatedValues.v2()),
             (s, t) -> {}
         );
+        clusterSettings.addSettingsUpdateConsumer(USE_REAL_MEMORY_USAGE_SETTING, this::updateUseRealMemorySetting);
 
+        this.overLimitStrategyFactory = overLimitStrategyFactory;
         this.overLimitStrategy = overLimitStrategyFactory.apply(this.trackRealMemoryUsage);
     }
 
@@ -267,6 +271,20 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.getBytes(), 1.0, CircuitBreaker.Type.PARENT, null);
     }
 
+    public void updateUseRealMemorySetting(boolean trackRealMemoryUsage) {
+        this.trackRealMemoryUsage = trackRealMemoryUsage;
+
+        this.overLimitStrategy = overLimitStrategyFactory.apply(this.trackRealMemoryUsage);
+    }
+
+    public boolean isTrackRealMemoryUsage() {
+        return trackRealMemoryUsage;
+    }
+
+    public OverLimitStrategy getOverLimitStrategy() {
+        return overLimitStrategy;
+    }
+
     /**
      * Validate that child settings are valid
      */

server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java

@@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.MemorySizeValue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
@@ -817,4 +818,63 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase {
     private static long mb(long size) {
         return new ByteSizeValue(size, ByteSizeUnit.MB).getBytes();
     }
+
+    public void testUpdatingUseRealMemory() {
+        try (
+            HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(
+                Settings.EMPTY,
+                Collections.emptyList(),
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+            )
+        ) {
+            // use real memory default true
+            assertTrue(service.isTrackRealMemoryUsage());
+            assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class));
+
+            // update use_real_memory to false
+            service.updateUseRealMemorySetting(false);
+            assertFalse(service.isTrackRealMemoryUsage());
+            assertThat(service.getOverLimitStrategy(), not(instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)));
+
+            // update use_real_memory to true
+            service.updateUseRealMemorySetting(true);
+            assertTrue(service.isTrackRealMemoryUsage());
+            assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class));
+        }
+    }
+
+    public void testApplySettingForUpdatingUseRealMemory() {
+        String useRealMemoryUsageSetting = HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey();
+        String totalCircuitBreakerLimitSetting = HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey();
+        Settings initialSettings = Settings.builder().put(useRealMemoryUsageSetting, "true").build();
+        ClusterSettings clusterSettings = new ClusterSettings(initialSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        try (
+            HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(
+                Settings.EMPTY,
+                Collections.emptyList(),
+                clusterSettings
+            )
+        ) {
+            // total.limit defaults to 95% of the JVM heap if use_real_memory is true
+            assertEquals(
+                MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(),
+                service.getParentLimit()
+            );
+
+            // total.limit defaults to 70% of the JVM heap if use_real_memory set to false
+            clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, false).build());
+            assertEquals(
+                MemorySizeValue.parseBytesSizeValueOrHeapRatio("70%", totalCircuitBreakerLimitSetting).getBytes(),
+                service.getParentLimit()
+            );
+
+            // total.limit defaults to 70% of the JVM heap if use_real_memory set to true
+            clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, true).build());
+            assertEquals(
+                MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(),
+                service.getParentLimit()
+            );
+        }
+    }
 }