|
|
@@ -60,7 +60,7 @@ import static org.hamcrest.Matchers.equalTo;
|
|
|
|
|
|
public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
|
|
|
- public void testMarkFloodStageIndicesReadOnly() {
|
|
|
+ private void doTestMarkFloodStageIndicesReadOnly(boolean testMaxHeadroom) {
|
|
|
AllocationService allocation = createAllocationService(
|
|
|
Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build()
|
|
|
);
|
|
|
@@ -128,10 +128,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
+ final long totalBytes = testMaxHeadroom ? ByteSizeValue.ofGb(10000).getBytes() : 100;
|
|
|
Map<String, DiskUsage> builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 30));
|
|
|
- builder.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(0, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(99).getBytes() : 4)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage("node2", "node2", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(250).getBytes() : 30)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage(
|
|
|
+ "frozen",
|
|
|
+ "frozen",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
final ClusterInfo initialClusterInfo = clusterInfo(builder);
|
|
|
monitor.onNewInfo(initialClusterInfo);
|
|
|
assertTrue(reroute.get()); // reroute on new nodes
|
|
|
@@ -144,9 +160,24 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
|
|
|
indices.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 5));
|
|
|
- builder.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(99).getBytes() : 4)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage("node2", "node2", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(100).getBytes() : 5)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage(
|
|
|
+ "frozen",
|
|
|
+ "frozen",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 19)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
currentTime.addAndGet(randomLongBetween(60000, 120000));
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertTrue(reroute.get());
|
|
|
@@ -195,15 +226,38 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indices.set(null);
|
|
|
reroute.set(false);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 5));
|
|
|
- builder.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(99).getBytes() : 4)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage("node2", "node2", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(100).getBytes() : 5)
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage(
|
|
|
+ "frozen",
|
|
|
+ "frozen",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 19)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertTrue(reroute.get());
|
|
|
assertEquals(Collections.singleton("test_1"), indices.get());
|
|
|
}
|
|
|
|
|
|
- public void testDoesNotSubmitRerouteTaskTooFrequently() {
|
|
|
+ public void testMarkFloodStageIndicesReadOnlyWithPercentages() {
|
|
|
+ doTestMarkFloodStageIndicesReadOnly(false);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void testMarkFloodStageIndicesReadOnlyWithMaxHeadroom() {
|
|
|
+ doTestMarkFloodStageIndicesReadOnly(true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void doTestDoesNotSubmitRerouteTaskTooFrequently(boolean testMaxHeadroom) {
|
|
|
final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
|
|
|
.nodes(DiscoveryNodes.builder().add(newNormalNode("node1")).add(newNormalNode("node2")))
|
|
|
.build();
|
|
|
@@ -227,13 +281,32 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
+ final long totalBytes = testMaxHeadroom ? ByteSizeValue.ofGb(10000).getBytes() : 100;
|
|
|
Map<String, DiskUsage> allDisksOk = new HashMap<>();
|
|
|
- allDisksOk.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 50));
|
|
|
- allDisksOk.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 50));
|
|
|
+ allDisksOk.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(500).getBytes() : 50)
|
|
|
+ );
|
|
|
+ allDisksOk.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage("node2", "node2", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(500).getBytes() : 50)
|
|
|
+ );
|
|
|
|
|
|
Map<String, DiskUsage> oneDiskAboveWatermark = new HashMap<>();
|
|
|
- oneDiskAboveWatermark.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
|
|
|
- oneDiskAboveWatermark.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 50));
|
|
|
+ oneDiskAboveWatermark.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(101, 149)).getBytes() : between(5, 9)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ oneDiskAboveWatermark.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage("node2", "node2", "/foo/bar", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(500).getBytes() : 50)
|
|
|
+ );
|
|
|
|
|
|
// should reroute when receiving info about previously-unknown nodes
|
|
|
currentTime.addAndGet(randomLongBetween(0, 120000));
|
|
|
@@ -326,7 +399,10 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
// should reroute again when one disk has reserved space that pushes it over the high watermark
|
|
|
Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpaces = Map.of(
|
|
|
new ClusterInfo.NodeAndPath("node1", "/foo/bar"),
|
|
|
- new ClusterInfo.ReservedSpace.Builder().add(new ShardId("baz", "quux", 0), between(41, 100)).build()
|
|
|
+ new ClusterInfo.ReservedSpace.Builder().add(
|
|
|
+ new ShardId("baz", "quux", 0),
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(401, 10000)).getBytes() : between(41, 100)
|
|
|
+ ).build()
|
|
|
);
|
|
|
|
|
|
currentTime.addAndGet(
|
|
|
@@ -338,10 +414,17 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
monitor.onNewInfo(clusterInfo(allDisksOk, reservedSpaces));
|
|
|
assertNotNull(listenerReference.get());
|
|
|
listenerReference.getAndSet(null).onResponse(null);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void testDoesNotSubmitRerouteTaskTooFrequentlyWithPercentages() {
|
|
|
+ doTestDoesNotSubmitRerouteTaskTooFrequently(false);
|
|
|
+ }
|
|
|
|
|
|
+ public void testDoesNotSubmitRerouteTaskTooFrequentlyWithMaxHeadroom() {
|
|
|
+ doTestDoesNotSubmitRerouteTaskTooFrequently(true);
|
|
|
}
|
|
|
|
|
|
- public void testAutoReleaseIndices() {
|
|
|
+ private void doTestAutoReleaseIndices(boolean testMaxHeadroom) {
|
|
|
AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
|
|
|
AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
|
|
|
AllocationService allocation = createAllocationService(
|
|
|
@@ -362,13 +445,15 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
);
|
|
|
assertThat(shardsWithState(clusterState.getRoutingNodes(), ShardRoutingState.STARTED).size(), equalTo(8));
|
|
|
|
|
|
+ final long totalBytes = testMaxHeadroom ? ByteSizeValue.ofGb(10000).getBytes() : 100;
|
|
|
+
|
|
|
Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpaces = new HashMap<>();
|
|
|
- final int reservedSpaceNode1 = between(0, 10);
|
|
|
+ final long reservedSpaceNode1 = testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 150)).getBytes() : between(0, 10);
|
|
|
reservedSpaces.put(
|
|
|
new ClusterInfo.NodeAndPath("node1", "/foo/bar"),
|
|
|
new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode1).build()
|
|
|
);
|
|
|
- final int reservedSpaceNode2 = between(0, 10);
|
|
|
+ final long reservedSpaceNode2 = testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 150)).getBytes() : between(0, 10);
|
|
|
reservedSpaces.put(
|
|
|
new ClusterInfo.NodeAndPath("node2", "/foo/bar"),
|
|
|
new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode2).build()
|
|
|
@@ -399,8 +484,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
Map<String, DiskUsage> builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -409,8 +512,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 90)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 90)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 9850)).getBytes() : between(5, 90)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 9850)).getBytes() : between(5, 90)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -452,22 +573,58 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
listener.onResponse(null);
|
|
|
}
|
|
|
};
|
|
|
- // When free disk on any of node1 or node2 goes below 5% flood watermark, then apply index block on indices not having the block
|
|
|
+ // When free disk on any of node1 or node2 goes below the flood watermark, then apply index block on indices not having the block
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 100)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
|
|
|
assertNull(indicesToRelease.get());
|
|
|
|
|
|
- // When free disk on node1 and node2 goes above 10% high watermark then release index block, ignoring reserved space
|
|
|
+ // When free disk on node1 and node2 goes above the high watermark then release index block, ignoring reserved space
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
assertThat(indicesToRelease.get(), contains("test_2"));
|
|
|
@@ -476,7 +633,16 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -485,10 +651,37 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 149)).getBytes() : between(5, 9)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 10000)).getBytes() : between(5, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
if (randomBoolean()) {
|
|
|
- builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node3",
|
|
|
+ new DiskUsage(
|
|
|
+ "node3",
|
|
|
+ "node3",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
}
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
@@ -498,9 +691,27 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 10000)).getBytes() : between(5, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
if (randomBoolean()) {
|
|
|
- builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node3",
|
|
|
+ new DiskUsage(
|
|
|
+ "node3",
|
|
|
+ "node3",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
}
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
@@ -510,16 +721,42 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
if (randomBoolean()) {
|
|
|
- builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node3",
|
|
|
+ new DiskUsage(
|
|
|
+ "node3",
|
|
|
+ "node3",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
}
|
|
|
monitor.onNewInfo(clusterInfo(builder));
|
|
|
assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
|
|
|
assertNull(indicesToRelease.get());
|
|
|
}
|
|
|
|
|
|
- public void testNoAutoReleaseOfIndicesOnReplacementNodes() {
|
|
|
+ public void testAutoReleaseIndicesWithPercentages() {
|
|
|
+ doTestAutoReleaseIndices(false);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void testAutoReleaseIndicesWithMaxHeadroom() {
|
|
|
+ doTestAutoReleaseIndices(true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void doTestNoAutoReleaseOfIndicesOnReplacementNodes(boolean testMaxHeadroom) {
|
|
|
AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
|
|
|
AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
|
|
|
AtomicReference<ClusterState> currentClusterState = new AtomicReference<>();
|
|
|
@@ -541,13 +778,15 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
);
|
|
|
assertThat(RoutingNodesHelper.shardsWithState(clusterState.getRoutingNodes(), ShardRoutingState.STARTED).size(), equalTo(8));
|
|
|
|
|
|
+ final long totalBytes = testMaxHeadroom ? ByteSizeValue.ofGb(10000).getBytes() : 100;
|
|
|
+
|
|
|
Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpaces = new HashMap<>();
|
|
|
- final int reservedSpaceNode1 = between(0, 10);
|
|
|
+ final long reservedSpaceNode1 = testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 150)).getBytes() : between(0, 10);
|
|
|
reservedSpaces.put(
|
|
|
new ClusterInfo.NodeAndPath("node1", "/foo/bar"),
|
|
|
new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode1).build()
|
|
|
);
|
|
|
- final int reservedSpaceNode2 = between(0, 10);
|
|
|
+ final long reservedSpaceNode2 = testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 150)).getBytes() : between(0, 10);
|
|
|
reservedSpaces.put(
|
|
|
new ClusterInfo.NodeAndPath("node2", "/foo/bar"),
|
|
|
new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode2).build()
|
|
|
@@ -580,8 +819,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
Map<String, DiskUsage> builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -590,8 +847,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 90)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 90)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 9850)).getBytes() : between(5, 90)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(100, 9850)).getBytes() : between(5, 90)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -643,12 +918,30 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
|
|
|
currentClusterState.set(clusterStateWithBlocks);
|
|
|
|
|
|
- // When free disk on any of node1 or node2 goes below 5% flood watermark, then apply index block on indices not having the block
|
|
|
+ // When free disk on any of node1 or node2 goes below the flood watermark, then apply index block on indices not having the block
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 100)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 10000)).getBytes() : between(0, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(0, 99)).getBytes() : between(0, 4)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -657,8 +950,26 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
assertNull(indicesToRelease.get());
|
|
|
@@ -676,15 +987,40 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
indicesToMarkReadOnly.set(null);
|
|
|
indicesToRelease.set(null);
|
|
|
builder = new HashMap<>();
|
|
|
- builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
|
|
|
- builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
|
|
|
+ builder.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage(
|
|
|
+ "node1",
|
|
|
+ "node1",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ builder.put(
|
|
|
+ "node2",
|
|
|
+ new DiskUsage(
|
|
|
+ "node2",
|
|
|
+ "node2",
|
|
|
+ "/foo/bar",
|
|
|
+ totalBytes,
|
|
|
+ testMaxHeadroom ? ByteSizeValue.ofGb(between(150, 10000)).getBytes() : between(10, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
monitor.onNewInfo(clusterInfo(builder, reservedSpaces));
|
|
|
assertNull(indicesToMarkReadOnly.get());
|
|
|
assertThat(indicesToRelease.get(), contains("test_2"));
|
|
|
}
|
|
|
|
|
|
- @TestLogging(value = "org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor:INFO", reason = "testing INFO/WARN logging")
|
|
|
- public void testDiskMonitorLogging() throws IllegalAccessException {
|
|
|
+ public void testNoAutoReleaseOfIndicesOnReplacementNodesWithPercentages() {
|
|
|
+ doTestNoAutoReleaseOfIndicesOnReplacementNodes(false);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void testNoAutoReleaseOfIndicesOnReplacementNodesWithMaxHeadroom() {
|
|
|
+ doTestNoAutoReleaseOfIndicesOnReplacementNodes(true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void doTestDiskMonitorLogging(boolean testHeadroom) throws IllegalAccessException {
|
|
|
final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
|
|
|
.nodes(DiscoveryNodes.builder().add(newNormalNode("node1")).add(newFrozenOnlyNode("frozen")))
|
|
|
.build();
|
|
|
@@ -725,53 +1061,62 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
+ long thousandTb = ByteSizeValue.ofTb(1000).getBytes();
|
|
|
+ long total = testHeadroom ? thousandTb : 100;
|
|
|
+
|
|
|
Map<String, DiskUsage> allDisksOk = new HashMap<>();
|
|
|
- allDisksOk.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(15, 100)));
|
|
|
- if (randomBoolean()) {
|
|
|
- allDisksOk.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(15, 100)));
|
|
|
- } else {
|
|
|
- allDisksOk.put(
|
|
|
+ allDisksOk.put("node1", new DiskUsage("node1", "node1", "/foo/bar", total, testHeadroom ? betweenGb(200, 1000) : between(15, 100)));
|
|
|
+ allDisksOk.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage(
|
|
|
"frozen",
|
|
|
- new DiskUsage(
|
|
|
- "frozen",
|
|
|
- "frozen",
|
|
|
- "/foo/bar",
|
|
|
- ByteSizeValue.ofGb(1000).getBytes(),
|
|
|
- (randomBoolean() ? ByteSizeValue.ofGb(between(20, 1000)) : ByteSizeValue.ofGb(between(20, 50))).getBytes()
|
|
|
- )
|
|
|
- );
|
|
|
- }
|
|
|
+ "frozen",
|
|
|
+ "/foo/bar",
|
|
|
+ total,
|
|
|
+ testHeadroom ? (randomBoolean() ? betweenGb(20, 1000) : betweenGb(20, 50)) : between(15, 100)
|
|
|
+ )
|
|
|
+ );
|
|
|
|
|
|
Map<String, DiskUsage> aboveLowWatermark = new HashMap<>();
|
|
|
- aboveLowWatermark.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 14)));
|
|
|
- aboveLowWatermark.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(10, 14)));
|
|
|
+ aboveLowWatermark.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", total, testHeadroom ? betweenGb(150, 199) : between(10, 14))
|
|
|
+ );
|
|
|
+ aboveLowWatermark.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage("frozen", "frozen", "/foo/bar", total, testHeadroom ? betweenGb(150, 199) : between(10, 14))
|
|
|
+ );
|
|
|
|
|
|
Map<String, DiskUsage> aboveHighWatermark = new HashMap<>();
|
|
|
- aboveHighWatermark.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
|
|
|
- aboveHighWatermark.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(5, 9)));
|
|
|
+ aboveHighWatermark.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", total, testHeadroom ? betweenGb(100, 149) : between(5, 9))
|
|
|
+ );
|
|
|
+ aboveHighWatermark.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage("frozen", "frozen", "/foo/bar", total, testHeadroom ? betweenGb(20, 99) : between(5, 9))
|
|
|
+ );
|
|
|
|
|
|
Map<String, DiskUsage> aboveFloodStageWatermark = new HashMap<>();
|
|
|
- aboveFloodStageWatermark.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
|
|
|
+ aboveFloodStageWatermark.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", total, testHeadroom ? betweenGb(0, 99) : between(0, 4))
|
|
|
+ );
|
|
|
// frozen is below flood stage, so no logging from it.
|
|
|
- aboveFloodStageWatermark.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(5, 9)));
|
|
|
+ aboveFloodStageWatermark.put(
|
|
|
+ "frozen",
|
|
|
+ new DiskUsage("frozen", "frozen", "/foo/bar", total, testHeadroom ? betweenGb(20, 99) : between(5, 9))
|
|
|
+ );
|
|
|
|
|
|
Map<String, DiskUsage> frozenAboveFloodStageWatermark = new HashMap<>();
|
|
|
// node1 is below low watermark, so no logging from it.
|
|
|
- frozenAboveFloodStageWatermark.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(15, 100)));
|
|
|
- frozenAboveFloodStageWatermark.put("frozen", new DiskUsage("frozen", "frozen", "/foo/bar", 100, between(0, 4)));
|
|
|
-
|
|
|
- Map<String, DiskUsage> frozenAboveFloodStageMaxHeadroom = new HashMap<>();
|
|
|
- // node1 is below low watermark, so no logging from it.
|
|
|
- frozenAboveFloodStageMaxHeadroom.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(15, 100)));
|
|
|
- frozenAboveFloodStageMaxHeadroom.put(
|
|
|
+ frozenAboveFloodStageWatermark.put(
|
|
|
+ "node1",
|
|
|
+ new DiskUsage("node1", "node1", "/foo/bar", total, testHeadroom ? betweenGb(200, 1000) : between(15, 100))
|
|
|
+ );
|
|
|
+ frozenAboveFloodStageWatermark.put(
|
|
|
"frozen",
|
|
|
- new DiskUsage(
|
|
|
- "frozen",
|
|
|
- "frozen",
|
|
|
- "/foo/bar",
|
|
|
- ByteSizeValue.ofGb(1000).getBytes(),
|
|
|
- ByteSizeValue.ofGb(between(0, 19)).getBytes()
|
|
|
- )
|
|
|
+ new DiskUsage("frozen", "frozen", "/foo/bar", total, testHeadroom ? betweenGb(0, 19) : between(0, 4))
|
|
|
);
|
|
|
|
|
|
advanceTime.set(true); // first check sees new nodes and triggers a reroute
|
|
|
@@ -779,17 +1124,24 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
advanceTime.set(randomBoolean()); // no new nodes so no reroute delay needed
|
|
|
assertNoLogging(monitor, allDisksOk);
|
|
|
|
|
|
+ String lowWatermarkString = testHeadroom ? "max_headroom=200gb" : "85%";
|
|
|
+ String highWatermarkString = testHeadroom ? "max_headroom=150gb" : "90%";
|
|
|
+ String floodWatermarkString = testHeadroom ? "max_headroom=100gb" : "95%";
|
|
|
+ String frozenFloodWatermarkString = testHeadroom ? "max_headroom=20gb" : "95%";
|
|
|
+
|
|
|
assertSingleInfoMessage(
|
|
|
monitor,
|
|
|
aboveLowWatermark,
|
|
|
- "low disk watermark [85%] exceeded on *node1* replicas will not be assigned to this node"
|
|
|
+ "low disk watermark [" + lowWatermarkString + "] exceeded on *node1* replicas will not be assigned to this node"
|
|
|
);
|
|
|
|
|
|
advanceTime.set(false); // will do one reroute and emit warnings, but subsequent reroutes and associated messages are delayed
|
|
|
assertSingleWarningMessage(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to continue to exceed the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
@@ -797,7 +1149,9 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to continue to exceed the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
@@ -805,15 +1159,19 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveFloodStageWatermark,
|
|
|
- "flood stage disk watermark [95%] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
+ "flood stage disk watermark ["
|
|
|
+ + floodWatermarkString
|
|
|
+ + "] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
);
|
|
|
|
|
|
- relocatingShardSizeRef.set(-5L);
|
|
|
+ relocatingShardSizeRef.set(testHeadroom ? (-1L) * ByteSizeValue.ofGb(100).getBytes() : -5L);
|
|
|
advanceTime.set(true);
|
|
|
assertSingleInfoMessage(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to be below the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
@@ -823,7 +1181,9 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
assertSingleWarningMessage(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to continue to exceed the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
@@ -831,7 +1191,9 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to continue to exceed the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
@@ -839,56 +1201,78 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
assertSingleInfoMessage(
|
|
|
monitor,
|
|
|
aboveLowWatermark,
|
|
|
- "high disk watermark [90%] no longer exceeded on *node1* but low disk watermark [85%] is still exceeded"
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] no longer exceeded on *node1* but low disk watermark ["
|
|
|
+ + lowWatermarkString
|
|
|
+ + "] is still exceeded"
|
|
|
);
|
|
|
|
|
|
advanceTime.set(true); // only log about dropping below the low disk watermark on a reroute
|
|
|
- assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [85%] no longer exceeded on *node1*");
|
|
|
+ assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [" + lowWatermarkString + "] no longer exceeded on *node1*");
|
|
|
|
|
|
advanceTime.set(randomBoolean());
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveFloodStageWatermark,
|
|
|
- "flood stage disk watermark [95%] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
+ "flood stage disk watermark ["
|
|
|
+ + floodWatermarkString
|
|
|
+ + "] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
);
|
|
|
|
|
|
- assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [85%] no longer exceeded on *node1*");
|
|
|
+ assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [" + lowWatermarkString + "] no longer exceeded on *node1*");
|
|
|
|
|
|
advanceTime.set(true);
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveHighWatermark,
|
|
|
- "high disk watermark [90%] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] exceeded on *node1* shards will be relocated away from this node* "
|
|
|
+ "the node is expected to continue to exceed the high disk watermark when these relocations are complete"
|
|
|
);
|
|
|
|
|
|
- assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [85%] no longer exceeded on *node1*");
|
|
|
+ assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [" + lowWatermarkString + "] no longer exceeded on *node1*");
|
|
|
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
aboveFloodStageWatermark,
|
|
|
- "flood stage disk watermark [95%] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
+ "flood stage disk watermark ["
|
|
|
+ + floodWatermarkString
|
|
|
+ + "] exceeded on *node1* all indices on this node will be marked read-only"
|
|
|
);
|
|
|
|
|
|
assertSingleInfoMessage(
|
|
|
monitor,
|
|
|
aboveLowWatermark,
|
|
|
- "high disk watermark [90%] no longer exceeded on *node1* but low disk watermark [85%] is still exceeded"
|
|
|
+ "high disk watermark ["
|
|
|
+ + highWatermarkString
|
|
|
+ + "] no longer exceeded on *node1* but low disk watermark ["
|
|
|
+ + lowWatermarkString
|
|
|
+ + "] is still exceeded"
|
|
|
);
|
|
|
|
|
|
- assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [85%] no longer exceeded on *node1*");
|
|
|
-
|
|
|
- assertRepeatedWarningMessages(monitor, frozenAboveFloodStageWatermark, "flood stage disk watermark [95%] exceeded on *frozen*");
|
|
|
+ assertSingleInfoMessage(monitor, allDisksOk, "low disk watermark [" + lowWatermarkString + "] no longer exceeded on *node1*");
|
|
|
|
|
|
assertRepeatedWarningMessages(
|
|
|
monitor,
|
|
|
- frozenAboveFloodStageMaxHeadroom,
|
|
|
- "flood stage disk watermark [max_headroom=20gb] exceeded on *frozen*"
|
|
|
+ frozenAboveFloodStageWatermark,
|
|
|
+ "flood stage disk watermark [" + frozenFloodWatermarkString + "] exceeded on *frozen*"
|
|
|
);
|
|
|
|
|
|
assertNoLogging(monitor, allDisksOk);
|
|
|
}
|
|
|
|
|
|
+ @TestLogging(value = "org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor:INFO", reason = "testing INFO/WARN logging")
|
|
|
+ public void testDiskMonitorLoggingWithPercentages() throws IllegalAccessException {
|
|
|
+ doTestDiskMonitorLogging(false);
|
|
|
+ }
|
|
|
+
|
|
|
+ @TestLogging(value = "org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor:INFO", reason = "testing INFO/WARN logging")
|
|
|
+ public void testDiskMonitorLoggingWithMaxHeadrooms() throws IllegalAccessException {
|
|
|
+ doTestDiskMonitorLogging(true);
|
|
|
+ }
|
|
|
+
|
|
|
private void assertNoLogging(DiskThresholdMonitor monitor, Map<String, DiskUsage> diskUsages) throws IllegalAccessException {
|
|
|
MockLogAppender mockAppender = new MockLogAppender();
|
|
|
mockAppender.start();
|
|
|
@@ -956,6 +1340,10 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
|
|
|
mockAppender.stop();
|
|
|
}
|
|
|
|
|
|
+ private static long betweenGb(int min, int max) {
|
|
|
+ return ByteSizeValue.ofGb(between(min, max)).getBytes();
|
|
|
+ }
|
|
|
+
|
|
|
private static ClusterInfo clusterInfo(Map<String, DiskUsage> diskUsages) {
|
|
|
return clusterInfo(diskUsages, Map.of());
|
|
|
}
|
Iraklis Psaroudakis