|
@@ -10,6 +10,7 @@ package org.elasticsearch.xpack.esql.ccq;
|
|
|
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
|
|
|
|
|
|
import org.apache.http.HttpHost;
|
|
|
+import org.elasticsearch.Version;
|
|
|
import org.elasticsearch.client.Request;
|
|
|
import org.elasticsearch.client.RestClient;
|
|
|
import org.elasticsearch.common.Strings;
|
|
@@ -29,7 +30,6 @@ import org.junit.rules.TestRule;
|
|
|
import java.io.IOException;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
-import java.util.Optional;
|
|
|
import java.util.Set;
|
|
|
import java.util.stream.Collectors;
|
|
|
import java.util.stream.IntStream;
|
|
@@ -127,10 +127,12 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
}
|
|
|
|
|
|
private Map<String, Object> run(String query, boolean includeCCSMetadata) throws IOException {
|
|
|
- Map<String, Object> resp = runEsql(
|
|
|
- new RestEsqlTestCase.RequestObjectBuilder().query(query).includeCCSMetadata(includeCCSMetadata).build()
|
|
|
- );
|
|
|
- logger.info("--> query {} response {}", query, resp);
|
|
|
+ var queryBuilder = new RestEsqlTestCase.RequestObjectBuilder().query(query);
|
|
|
+ if (includeCCSMetadata) {
|
|
|
+ queryBuilder.includeCCSMetadata(true);
|
|
|
+ }
|
|
|
+ Map<String, Object> resp = runEsql(queryBuilder.build());
|
|
|
+ logger.info("--> query {} response {}", queryBuilder, resp);
|
|
|
return resp;
|
|
|
}
|
|
|
|
|
@@ -156,7 +158,7 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
|
|
|
public void testCount() throws Exception {
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata);
|
|
|
var columns = List.of(Map.of("name", "c", "type", "long"));
|
|
|
var values = List.of(List.of(localDocs.size() + remoteDocs.size()));
|
|
@@ -165,13 +167,16 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, false);
|
|
|
}
|
|
|
}
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run("FROM *:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata);
|
|
|
var columns = List.of(Map.of("name", "c", "type", "long"));
|
|
|
var values = List.of(List.of(remoteDocs.size()));
|
|
@@ -180,7 +185,10 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, true);
|
|
|
}
|
|
@@ -189,7 +197,7 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
|
|
|
public void testUngroupedAggs() throws Exception {
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data)", includeCCSMetadata);
|
|
|
var columns = List.of(Map.of("name", "total", "type", "long"));
|
|
|
long sum = Stream.concat(localDocs.stream(), remoteDocs.stream()).mapToLong(d -> d.data).sum();
|
|
@@ -200,13 +208,16 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, false);
|
|
|
}
|
|
|
}
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data)", includeCCSMetadata);
|
|
|
var columns = List.of(Map.of("name", "total", "type", "long"));
|
|
|
long sum = remoteDocs.stream().mapToLong(d -> d.data).sum();
|
|
@@ -216,12 +227,16 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, true);
|
|
|
}
|
|
|
}
|
|
|
{
|
|
|
+ assumeTrue("requires ccs metadata", ccsMetadataAvailable());
|
|
|
Map<String, Object> result = runWithColumnarAndIncludeCCSMetadata("FROM *:test-remote-index | STATS total = SUM(data)");
|
|
|
var columns = List.of(Map.of("name", "total", "type", "long"));
|
|
|
long sum = remoteDocs.stream().mapToLong(d -> d.data).sum();
|
|
@@ -293,7 +308,7 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
|
|
|
public void testGroupedAggs() throws Exception {
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run(
|
|
|
"FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color",
|
|
|
includeCCSMetadata
|
|
@@ -311,13 +326,16 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, false);
|
|
|
}
|
|
|
}
|
|
|
{
|
|
|
- boolean includeCCSMetadata = randomBoolean();
|
|
|
+ boolean includeCCSMetadata = includeCCSMetadata();
|
|
|
Map<String, Object> result = run(
|
|
|
"FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color",
|
|
|
includeCCSMetadata
|
|
@@ -336,29 +354,57 @@ public class MultiClustersIT extends ESRestTestCase {
|
|
|
if (includeCCSMetadata) {
|
|
|
mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
|
|
|
}
|
|
|
- assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
if (includeCCSMetadata) {
|
|
|
assertClusterDetailsMap(result, true);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ public void testIndexPattern() throws Exception {
|
|
|
+ {
|
|
|
+ String indexPattern = randomFrom(
|
|
|
+ "test-local-index,*:test-remote-index",
|
|
|
+ "test-local-index,*:test-remote-*",
|
|
|
+ "test-local-index,*:test-*",
|
|
|
+ "test-*,*:test-remote-index"
|
|
|
+ );
|
|
|
+ Map<String, Object> result = run("FROM " + indexPattern + " | STATS c = COUNT(*)", false);
|
|
|
+ var columns = List.of(Map.of("name", "c", "type", "long"));
|
|
|
+ var values = List.of(List.of(localDocs.size() + remoteDocs.size()));
|
|
|
+ MapMatcher mapMatcher = matchesMap();
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
+ }
|
|
|
+ {
|
|
|
+ String indexPattern = randomFrom("*:test-remote-index", "*:test-remote-*", "*:test-*");
|
|
|
+ Map<String, Object> result = run("FROM " + indexPattern + " | STATS c = COUNT(*)", false);
|
|
|
+ var columns = List.of(Map.of("name", "c", "type", "long"));
|
|
|
+ var values = List.of(List.of(remoteDocs.size()));
|
|
|
+
|
|
|
+ MapMatcher mapMatcher = matchesMap();
|
|
|
+ if (ccsMetadataAvailable()) {
|
|
|
+ mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
|
|
|
+ }
|
|
|
+ assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
private RestClient remoteClusterClient() throws IOException {
|
|
|
var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses());
|
|
|
return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
|
|
|
}
|
|
|
|
|
|
- private TestFeatureService remoteFeaturesService() throws IOException {
|
|
|
- if (remoteFeaturesService == null) {
|
|
|
- try (RestClient remoteClient = remoteClusterClient()) {
|
|
|
- var remoteNodeVersions = readVersionsFromNodesInfo(remoteClient);
|
|
|
- var semanticNodeVersions = remoteNodeVersions.stream()
|
|
|
- .map(ESRestTestCase::parseLegacyVersion)
|
|
|
- .flatMap(Optional::stream)
|
|
|
- .collect(Collectors.toSet());
|
|
|
- remoteFeaturesService = createTestFeatureService(getClusterStateFeatures(remoteClient), semanticNodeVersions);
|
|
|
- }
|
|
|
- }
|
|
|
- return remoteFeaturesService;
|
|
|
+ private static boolean ccsMetadataAvailable() {
|
|
|
+ return Clusters.localClusterVersion().onOrAfter(Version.V_8_16_0);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean includeCCSMetadata() {
|
|
|
+ return ccsMetadataAvailable() && randomBoolean();
|
|
|
}
|
|
|
}
|