|
@@ -29,7 +29,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper;
|
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
|
import org.elasticsearch.index.query.RangeQueryBuilder;
|
|
|
import org.elasticsearch.index.shard.IndexLongFieldRange;
|
|
|
-import org.elasticsearch.indices.DateFieldRangeInfo;
|
|
|
import org.elasticsearch.indices.IndicesService;
|
|
|
import org.elasticsearch.indices.recovery.RecoveryState;
|
|
|
import org.elasticsearch.plugins.Plugin;
|
|
@@ -101,11 +100,11 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
- createIndexWithTimestampAndEventIngested(indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
|
|
|
+ createIndexWithTimestamp(indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
|
|
|
|
|
|
final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
- createIndexWithTimestampAndEventIngested(
|
|
|
+ createIndexWithTimestamp(
|
|
|
indexWithinSearchRange,
|
|
|
indexWithinSearchRangeShardCount,
|
|
|
Settings.builder()
|
|
@@ -118,10 +117,11 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
// Either add data outside of the range, or documents that don't have timestamp data
|
|
|
final boolean indexDataWithTimestamp = randomBoolean();
|
|
|
// Add enough documents to have non-metadata segment files in all shards,
|
|
|
- // otherwise the mount operation might go through as the read won't be blocked
|
|
|
+ // otherwise the mount operation might go through as the read won't be
|
|
|
+ // blocked
|
|
|
final int numberOfDocsInIndexOutsideSearchRange = between(350, 1000);
|
|
|
if (indexDataWithTimestamp) {
|
|
|
- indexDocumentsWithTimestampAndEventIngestedDates(
|
|
|
+ indexDocumentsWithTimestampWithinDate(
|
|
|
indexOutsideSearchRange,
|
|
|
numberOfDocsInIndexOutsideSearchRange,
|
|
|
TIMESTAMP_TEMPLATE_OUTSIDE_RANGE
|
|
@@ -132,7 +132,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
// Index enough documents to ensure that all shards have at least some documents
|
|
|
int numDocsWithinRange = between(100, 1000);
|
|
|
- indexDocumentsWithTimestampAndEventIngestedDates(indexWithinSearchRange, numDocsWithinRange, TIMESTAMP_TEMPLATE_WITHIN_RANGE);
|
|
|
+ indexDocumentsWithTimestampWithinDate(indexWithinSearchRange, numDocsWithinRange, TIMESTAMP_TEMPLATE_WITHIN_RANGE);
|
|
|
|
|
|
final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
createRepository(repositoryName, "mock");
|
|
@@ -166,10 +166,9 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
- assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
|
|
|
- DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
|
|
|
- assertThat(timestampFieldTypeInfo, nullValue());
|
|
|
+ DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
|
|
|
+ assertThat(timestampFieldType, nullValue());
|
|
|
|
|
|
final boolean includeIndexCoveringSearchRangeInSearchRequest = randomBoolean();
|
|
|
List<String> indicesToSearch = new ArrayList<>();
|
|
@@ -177,9 +176,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
indicesToSearch.add(indexWithinSearchRange);
|
|
|
}
|
|
|
indicesToSearch.add(searchableSnapshotIndexOutsideSearchRange);
|
|
|
-
|
|
|
- String timeField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
|
|
|
- RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timeField)
|
|
|
+ RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
|
|
|
.from("2020-11-28T00:00:00.000000000Z", true)
|
|
|
.to("2020-11-29T00:00:00.000000000Z");
|
|
|
|
|
@@ -253,44 +250,20 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
ensureGreen(searchableSnapshotIndexOutsideSearchRange);
|
|
|
|
|
|
final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
-
|
|
|
- // check that @timestamp and 'event.ingested' are now in cluster state
|
|
|
final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
|
|
|
+ final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
|
|
|
+ assertThat(dateFieldType, notNullValue());
|
|
|
+ final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
|
|
|
assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
|
|
|
- final IndexLongFieldRange updatedEventIngestedRange = updatedIndexMetadata.getEventIngestedRange();
|
|
|
- assertThat(updatedEventIngestedRange.isComplete(), equalTo(true));
|
|
|
-
|
|
|
- timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
|
|
|
- final DateFieldMapper.DateFieldType timestampDataFieldType = timestampFieldTypeInfo.getTimestampFieldType();
|
|
|
- assertThat(timestampDataFieldType, notNullValue());
|
|
|
- final DateFieldMapper.DateFieldType eventIngestedDataFieldType = timestampFieldTypeInfo.getEventIngestedFieldType();
|
|
|
- assertThat(eventIngestedDataFieldType, notNullValue());
|
|
|
-
|
|
|
- final DateFieldMapper.Resolution timestampResolution = timestampDataFieldType.resolution();
|
|
|
- final DateFieldMapper.Resolution eventIngestedResolution = eventIngestedDataFieldType.resolution();
|
|
|
if (indexDataWithTimestamp) {
|
|
|
assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
assertThat(
|
|
|
updatedTimestampMillisRange.getMin(),
|
|
|
- greaterThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedTimestampMillisRange.getMax(),
|
|
|
- lessThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
|
|
|
- );
|
|
|
-
|
|
|
- assertThat(updatedEventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedRange.getMin(),
|
|
|
- greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedRange.getMax(),
|
|
|
- lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
|
|
|
+ greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
|
|
|
);
|
|
|
+ assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
|
|
|
} else {
|
|
|
assertThat(updatedTimestampMillisRange, sameInstance(IndexLongFieldRange.EMPTY));
|
|
|
- assertThat(updatedEventIngestedRange, sameInstance(IndexLongFieldRange.EMPTY));
|
|
|
}
|
|
|
|
|
|
// Stop the node holding the searchable snapshots, and since we defined
|
|
@@ -410,171 +383,6 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Test shard skipping when only 'event.ingested' is in the index and cluster state.
|
|
|
- */
|
|
|
- public void testEventIngestedRangeInSearchAgainstSearchableSnapshotShards() throws Exception {
|
|
|
- internalCluster().startMasterOnlyNode();
|
|
|
- internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
|
|
|
- final String dataNodeHoldingRegularIndex = internalCluster().startDataOnlyNode();
|
|
|
- final String dataNodeHoldingSearchableSnapshot = internalCluster().startDataOnlyNode();
|
|
|
- final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, dataNodeHoldingSearchableSnapshot);
|
|
|
-
|
|
|
- final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
- final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
-
|
|
|
- final String timestampField = IndexMetadata.EVENT_INGESTED_FIELD_NAME;
|
|
|
-
|
|
|
- createIndexWithOnlyOneTimestampField(timestampField, indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
|
|
|
-
|
|
|
- final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
- final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
- createIndexWithOnlyOneTimestampField(
|
|
|
- timestampField,
|
|
|
- indexWithinSearchRange,
|
|
|
- indexWithinSearchRangeShardCount,
|
|
|
- Settings.builder()
|
|
|
- .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), dataNodeHoldingRegularIndex)
|
|
|
- .build()
|
|
|
- );
|
|
|
-
|
|
|
- final int totalShards = indexOutsideSearchRangeShardCount + indexWithinSearchRangeShardCount;
|
|
|
-
|
|
|
- // Add enough documents to have non-metadata segment files in all shards,
|
|
|
- // otherwise the mount operation might go through as the read won't be blocked
|
|
|
- final int numberOfDocsInIndexOutsideSearchRange = between(350, 1000);
|
|
|
-
|
|
|
- indexDocumentsWithOnlyOneTimestampField(
|
|
|
- timestampField,
|
|
|
- indexOutsideSearchRange,
|
|
|
- numberOfDocsInIndexOutsideSearchRange,
|
|
|
- TIMESTAMP_TEMPLATE_OUTSIDE_RANGE
|
|
|
- );
|
|
|
-
|
|
|
- // Index enough documents to ensure that all shards have at least some documents
|
|
|
- int numDocsWithinRange = between(100, 1000);
|
|
|
- indexDocumentsWithOnlyOneTimestampField(
|
|
|
- timestampField,
|
|
|
- indexWithinSearchRange,
|
|
|
- numDocsWithinRange,
|
|
|
- TIMESTAMP_TEMPLATE_WITHIN_RANGE
|
|
|
- );
|
|
|
-
|
|
|
- final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
- createRepository(repositoryName, "mock");
|
|
|
-
|
|
|
- final SnapshotId snapshotId = createSnapshot(repositoryName, "snapshot-1", List.of(indexOutsideSearchRange)).snapshotId();
|
|
|
- assertAcked(indicesAdmin().prepareDelete(indexOutsideSearchRange));
|
|
|
-
|
|
|
- final String searchableSnapshotIndexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
-
|
|
|
- // Block the repository for the node holding the searchable snapshot shards
|
|
|
- // to delay its restore
|
|
|
- blockDataNode(repositoryName, dataNodeHoldingSearchableSnapshot);
|
|
|
-
|
|
|
- // Force the searchable snapshot to be allocated in a particular node
|
|
|
- Settings restoredIndexSettings = Settings.builder()
|
|
|
- .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), dataNodeHoldingSearchableSnapshot)
|
|
|
- .build();
|
|
|
-
|
|
|
- final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
|
|
|
- TEST_REQUEST_TIMEOUT,
|
|
|
- searchableSnapshotIndexOutsideSearchRange,
|
|
|
- repositoryName,
|
|
|
- snapshotId.getName(),
|
|
|
- indexOutsideSearchRange,
|
|
|
- restoredIndexSettings,
|
|
|
- Strings.EMPTY_ARRAY,
|
|
|
- false,
|
|
|
- randomFrom(MountSearchableSnapshotRequest.Storage.values())
|
|
|
- );
|
|
|
- client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).actionGet();
|
|
|
-
|
|
|
- final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
- assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
- assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
-
|
|
|
- // Allow the searchable snapshots to be finally mounted
|
|
|
- unblockNode(repositoryName, dataNodeHoldingSearchableSnapshot);
|
|
|
- waitUntilRecoveryIsDone(searchableSnapshotIndexOutsideSearchRange);
|
|
|
- ensureGreen(searchableSnapshotIndexOutsideSearchRange);
|
|
|
-
|
|
|
- IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
- IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
|
|
|
- IndexLongFieldRange updatedEventIngestedMillisRange = updatedIndexMetadata.getEventIngestedRange();
|
|
|
-
|
|
|
- // @timestamp range should be null since it was not included in the index or indexed docs
|
|
|
- assertThat(updatedTimestampMillisRange, equalTo(IndexLongFieldRange.UNKNOWN));
|
|
|
- assertThat(updatedEventIngestedMillisRange, not(equalTo(IndexLongFieldRange.UNKNOWN)));
|
|
|
-
|
|
|
- DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
|
|
|
-
|
|
|
- DateFieldMapper.DateFieldType timestampDataFieldType = timestampFieldTypeInfo.getTimestampFieldType();
|
|
|
- assertThat(timestampDataFieldType, nullValue());
|
|
|
-
|
|
|
- DateFieldMapper.DateFieldType eventIngestedFieldType = timestampFieldTypeInfo.getEventIngestedFieldType();
|
|
|
- assertThat(eventIngestedFieldType, notNullValue());
|
|
|
-
|
|
|
- DateFieldMapper.Resolution eventIngestedResolution = eventIngestedFieldType.resolution();
|
|
|
- assertThat(updatedEventIngestedMillisRange.isComplete(), equalTo(true));
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedMillisRange.getMin(),
|
|
|
- greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedMillisRange.getMax(),
|
|
|
- lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
|
|
|
- );
|
|
|
-
|
|
|
- // now do a search against event.ingested
|
|
|
- List<String> indicesToSearch = new ArrayList<>();
|
|
|
- indicesToSearch.add(indexWithinSearchRange);
|
|
|
- indicesToSearch.add(searchableSnapshotIndexOutsideSearchRange);
|
|
|
-
|
|
|
- {
|
|
|
- RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
|
|
|
- .from("2020-11-28T00:00:00.000000000Z", true)
|
|
|
- .to("2020-11-29T00:00:00.000000000Z");
|
|
|
-
|
|
|
- SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
|
|
|
- .source(new SearchSourceBuilder().query(rangeQuery));
|
|
|
-
|
|
|
- assertResponse(client().search(request), searchResponse -> {
|
|
|
- // All the regular index searches succeeded
|
|
|
- assertThat(searchResponse.getSuccessfulShards(), equalTo(totalShards));
|
|
|
- assertThat(searchResponse.getFailedShards(), equalTo(0));
|
|
|
- // All the searchable snapshots shards were skipped
|
|
|
- assertThat(searchResponse.getSkippedShards(), equalTo(indexOutsideSearchRangeShardCount));
|
|
|
- assertThat(searchResponse.getTotalShards(), equalTo(totalShards));
|
|
|
- });
|
|
|
-
|
|
|
- SearchShardAPIResult searchShardResult = doSearchShardAPIQuery(indicesToSearch, rangeQuery, true, totalShards);
|
|
|
- assertThat(searchShardResult.skipped().size(), equalTo(indexOutsideSearchRangeShardCount));
|
|
|
- assertThat(searchShardResult.notSkipped().size(), equalTo(indexWithinSearchRangeShardCount));
|
|
|
- }
|
|
|
-
|
|
|
- // query a range that covers both indexes - all shards should be searched, none skipped
|
|
|
- {
|
|
|
- RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
|
|
|
- .from("2019-11-28T00:00:00.000000000Z", true)
|
|
|
- .to("2021-11-29T00:00:00.000000000Z");
|
|
|
-
|
|
|
- SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
|
|
|
- .source(new SearchSourceBuilder().query(rangeQuery));
|
|
|
-
|
|
|
- assertResponse(client().search(request), searchResponse -> {
|
|
|
- assertThat(searchResponse.getSuccessfulShards(), equalTo(totalShards));
|
|
|
- assertThat(searchResponse.getFailedShards(), equalTo(0));
|
|
|
- assertThat(searchResponse.getSkippedShards(), equalTo(0));
|
|
|
- assertThat(searchResponse.getTotalShards(), equalTo(totalShards));
|
|
|
- });
|
|
|
-
|
|
|
- SearchShardAPIResult searchShardResult = doSearchShardAPIQuery(indicesToSearch, rangeQuery, true, totalShards);
|
|
|
- assertThat(searchShardResult.skipped().size(), equalTo(0));
|
|
|
- assertThat(searchShardResult.notSkipped().size(), equalTo(totalShards));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
/**
|
|
|
* Can match against searchable snapshots is tested via both the Search API and the SearchShards (transport-only) API.
|
|
|
* The latter is a way to do only a can-match rather than all search phases.
|
|
@@ -588,7 +396,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
- createIndexWithTimestampAndEventIngested(
|
|
|
+ createIndexWithTimestamp(
|
|
|
indexOutsideSearchRange,
|
|
|
indexOutsideSearchRangeShardCount,
|
|
|
Settings.builder()
|
|
@@ -596,7 +404,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
.build()
|
|
|
);
|
|
|
|
|
|
- indexDocumentsWithTimestampAndEventIngestedDates(indexOutsideSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_OUTSIDE_RANGE);
|
|
|
+ indexDocumentsWithTimestampWithinDate(indexOutsideSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_OUTSIDE_RANGE);
|
|
|
|
|
|
final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
createRepository(repositoryName, "mock");
|
|
@@ -630,14 +438,11 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
- assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
|
|
|
- DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
|
|
|
- assertThat(timestampFieldTypeInfo, nullValue());
|
|
|
+ DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
|
|
|
+ assertThat(timestampFieldType, nullValue());
|
|
|
|
|
|
- final String timestampField = randomFrom(DataStream.TIMESTAMP_FIELD_NAME, IndexMetadata.EVENT_INGESTED_FIELD_NAME);
|
|
|
-
|
|
|
- RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
|
|
|
+ RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
|
|
|
.from("2020-11-28T00:00:00.000000000Z", true)
|
|
|
.to("2020-11-29T00:00:00.000000000Z");
|
|
|
|
|
@@ -695,29 +500,14 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
ensureGreen(searchableSnapshotIndexOutsideSearchRange);
|
|
|
|
|
|
final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
|
|
|
- timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
|
|
|
- assertThat(timestampFieldTypeInfo, notNullValue());
|
|
|
-
|
|
|
- final IndexLongFieldRange updatedTimestampRange = updatedIndexMetadata.getTimestampRange();
|
|
|
- DateFieldMapper.Resolution tsResolution = timestampFieldTypeInfo.getTimestampFieldType().resolution();
|
|
|
- ;
|
|
|
- assertThat(updatedTimestampRange.isComplete(), equalTo(true));
|
|
|
- assertThat(updatedTimestampRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
- assertThat(updatedTimestampRange.getMin(), greaterThanOrEqualTo(tsResolution.convert(Instant.parse("2020-11-26T00:00:00Z"))));
|
|
|
- assertThat(updatedTimestampRange.getMax(), lessThanOrEqualTo(tsResolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
|
|
|
-
|
|
|
- final IndexLongFieldRange updatedEventIngestedRange = updatedIndexMetadata.getEventIngestedRange();
|
|
|
- DateFieldMapper.Resolution eventIngestedResolution = timestampFieldTypeInfo.getEventIngestedFieldType().resolution();
|
|
|
- assertThat(updatedEventIngestedRange.isComplete(), equalTo(true));
|
|
|
- assertThat(updatedEventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedRange.getMin(),
|
|
|
- greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedRange.getMax(),
|
|
|
- lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
|
|
|
- );
|
|
|
+ final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
|
|
|
+ final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
|
|
|
+ assertThat(dateFieldType, notNullValue());
|
|
|
+ final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
|
|
|
+ assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
|
|
|
+ assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
+ assertThat(updatedTimestampMillisRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z"))));
|
|
|
+ assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
|
|
|
|
|
|
// Stop the node holding the searchable snapshots, and since we defined
|
|
|
// the index allocation criteria to require the searchable snapshot
|
|
@@ -789,7 +579,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
|
|
|
- createIndexWithTimestampAndEventIngested(
|
|
|
+ createIndexWithTimestamp(
|
|
|
indexWithinSearchRange,
|
|
|
indexWithinSearchRangeShardCount,
|
|
|
Settings.builder()
|
|
@@ -797,7 +587,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
.build()
|
|
|
);
|
|
|
|
|
|
- indexDocumentsWithTimestampAndEventIngestedDates(indexWithinSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_WITHIN_RANGE);
|
|
|
+ indexDocumentsWithTimestampWithinDate(indexWithinSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_WITHIN_RANGE);
|
|
|
|
|
|
final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
|
|
|
createRepository(repositoryName, "mock");
|
|
@@ -831,13 +621,11 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexWithinSearchRange);
|
|
|
assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
- assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
|
|
|
|
|
|
- DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
|
|
|
- assertThat(timestampFieldTypeInfo, nullValue());
|
|
|
+ DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
|
|
|
+ assertThat(timestampFieldType, nullValue());
|
|
|
|
|
|
- String timeField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
|
|
|
- RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timeField)
|
|
|
+ RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
|
|
|
.from("2020-11-28T00:00:00.000000000Z", true)
|
|
|
.to("2020-11-29T00:00:00.000000000Z");
|
|
|
|
|
@@ -892,32 +680,13 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
|
|
|
final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexWithinSearchRange);
|
|
|
final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
|
|
|
- timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
|
|
|
- assertThat(timestampFieldTypeInfo, notNullValue());
|
|
|
- final DateFieldMapper.Resolution timestampResolution = timestampFieldTypeInfo.getTimestampFieldType().resolution();
|
|
|
+ final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
|
|
|
+ assertThat(dateFieldType, notNullValue());
|
|
|
+ final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
|
|
|
assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
|
|
|
assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
- assertThat(
|
|
|
- updatedTimestampMillisRange.getMin(),
|
|
|
- greaterThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-28T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedTimestampMillisRange.getMax(),
|
|
|
- lessThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-29T00:00:00Z")))
|
|
|
- );
|
|
|
-
|
|
|
- final IndexLongFieldRange updatedEventIngestedMillisRange = updatedIndexMetadata.getEventIngestedRange();
|
|
|
- final DateFieldMapper.Resolution eventIngestedResolution = timestampFieldTypeInfo.getEventIngestedFieldType().resolution();
|
|
|
- assertThat(updatedEventIngestedMillisRange.isComplete(), equalTo(true));
|
|
|
- assertThat(updatedEventIngestedMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedMillisRange.getMin(),
|
|
|
- greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-28T00:00:00Z")))
|
|
|
- );
|
|
|
- assertThat(
|
|
|
- updatedEventIngestedMillisRange.getMax(),
|
|
|
- lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-29T00:00:00Z")))
|
|
|
- );
|
|
|
+ assertThat(updatedTimestampMillisRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-28T00:00:00Z"))));
|
|
|
+ assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-29T00:00:00Z"))));
|
|
|
|
|
|
// Stop the node holding the searchable snapshots, and since we defined
|
|
|
// the index allocation criteria to require the searchable snapshot
|
|
@@ -955,24 +724,17 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private void createIndexWithTimestampAndEventIngested(String indexName, int numShards, Settings extraSettings) throws IOException {
|
|
|
+ private void createIndexWithTimestamp(String indexName, int numShards, Settings extraSettings) throws IOException {
|
|
|
assertAcked(
|
|
|
indicesAdmin().prepareCreate(indexName)
|
|
|
.setMapping(
|
|
|
XContentFactory.jsonBuilder()
|
|
|
.startObject()
|
|
|
.startObject("properties")
|
|
|
-
|
|
|
.startObject(DataStream.TIMESTAMP_FIELD_NAME)
|
|
|
.field("type", randomFrom("date", "date_nanos"))
|
|
|
.field("format", "strict_date_optional_time_nanos")
|
|
|
.endObject()
|
|
|
-
|
|
|
- .startObject(IndexMetadata.EVENT_INGESTED_FIELD_NAME)
|
|
|
- .field("type", randomFrom("date", "date_nanos"))
|
|
|
- .field("format", "strict_date_optional_time_nanos")
|
|
|
- .endObject()
|
|
|
-
|
|
|
.endObject()
|
|
|
.endObject()
|
|
|
)
|
|
@@ -981,70 +743,12 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
ensureGreen(indexName);
|
|
|
}
|
|
|
|
|
|
- private void createIndexWithOnlyOneTimestampField(String timestampField, String index, int numShards, Settings extraSettings)
|
|
|
- throws IOException {
|
|
|
- assertAcked(
|
|
|
- indicesAdmin().prepareCreate(index)
|
|
|
- .setMapping(
|
|
|
- XContentFactory.jsonBuilder()
|
|
|
- .startObject()
|
|
|
- .startObject("properties")
|
|
|
-
|
|
|
- .startObject(timestampField)
|
|
|
- .field("type", randomFrom("date", "date_nanos"))
|
|
|
- .field("format", "strict_date_optional_time_nanos")
|
|
|
- .endObject()
|
|
|
-
|
|
|
- .endObject()
|
|
|
- .endObject()
|
|
|
- )
|
|
|
- .setSettings(indexSettingsNoReplicas(numShards).put(INDEX_SOFT_DELETES_SETTING.getKey(), true).put(extraSettings))
|
|
|
- );
|
|
|
- ensureGreen(index);
|
|
|
- }
|
|
|
-
|
|
|
- private void indexDocumentsWithOnlyOneTimestampField(String timestampField, String index, int docCount, String timestampTemplate)
|
|
|
- throws Exception {
|
|
|
- final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
|
|
|
- for (int i = 0; i < docCount; i++) {
|
|
|
- indexRequestBuilders.add(
|
|
|
- prepareIndex(index).setSource(
|
|
|
- timestampField,
|
|
|
- String.format(
|
|
|
- Locale.ROOT,
|
|
|
- timestampTemplate,
|
|
|
- between(0, 23),
|
|
|
- between(0, 59),
|
|
|
- between(0, 59),
|
|
|
- randomLongBetween(0, 999999999L)
|
|
|
- )
|
|
|
- )
|
|
|
- );
|
|
|
- }
|
|
|
- indexRandom(true, false, indexRequestBuilders);
|
|
|
-
|
|
|
- assertThat(indicesAdmin().prepareForceMerge(index).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), equalTo(0));
|
|
|
- refresh(index);
|
|
|
- forceMerge();
|
|
|
- }
|
|
|
-
|
|
|
- private void indexDocumentsWithTimestampAndEventIngestedDates(String indexName, int docCount, String timestampTemplate)
|
|
|
- throws Exception {
|
|
|
-
|
|
|
+ private void indexDocumentsWithTimestampWithinDate(String indexName, int docCount, String timestampTemplate) throws Exception {
|
|
|
final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
|
|
|
for (int i = 0; i < docCount; i++) {
|
|
|
indexRequestBuilders.add(
|
|
|
prepareIndex(indexName).setSource(
|
|
|
DataStream.TIMESTAMP_FIELD_NAME,
|
|
|
- String.format(
|
|
|
- Locale.ROOT,
|
|
|
- timestampTemplate,
|
|
|
- between(0, 23),
|
|
|
- between(0, 59),
|
|
|
- between(0, 59),
|
|
|
- randomLongBetween(0, 999999999L)
|
|
|
- ),
|
|
|
- IndexMetadata.EVENT_INGESTED_FIELD_NAME,
|
|
|
String.format(
|
|
|
Locale.ROOT,
|
|
|
timestampTemplate,
|
|
@@ -1085,39 +789,4 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
|
|
|
private void waitUntilAllShardsAreUnassigned(Index index) throws Exception {
|
|
|
awaitClusterState(state -> state.getRoutingTable().index(index).allPrimaryShardsUnassigned());
|
|
|
}
|
|
|
-
|
|
|
- record SearchShardAPIResult(List<SearchShardsGroup> skipped, List<SearchShardsGroup> notSkipped) {}
|
|
|
-
|
|
|
- private static SearchShardAPIResult doSearchShardAPIQuery(
|
|
|
- List<String> indicesToSearch,
|
|
|
- RangeQueryBuilder rangeQuery,
|
|
|
- boolean allowPartialSearchResults,
|
|
|
- int expectedTotalShards
|
|
|
- ) {
|
|
|
- SearchShardsRequest searchShardsRequest = new SearchShardsRequest(
|
|
|
- indicesToSearch.toArray(new String[0]),
|
|
|
- SearchRequest.DEFAULT_INDICES_OPTIONS,
|
|
|
- rangeQuery,
|
|
|
- null,
|
|
|
- null,
|
|
|
- allowPartialSearchResults,
|
|
|
- null
|
|
|
- );
|
|
|
-
|
|
|
- SearchShardsResponse searchShardsResponse = client().execute(TransportSearchShardsAction.TYPE, searchShardsRequest).actionGet();
|
|
|
- assertThat(searchShardsResponse.getGroups().size(), equalTo(expectedTotalShards));
|
|
|
- List<List<SearchShardsGroup>> partitionedBySkipped = searchShardsResponse.getGroups()
|
|
|
- .stream()
|
|
|
- .collect(
|
|
|
- Collectors.teeing(
|
|
|
- Collectors.filtering(g -> g.skipped(), Collectors.toList()),
|
|
|
- Collectors.filtering(g -> g.skipped() == false, Collectors.toList()),
|
|
|
- List::of
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- List<SearchShardsGroup> skipped = partitionedBySkipped.get(0);
|
|
|
- List<SearchShardsGroup> notSkipped = partitionedBySkipped.get(1);
|
|
|
- return new SearchShardAPIResult(skipped, notSkipped);
|
|
|
- }
|
|
|
}
|