Merge pull request ESQL-1118 from elastic/main

🤖 ESQL: Merge upstream

docs/changelog/95865.yaml

@@ -0,0 +1,6 @@
+pr: 95865
+summary: Fix searching a filtered and unfiltered data stream alias
+area: Data streams
+type: bug
+issues:
+ - 95786

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -133,6 +133,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItemInArray;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -801,6 +802,53 @@ public class DataStreamIT extends ESIntegTestCase {
         assertSearchHits(searchResponse, "1");
     }
 
+    public void testSearchFilteredAndUnfilteredAlias() throws Exception {
+        putComposableIndexTemplate("id1", List.of("logs-*"));
+        String dataStreamName = "logs-foobar";
+        client().prepareIndex(dataStreamName)
+            .setId("1")
+            .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON)
+            .setOpType(DocWriteRequest.OpType.CREATE)
+            .get();
+        client().prepareIndex(dataStreamName)
+            .setId("2")
+            .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON)
+            .setOpType(DocWriteRequest.OpType.CREATE)
+            .get();
+        refresh(dataStreamName);
+
+        AliasActions addFilteredAliasAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName)
+            .aliases("foo")
+            .filter(Map.of("term", Map.of("type", Map.of("value", "y"))));
+        AliasActions addUnfilteredAliasAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName).aliases("bar");
+
+        IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
+        aliasesAddRequest.addAliasAction(addFilteredAliasAction);
+        aliasesAddRequest.addAliasAction(addUnfilteredAliasAction);
+        assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet());
+        GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest()).actionGet();
+        assertThat(response.getDataStreamAliases(), hasKey("logs-foobar"));
+        assertThat(
+            response.getDataStreamAliases().get("logs-foobar"),
+            containsInAnyOrder(
+                new DataStreamAlias("bar", List.of("logs-foobar"), null, null),
+                new DataStreamAlias(
+                    "foo",
+                    List.of("logs-foobar"),
+                    null,
+                    Map.of("logs-foobar", Map.of("term", Map.of("type", Map.of("value", "y"))))
+                )
+            )
+        );
+
+        // Searching the filtered and unfiltered aliases should return all results (unfiltered):
+        SearchResponse searchResponse = client().prepareSearch("foo", "bar").get();
+        assertSearchHits(searchResponse, "1", "2");
+        // Searching the data stream name and the filtered alias should return all results (unfiltered):
+        searchResponse = client().prepareSearch("foo", dataStreamName).get();
+        assertSearchHits(searchResponse, "1", "2");
+    }
+
     public void testRandomDataSteamAliasesUpdate() throws Exception {
         putComposableIndexTemplate("id1", List.of("log-*"));
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -701,19 +701,41 @@ public class IndexNameExpressionResolver {
         IndexAbstraction ia = state.metadata().getIndicesLookup().get(index);
         DataStream dataStream = ia.getParentDataStream();
         if (dataStream != null) {
+            if (skipIdentity == false && resolvedExpressions.contains(dataStream.getName())) {
+                // skip the filters when the request targets the data stream name
+                return null;
+            }
             Map<String, DataStreamAlias> dataStreamAliases = state.metadata().dataStreamAliases();
-            Stream<DataStreamAlias> stream;
+            List<DataStreamAlias> aliasesForDataStream;
             if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) {
-                stream = dataStreamAliases.values()
+                aliasesForDataStream = dataStreamAliases.values()
                     .stream()
-                    .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName()));
+                    .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName()))
+                    .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
+                    .toList();
             } else {
-                stream = resolvedExpressions.stream().map(dataStreamAliases::get).filter(Objects::nonNull);
+                aliasesForDataStream = resolvedExpressions.stream()
+                    .map(dataStreamAliases::get)
+                    .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName()))
+                    .toList();
+            }
+
+            List<String> requiredAliases = null;
+            for (DataStreamAlias dataStreamAlias : aliasesForDataStream) {
+                if (requiredDataStreamAlias.test(dataStreamAlias)) {
+                    if (requiredAliases == null) {
+                        requiredAliases = new ArrayList<>(aliasesForDataStream.size());
+                    }
+                    requiredAliases.add(dataStreamAlias.getName());
+                } else {
+                    // we have a non-required alias for this data stream so no need to check further
+                    return null;
+                }
+            }
+            if (requiredAliases == null) {
+                return null;
             }
-            return stream.filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
-                .filter(requiredDataStreamAlias)
-                .map(DataStreamAlias::getName)
-                .toArray(String[]::new);
+            return requiredAliases.toArray(Strings.EMPTY_ARRAY);
         } else {
             final Map<String, AliasMetadata> indexAliases = indexMetadata.getAliases();
             final AliasMetadata[] aliasCandidates;

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -1672,9 +1672,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, false, resolvedExpressions);
             assertThat(result, nullValue());
         }
-
         {
-            // Only resolve aliases that refer to dataStreamName1 where the filter is not null
+            // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1
             Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*");
             String index = backingIndex1.getIndex().getName();
             String[] result = indexNameExpressionResolver.indexAliases(
@@ -1685,7 +1684,49 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 true,
                 resolvedExpressions
             );
-            assertThat(result, arrayContainingInAnyOrder("logs", "logs_foo"));
+            assertThat(result, nullValue());
+        }
+        {
+            // Null is returned, because an unfiltered alias is targeting the same data stream
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                true,
+                resolvedExpressions
+            );
+            assertThat(result, nullValue());
+        }
+        {
+            // The filtered alias is returned because although we target the data stream name, skipIdentity is true
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                true,
+                resolvedExpressions
+            );
+            assertThat(result, arrayContainingInAnyOrder("logs"));
+        }
+        {
+            // Null is returned because we target the data stream name and skipIdentity is false
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                false,
+                resolvedExpressions
+            );
+            assertThat(result, nullValue());
         }
     }
 

server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java

@@ -88,6 +88,7 @@ import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -733,14 +734,16 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bax")));
         }
         {
+            // querying an unfiltered and a filtered alias for the same data stream should drop the filters
             String index = backingIndex1.getIndex().getName();
             AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs", "logs_bar"));
-            assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs"));
-            BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder();
-            assertThat(filter.filter(), empty());
-            assertThat(filter.must(), empty());
-            assertThat(filter.mustNot(), empty());
-            assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar")));
+            assertThat(result, is(AliasFilter.EMPTY));
+        }
+        {
+            // similarly, querying the data stream name and a filtered alias should drop the filter
+            String index = backingIndex1.getIndex().getName();
+            AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs", dataStreamName1));
+            assertThat(result, is(AliasFilter.EMPTY));
         }
     }
 }

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/entsearch/analytics/behavioral_analytics-events-template.json

@@ -8,6 +8,7 @@
     "behavioral_analytics-events-settings",
     "behavioral_analytics-events-mappings"
   ],
+  "allow_auto_create": false,
   "_meta": {
     "description": "Built-in template applied by default to behavioral analytics event data streams.",
     "managed": true

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java

@@ -108,8 +108,6 @@ public class AnalyticsEventEmitter extends AbstractLifecycleComponent {
 
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             return client.prepareIndex(collection.getEventDataStream())
-                // disable auto-creation of the underlying index
-                .setRequireAlias(true)
                 .setCreate(true)
                 .setSource(event.toXContent(builder, ToXContent.EMPTY_PARAMS))
                 .request();