Check nested fields earlier in kNN search (#80516)

Currently, we don't support kNN search against fields in a `nested` mapping.
Before, we were checking this at search-time. This commit moves it earlier, so
you aren't even allowed to set `index: true` if the vector is in a nested
mapping. That way, users are aware of the limitation before they start to index
documents.

Relates to #78473.

docs/reference/mapping/types/dense-vector.asciidoc

@@ -92,6 +92,9 @@ NOTE: Indexing vectors for approximate kNN search is an expensive process. It ca
 substantial time to ingest documents that contain vector fields with `index`
 enabled.
 
+NOTE: Dense vector fields cannot be indexed if they are within
+<<nested, `nested`>> mappings.
+
 [role="child_attributes"]
 [[dense-vector-params]]
 ==== Parameters for dense vector fields

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/40_knn_search.yml

@@ -14,16 +14,6 @@ setup:
                 dims: 5
                 index: true
                 similarity: l2_norm
-              comments:
-                type: nested
-                properties:
-                  body:
-                    type: text
-                  vector:
-                    type: dense_vector
-                    dims: 5
-                    index: true
-                    similarity: l2_norm
 
   - do:
       index:
@@ -107,19 +97,3 @@ setup:
               num_candidates: 1
   - match: { error.root_cause.0.type: "illegal_argument_exception" }
   - match: { error.root_cause.0.reason: "[knn] queries cannot be provided directly, use the [_knn_search] endpoint instead" }
-
----
-"kNN searches on nested fields are disallowed":
-  - do:
-      catch: bad_request
-      knn_search:
-        index: test
-        body:
-          fields: [ "nonexistent" ]
-          knn:
-            field: comments.vector
-            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
-            k: 2
-            num_candidates: 3
-  - match: { error.root_cause.0.type: "query_shard_exception" }
-  - match: { error.root_cause.0.reason: "failed to create query: [knn] queries are not supported on nested fields" }

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java

@@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.MappingParser;
 import org.elasticsearch.index.mapper.PerFieldKnnVectorsFormatFieldMapper;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
@@ -494,6 +495,13 @@ public class DenseVectorFieldMapper extends FieldMapper implements PerFieldKnnVe
         return new Builder(simpleName(), indexCreatedVersion).init(this);
     }
 
+    @Override
+    public void doValidate(MappingLookup mappers) {
+        if (indexed && mappers.getNestedParent(name()) != null) {
+            throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields cannot be indexed if they're" + " within [nested] mappings");
+        }
+    }
+
     private static IndexOptions parseIndexOptions(String fieldName, Object propNode) {
         @SuppressWarnings("unchecked")
         Map<String, ?> indexOptionsMap = (Map<String, ?>) propNode;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilder.java

@@ -84,10 +84,6 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
             );
         }
 
-        if (context.getNestedParent(fieldType.name()) != null) {
-            throw new IllegalArgumentException("[" + NAME + "] queries are not supported on nested fields");
-        }
-
         DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType;
         return vectorFieldType.createKnnQuery(queryVector, numCands);
     }

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java

@@ -412,6 +412,26 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         assertThat(e.getMessage(), containsString("Field [vectors] of type [dense_vector] can't be used in multifields"));
     }
 
+    public void testNestedVectorsCannotBeIndexed() {
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> createMapperService(
+                fieldMapping(
+                    b -> b.field("type", "nested")
+                        .startObject("properties")
+                        .startObject("vector")
+                        .field("type", "dense_vector")
+                        .field("dims", 4)
+                        .field("index", true)
+                        .field("similarity", "dot_product")
+                        .endObject()
+                        .endObject()
+                )
+            )
+        );
+        assertThat(e.getMessage(), containsString("[dense_vector] fields cannot be indexed if they're within [nested] mappings"));
+    }
+
     public void testKnnVectorsFormat() throws IOException {
         final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10);
         final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10);