Fix for wildcard fields that returned ByteRefs not Strings to scripts. (#58060)

This need some reorg of BinaryDV field data classes to allow specialisation of scripted doc values.
Moved common logic to a new abstract base class and added a new subclass to return string-based representations to scripts.

Closes #58044

server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+abstract class  AbstractBinaryDVLeafFieldData implements LeafFieldData {
+    private final BinaryDocValues values;
+
+    AbstractBinaryDVLeafFieldData(BinaryDocValues values) {
+        super();
+        this.values = values;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return 0; // not exposed by Lucene
+    }
+
+    @Override
+    public Collection<Accountable> getChildResources() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public SortedBinaryDocValues getBytesValues() {
+        return new SortedBinaryDocValues() {
+
+            int count;
+            final ByteArrayDataInput in = new ByteArrayDataInput();
+            final BytesRef scratch = new BytesRef();
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                if (values.advanceExact(doc)) {
+                    final BytesRef bytes = values.binaryValue();
+                    assert bytes.length > 0;
+                    in.reset(bytes.bytes, bytes.offset, bytes.length);
+                    count = in.readVInt();
+                    scratch.bytes = bytes.bytes;
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            @Override
+            public int docValueCount() {
+                return count;
+            }
+
+            @Override
+            public BytesRef nextValue() throws IOException {
+                scratch.length = in.readVInt();
+                scratch.offset = in.getPosition();
+                in.setPosition(scratch.offset + scratch.length);
+                return scratch;
+            }
+
+        };
+    }
+
+
+    @Override
+    public void close() {
+        // no-op
+    }
+}

server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java

@@ -20,83 +20,18 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-final class BytesBinaryDVLeafFieldData implements LeafFieldData {
-
-    private final BinaryDocValues values;
 
+final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData {
     BytesBinaryDVLeafFieldData(BinaryDocValues values) {
-        super();
-        this.values = values;
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return 0; // not exposed by Lucene
-    }
-
-    @Override
-    public Collection<Accountable> getChildResources() {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public SortedBinaryDocValues getBytesValues() {
-        return new SortedBinaryDocValues() {
-
-            int count;
-            final ByteArrayDataInput in = new ByteArrayDataInput();
-            final BytesRef scratch = new BytesRef();
-
-            @Override
-            public boolean advanceExact(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    final BytesRef bytes = values.binaryValue();
-                    assert bytes.length > 0;
-                    in.reset(bytes.bytes, bytes.offset, bytes.length);
-                    count = in.readVInt();
-                    scratch.bytes = bytes.bytes;
-                    return true;
-                } else {
-                    return false;
-                }
-            }
-
-            @Override
-            public int docValueCount() {
-                return count;
-            }
-
-            @Override
-            public BytesRef nextValue() throws IOException {
-                scratch.length = in.readVInt();
-                scratch.offset = in.getPosition();
-                in.setPosition(scratch.offset + scratch.length);
-                return scratch;
-            }
-
-        };
+        super(values);
     }
 
     @Override
     public ScriptDocValues<BytesRef> getScriptValues() {
         return new ScriptDocValues.BytesRefs(getBytesValues());
     }
-
-    @Override
-    public void close() {
-        // no-op
-    }
-
-
 }
+

server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+
+final class StringBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData{
+    StringBinaryDVLeafFieldData(BinaryDocValues values) {
+        super(values);
+    }
+
+    @Override
+    public ScriptDocValues<String> getScriptValues() {
+        return new ScriptDocValues.Strings(getBytesValues());
+    }
+}

server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.sort.BucketedSort;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.io.IOException;
+
+public class StringBinaryIndexFieldData implements IndexFieldData<StringBinaryDVLeafFieldData> {
+
+    protected final Index index;
+    protected final String fieldName;
+    protected final ValuesSourceType valuesSourceType;
+
+    public StringBinaryIndexFieldData(Index index, String fieldName, ValuesSourceType valuesSourceType) {
+        this.index = index;
+        this.fieldName = fieldName;
+        this.valuesSourceType = valuesSourceType;
+    }
+
+    @Override
+    public final String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    public ValuesSourceType getValuesSourceType() {
+        return valuesSourceType;
+    }
+
+    @Override
+    public final void clear() {
+        // can't do
+    }
+
+    @Override
+    public final Index index() {
+        return index;
+    }
+    
+    @Override
+    public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
+                sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
+    }
+    
+    @Override
+    public StringBinaryDVLeafFieldData load(LeafReaderContext context) {
+        try {
+            return new StringBinaryDVLeafFieldData(DocValues.getBinary(context.reader(), fieldName));
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot load doc values", e);
+        }
+    } 
+    @Override
+    public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
+            SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
+        throw new IllegalArgumentException("can't sort on binary field");
+    }
+
+    @Override
+    public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
+        return load(context);
+    }
+}

x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml

@@ -229,6 +229,23 @@ setup:
   - length: { aggregations.top_vals.buckets: 2 }
 
 ---
+"Scripted Aggs work":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*goodbye*" }
+          aggs:
+            top_vals:
+              terms: {script: "doc['my_wildcard']" }
+
+
+  - match: {hits.total.value: 1}
+  - length: { aggregations.top_vals.buckets: 1 }
+  - match: { aggregations.top_vals.buckets.0.key: "goodbye world" }
+---
 "Sort works":
   - do:
       search:

x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java

@@ -28,7 +28,6 @@ import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
@@ -47,16 +46,13 @@ import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.plain.BytesBinaryIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.StringBinaryIndexFieldData;
 import org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -68,9 +64,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -901,27 +895,12 @@ public class WildcardFieldMapper extends FieldMapper {
                 @Override
                 public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                         CircuitBreakerService breakerService, MapperService mapperService) {
-                    return new WildcardBytesBinaryIndexFieldData(indexSettings.getIndex(), fieldType.name(), CoreValuesSourceType.BYTES);
+                    return new StringBinaryIndexFieldData(indexSettings.getIndex(), fieldType.name(), CoreValuesSourceType.BYTES);
                 }};
         }
 
      }
 
-    static class WildcardBytesBinaryIndexFieldData extends BytesBinaryIndexFieldData {
-
-        WildcardBytesBinaryIndexFieldData(Index index, String fieldName, ValuesSourceType valuesSourceType) {
-            super(index, fieldName, valuesSourceType);
-        }
-
-        @Override
-        public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
-            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
-                    sortMode, nested);
-            return new SortField(getFieldName(), source, reverse);
-        }
-
-    }
-
     private int ignoreAbove;
 
     private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,