Add source fallback support for unsigned long mapped type (#89349)

This change adds source fallback support for unsigned long by adding a new 
SourceValueFetcherSortedUnsignedLongIndexFieldData similar to the other numeric types.

docs/changelog/89349.yaml

@@ -0,0 +1,5 @@
+pr: 89349
+summary: Add source fallback support for unsigned long mapped type
+area: Mapping
+type: enhancement
+issues: []

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/SourceValueFetcherSortedUnsignedLongIndexFieldData.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.unsignedlong;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.SourceValueFetcherIndexFieldData;
+import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
+import org.elasticsearch.script.field.ToScriptFieldFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.elasticsearch.search.DocValueFormat.MASK_2_63;
+
+/**
+ * {@code SourceValueFetcherSortedUnsignedLongIndexFieldData} uses a {@link ValueFetcher} to
+ * retrieve values from source that are parsed as an unsigned long. These values are used to
+ * emulate unsigned long values pulled directly from a doc values data structure through a
+ * {@link SortedNumericDocValues}.
+ */
+public class SourceValueFetcherSortedUnsignedLongIndexFieldData extends SourceValueFetcherIndexFieldData<SortedNumericDocValues> {
+
+    public static class Builder extends SourceValueFetcherIndexFieldData.Builder<SortedNumericDocValues> {
+
+        public Builder(
+            String fieldName,
+            ValuesSourceType valuesSourceType,
+            ValueFetcher valueFetcher,
+            SourceLookup sourceLookup,
+            ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
+        ) {
+            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        }
+
+        @Override
+        public SourceValueFetcherSortedUnsignedLongIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) {
+            return new SourceValueFetcherSortedUnsignedLongIndexFieldData(
+                fieldName,
+                valuesSourceType,
+                valueFetcher,
+                sourceLookup,
+                toScriptFieldFactory
+            );
+        }
+    }
+
+    protected SourceValueFetcherSortedUnsignedLongIndexFieldData(
+        String fieldName,
+        ValuesSourceType valuesSourceType,
+        ValueFetcher valueFetcher,
+        SourceLookup sourceLookup,
+        ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
+    ) {
+        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+    }
+
+    @Override
+    public SourceValueFetcherLeafFieldData<SortedNumericDocValues> loadDirect(LeafReaderContext context) {
+        return new SourceValueFetcherSortedUnsignedLongLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+    }
+
+    private static class SourceValueFetcherSortedUnsignedLongLeafFieldData extends SourceValueFetcherLeafFieldData<SortedNumericDocValues> {
+
+        private SourceValueFetcherSortedUnsignedLongLeafFieldData(
+            ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory,
+            LeafReaderContext leafReaderContext,
+            ValueFetcher valueFetcher,
+            SourceLookup sourceLookup
+        ) {
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+        }
+
+        @Override
+        public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
+            return toScriptFieldFactory.getScriptFieldFactory(
+                new SourceValueFetcherSortedUnsignedLongDocValues(leafReaderContext, valueFetcher, sourceLookup),
+                name
+            );
+        }
+    }
+
+    private static class SourceValueFetcherSortedUnsignedLongDocValues extends SortedNumericDocValues implements ValueFetcherDocValues {
+
+        private final LeafReaderContext leafReaderContext;
+
+        private final ValueFetcher valueFetcher;
+        private final SourceLookup sourceLookup;
+
+        private List<Long> values;
+        private Iterator<Long> iterator;
+
+        private SourceValueFetcherSortedUnsignedLongDocValues(
+            LeafReaderContext leafReaderContext,
+            ValueFetcher valueFetcher,
+            SourceLookup sourceLookup
+        ) {
+            this.leafReaderContext = leafReaderContext;
+            this.valueFetcher = valueFetcher;
+            this.sourceLookup = sourceLookup;
+
+            values = new ArrayList<>();
+        }
+
+        @Override
+        public boolean advanceExact(int doc) throws IOException {
+            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
+            values.clear();
+
+            for (Object value : valueFetcher.fetchValues(sourceLookup, Collections.emptyList())) {
+                assert value instanceof Number;
+                values.add(((Number) value).longValue() ^ MASK_2_63);
+            }
+
+            values.sort(Long::compare);
+            iterator = values.iterator();
+
+            return true;
+        }
+
+        @Override
+        public int docValueCount() {
+            return values.size();
+        }
+
+        @Override
+        public long nextValue() throws IOException {
+            assert iterator.hasNext();
+            return iterator.next();
+        }
+
+        @Override
+        public int docID() {
+            throw new UnsupportedOperationException("not supported for source fallback");
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            throw new UnsupportedOperationException("not supported for source fallback");
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            throw new UnsupportedOperationException("not supported for source fallback");
+        }
+
+        @Override
+        public long cost() {
+            throw new UnsupportedOperationException("not supported for source fallback");
+        }
+    }
+}

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -36,6 +36,8 @@ import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -50,6 +52,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.unsignedlong.UnsignedLongLeafFieldData.convertUnsignedLongToDouble;
@@ -288,15 +291,37 @@ public class UnsignedLongFieldMapper extends FieldMapper {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
-            failIfNoDocValues();
-            return (cache, breakerService) -> {
-                final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder(
+            FielddataOperation operation = fieldDataContext.fielddataOperation();
+
+            if (operation == FielddataOperation.SEARCH) {
+                failIfNoDocValues();
+            }
+
+            if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) {
+                return (cache, breakerService) -> {
+                    final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder(
+                        name(),
+                        IndexNumericFieldData.NumericType.LONG,
+                        (dv, n) -> { throw new UnsupportedOperationException(); }
+                    ).build(cache, breakerService);
+                    return new UnsignedLongIndexFieldData(signedLongValues, UnsignedLongDocValuesField::new);
+                };
+            }
+
+            if (operation == FielddataOperation.SCRIPT) {
+                SearchLookup searchLookup = fieldDataContext.lookupSupplier().get();
+                Set<String> sourcePaths = fieldDataContext.sourcePathsLookup().apply(name());
+
+                return new SourceValueFetcherSortedUnsignedLongIndexFieldData.Builder(
                     name(),
-                    IndexNumericFieldData.NumericType.LONG,
-                    (dv, n) -> { throw new UnsupportedOperationException(); }
-                ).build(cache, breakerService);
-                return new UnsignedLongIndexFieldData(signedLongValues, UnsignedLongDocValuesField::new);
-            };
+                    CoreValuesSourceType.NUMERIC,
+                    sourceValueFetcher(sourcePaths),
+                    searchLookup.source(),
+                    UnsignedLongDocValuesField::new
+                );
+            }
+
+            throw new IllegalStateException("unknown field data operation [" + operation.name() + "]");
         }
 
         @Override
@@ -304,8 +329,11 @@ public class UnsignedLongFieldMapper extends FieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
+            return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
+        }
 
-            return new SourceValueFetcher(name(), context, nullValueFormatted) {
+        private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
+            return new SourceValueFetcher(sourcePaths, nullValueFormatted) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     if (value.equals("")) {

x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml

@@ -12,6 +12,14 @@ setup:
             properties:
               ul:
                 type: unsigned_long
+              ul_ndv:
+                type: unsigned_long
+                doc_values: false
+              ul_multi:
+                type: unsigned_long
+              ul_ndv_multi:
+                type: unsigned_long
+                doc_values: false
 
   - do:
       bulk:
@@ -19,15 +27,15 @@ setup:
         refresh: true
         body: |
           { "index": {"_id" : "1"} }
-          { "ul": 0 }
+          { "ul": 0, "ul_ndv": 0, "ul_multi": [18446744073709551615, 9223372036854775808, 0, 0],  "ul_ndv_multi": [0, 18446744073709551615, 0, 9223372036854775808]}
           { "index": {"_id" : "2"} }
-          { "ul": 9223372036854775807 }
+          { "ul": 9223372036854775807, "ul_ndv": 9223372036854775807 }
           { "index": {"_id" : "3"} }
-          { "ul": 9223372036854775808 }
+          { "ul": 9223372036854775808, "ul_ndv": 9223372036854775808 }
           { "index": {"_id" : "4"} }
-          { "ul": 18446744073709551613 }
+          { "ul": 18446744073709551613, "ul_ndv": 18446744073709551613 }
           { "index": {"_id" : "5"} }
-          { "ul": 18446744073709551615 }
+          { "ul": 18446744073709551615, "ul_ndv": 18446744073709551615 }
 
 ---
 "Scripted fields values return Long":
@@ -63,6 +71,89 @@ setup:
   - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 }
   - match: { hits.hits.4.fields.scripted_ul.0: 0 }
 
+---
+"Scripted fields using multi-value unsigned long":
+  - do:
+      search:
+        index: test1
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            scripted_ul_0:
+              script:
+                source: "field('ul_multi').getValue(1000L)"
+            scripted_ul_1:
+              script:
+                source: "field('ul_multi').getValue(1, 1000L)"
+            scripted_ul_2:
+              script:
+                source: "field('ul_multi').getValue(2, 1000L)"
+            scripted_ul_3:
+              script:
+                source: "field('ul_multi').getValue(3, 1000L)"
+
+  - match: { hits.hits.0.fields.scripted_ul_0.0: 0 }
+  - match: { hits.hits.0.fields.scripted_ul_1.0: 0 }
+  - match: { hits.hits.0.fields.scripted_ul_2.0: -9223372036854775808 }
+  - match: { hits.hits.0.fields.scripted_ul_3.0: -1 }
+
+---
+"No Doc Values: Scripted fields values return Long":
+  - do:
+      search:
+        index: test1
+        body:
+          sort: [ { ul: desc } ]
+          script_fields:
+            scripted_ul:
+              script:
+                source: "field('ul_ndv').getValue(1000L)"
+
+  - match: { hits.hits.0.fields.scripted_ul.0: -1 }
+  - match: { hits.hits.1.fields.scripted_ul.0: -3 }
+  - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 }
+  - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 }
+  - match: { hits.hits.4.fields.scripted_ul.0: 0 }
+
+  - do:
+      catch: bad_request
+      search:
+        index: test1
+        body:
+          sort: [ { ul: desc } ]
+          script_fields:
+            scripted_ul:
+              script:
+                source: "doc['ul_ndv'].value"
+  - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
+
+---
+"No Doc Values: Scripted fields using multi-value unsigned long":
+  - do:
+      search:
+        index: test1
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            scripted_ul_0:
+              script:
+                source: "field('ul_ndv_multi').getValue(1000L)"
+            scripted_ul_1:
+              script:
+                source: "field('ul_ndv_multi').getValue(1, 1000L)"
+            scripted_ul_2:
+              script:
+                source: "field('ul_ndv_multi').getValue(2, 1000L)"
+            scripted_ul_3:
+              script:
+                source: "field('ul_ndv_multi').getValue(3, 1000L)"
+
+  - match: { hits.hits.0.fields.scripted_ul_0.0: 0 }
+  - match: { hits.hits.0.fields.scripted_ul_1.0: 0 }
+  - match: { hits.hits.0.fields.scripted_ul_2.0: -9223372036854775808 }
+  - match: { hits.hits.0.fields.scripted_ul_3.0: -1 }
+
+
 ---
 "Scripted sort values":
   - do: