Add synthetic source support for geo_shape via fallback implementation (#108881)

This PR enables geo_shape mapper to use fallback synthetic source infrastructure and as such adds synthetic source support for this field type.

docs/changelog/108881.yaml

@@ -0,0 +1,5 @@
+pr: 108881
+summary: Add synthetic source support for `geo_shape` via fallback implementation
+area: Mapping
+type: feature
+issues: []

docs/reference/mapping/fields/synthetic-source.asciidoc

@@ -52,6 +52,7 @@ types:
 ** <<flattened-synthetic-source, `flattened`>>
 ** <<numeric-synthetic-source,`float`>>
 ** <<geo-point-synthetic-source,`geo_point`>>
+** <<geo-shape-synthetic-source,`geo_shape`>>
 ** <<numeric-synthetic-source,`half_float`>>
 ** <<histogram-synthetic-source,`histogram`>>
 ** <<numeric-synthetic-source,`integer`>>

docs/reference/mapping/types/geo-shape.asciidoc

@@ -493,3 +493,16 @@ Due to the complex input structure and index representation of shapes,
 it is not currently possible to sort shapes or retrieve their fields
 directly. The `geo_shape` value is only retrievable through the `_source`
 field.
+
+[[geo-shape-synthetic-source]]
+==== Synthetic source
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will work to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
+`geo_shape` fields support <<synthetic-source,synthetic `_source`>> in their
+default configuration.

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java

@@ -482,6 +482,11 @@ public class GeoShapeWithDocValuesFieldMapper extends AbstractShapeGeometryField
         super.checkIncomingMergeType(mergeWith);
     }
 
+    @Override
+    protected SyntheticSourceMode syntheticSourceMode() {
+        return SyntheticSourceMode.FALLBACK;
+    }
+
     public static class GeoShapeDocValuesField extends AbstractScriptFieldFactory<GeoShapeValues.GeoShapeValue>
         implements
             Field<GeoShapeValues.GeoShapeValue>,

x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoFieldMapperTests.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.spatial.index.mapper;
 
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperTestCase;
+import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
@@ -16,6 +17,7 @@ import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 /** Base class for testing geo field mappers */
@@ -25,7 +27,15 @@ public abstract class GeoFieldMapperTests extends MapperTestCase {
 
     @Override
     protected Collection<Plugin> getPlugins() {
-        return Collections.singletonList(new LocalStateSpatialPlugin());
+        var plugin = new LocalStateSpatialPlugin();
+        plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() {
+            @Override
+            public <T> List<T> loadExtensions(Class<T> extensionPointType) {
+                return List.of();
+            }
+        });
+
+        return Collections.singletonList(plugin);
     }
 
     @Override

x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java

@@ -7,8 +7,16 @@
 package org.elasticsearch.xpack.spatial.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.GeoJson;
+import org.elasticsearch.common.geo.GeometryNormalizer;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.GeometryValidator;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
@@ -22,13 +30,17 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -422,7 +434,161 @@ public class GeoShapeWithDocValuesFieldMapperTests extends GeoFieldMapperTests {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        // Almost like GeoShapeType but no circles
+        enum ShapeType {
+            POINT,
+            LINESTRING,
+            POLYGON,
+            MULTIPOINT,
+            MULTILINESTRING,
+            MULTIPOLYGON,
+            GEOMETRYCOLLECTION,
+            ENVELOPE
+        }
+
+        return new SyntheticSourceSupport() {
+            @Override
+            public boolean preservesExactSource() {
+                return true;
+            }
+
+            @Override
+            public SyntheticSourceExample example(int maxValues) throws IOException {
+                if (randomBoolean()) {
+                    Value v = generateValue();
+                    if (v.blockLoaderOutput != null) {
+                        return new SyntheticSourceExample(v.input, v.output, v.blockLoaderOutput, this::mapping);
+                    }
+                    return new SyntheticSourceExample(v.input, v.output, this::mapping);
+                }
+
+                List<Value> values = randomList(1, maxValues, this::generateValue);
+                List<Object> in = values.stream().map(Value::input).toList();
+                List<Object> out = values.stream().map(Value::output).toList();
+
+                // Block loader infrastructure will never return nulls
+                List<Object> outBlockList = values.stream()
+                    .filter(v -> v.input != null)
+                    .map(v -> v.blockLoaderOutput != null ? v.blockLoaderOutput : v.output)
+                    .toList();
+                var outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+
+                return new SyntheticSourceExample(in, out, outBlock, this::mapping);
+            }
+
+            private record Value(Object input, Object output, String blockLoaderOutput) {
+                Value(Object input, Object output) {
+                    this(input, output, null);
+                }
+            }
+
+            private Value generateValue() {
+                if (ignoreMalformed && randomBoolean()) {
+                    List<Supplier<Object>> choices = List.of(
+                        () -> randomAlphaOfLength(3),
+                        ESTestCase::randomInt,
+                        ESTestCase::randomLong,
+                        ESTestCase::randomFloat,
+                        ESTestCase::randomDouble
+                    );
+                    Object v = randomFrom(choices).get();
+                    return new Value(v, v);
+                }
+                if (randomBoolean()) {
+                    return new Value(null, null);
+                }
+
+                var type = randomFrom(ShapeType.values());
+                var isGeoJson = randomBoolean();
+
+                switch (type) {
+                    case POINT -> {
+                        var point = GeometryTestUtils.randomPoint(false);
+                        return value(point, isGeoJson);
+                    }
+                    case LINESTRING -> {
+                        var line = GeometryTestUtils.randomLine(false);
+                        return value(line, isGeoJson);
+                    }
+                    case POLYGON -> {
+                        var polygon = GeometryTestUtils.randomPolygon(false);
+                        return value(polygon, isGeoJson);
+                    }
+                    case MULTIPOINT -> {
+                        var multiPoint = GeometryTestUtils.randomMultiPoint(false);
+                        return value(multiPoint, isGeoJson);
+                    }
+                    case MULTILINESTRING -> {
+                        var multiPoint = GeometryTestUtils.randomMultiLine(false);
+                        return value(multiPoint, isGeoJson);
+                    }
+                    case MULTIPOLYGON -> {
+                        var multiPolygon = GeometryTestUtils.randomMultiPolygon(false);
+                        return value(multiPolygon, isGeoJson);
+                    }
+                    case GEOMETRYCOLLECTION -> {
+                        var multiPolygon = GeometryTestUtils.randomGeometryCollectionWithoutCircle(false);
+                        return value(multiPolygon, isGeoJson);
+                    }
+                    case ENVELOPE -> {
+                        var rectangle = GeometryTestUtils.randomRectangle();
+                        var wktString = WellKnownText.toWKT(rectangle);
+
+                        return new Value(wktString, wktString);
+                    }
+                    default -> throw new UnsupportedOperationException("Unsupported shape");
+                }
+            }
+
+            private static Value value(Geometry geometry, boolean isGeoJson) {
+                var wktString = WellKnownText.toWKT(geometry);
+                var normalizedWktString = GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry)
+                    ? WellKnownText.toWKT(GeometryNormalizer.apply(Orientation.RIGHT, geometry))
+                    : wktString;
+
+                if (isGeoJson) {
+                    var map = GeoJson.toMap(geometry);
+                    return new Value(map, map, normalizedWktString);
+                }
+
+                return new Value(wktString, wktString, normalizedWktString);
+            }
+
+            private void mapping(XContentBuilder b) throws IOException {
+                b.field("type", "geo_shape");
+                if (rarely()) {
+                    b.field("index", false);
+                }
+                if (rarely()) {
+                    b.field("doc_values", false);
+                }
+                if (ignoreMalformed) {
+                    b.field("ignore_malformed", true);
+                }
+            }
+
+            @Override
+            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+                return List.of();
+            }
+        };
+    }
+
+    @Override
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        return v -> asWKT((BytesRef) v);
+    }
+
+    protected static Object asWKT(BytesRef value) {
+        // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability
+        Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes);
+        return WellKnownText.toWKT(geometry);
+    }
+
+    @Override
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        // Synthetic source is currently not supported.
+        return new BlockReaderSupport(false, false, mapper, loaderFieldName);
     }
 
     @Override

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml

@@ -0,0 +1,251 @@
+---
+"geo_shape":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              shape:
+                type: geo_shape
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          shape:
+            type: "Polygon"
+            coordinates: [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          shape: "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2))"
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          shape: ["POINT (-77.03653 38.897676)", {"type" : "LineString", "coordinates" : [[-77.03653, 38.897676], [-77.009051, 38.889939]]}]
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.shape.type: "Polygon" }
+  - match: { _source.shape.coordinates: [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]] }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.shape: "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2))" }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.shape: ["POINT (-77.03653 38.897676)", {"type" : "LineString", "coordinates" : [[-77.03653, 38.897676], [-77.009051, 38.889939]]}] }
+
+---
+"geo_shape with ignore_malformed":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              shape:
+                type: geo_shape
+                ignore_malformed: true
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          shape: 500
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          shape:
+            string: "string"
+            array: [{ "a": 1 }, { "b": 2 }]
+            object: { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          shape: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"]
+
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          shape: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"]
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.shape: 500 }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.shape.string: "string" }
+  - match: { _source.shape.array: [{ "a": 1 }, { "b": 2 }] }
+  - match: { _source.shape.object: { "foo": "bar" } }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.shape: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"] }
+
+  - do:
+      get:
+        index: test
+        id: "4"
+
+  - match: { _source.shape: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"] }
+
+---
+"geo_point":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              point:
+                type: geo_point
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          point:
+            type: "Point"
+            coordinates: [-71.34, 41.12]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          point: "POINT (-71.34 41.12)"
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          point:
+            lat: 41.12
+            lon: -71.34
+
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          point: [ -71.34, 41.12 ]
+
+  - do:
+      index:
+        index: test
+        id: "5"
+        body:
+          point: "41.12,-71.34"
+
+  - do:
+      index:
+        index: test
+        id: "6"
+        body:
+          point: "drm3btev3e86"
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.point.lon: -71.34000004269183 }
+  - match: { _source.point.lat: 41.1199999647215 }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.point.lon: -71.34000004269183 }
+  - match: { _source.point.lat: 41.1199999647215 }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.point.lon: -71.34000004269183 }
+  - match: { _source.point.lat: 41.1199999647215 }
+
+  - do:
+      get:
+        index: test
+        id: "4"
+
+  - match: { _source.point.lon: -71.34000004269183 }
+  - match: { _source.point.lat: 41.1199999647215 }
+
+  - do:
+      get:
+        index: test
+        id: "5"
+
+  - match: { _source.point.lon: -71.34000004269183 }
+  - match: { _source.point.lat: 41.1199999647215 }
+
+  - do:
+      get:
+        index: test
+        id: "6"
+
+  - match: { _source.point.lon: -71.34000029414892 }
+  - match: { _source.point.lat: 41.119999922811985 }