Add support for Well Known Binary (WKB) in the fields API for spatial fields (#103461)

docs/changelog/103461.yaml

@@ -0,0 +1,5 @@
+pr: 103461
+summary: Add support for Well Known Binary (WKB) in the fields API for spatial fields
+area: Geo
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/common/geo/GeometryFormatterFactory.java

@@ -9,8 +9,10 @@
 package org.elasticsearch.common.geo;
 
 import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
 import org.elasticsearch.geometry.utils.WellKnownText;
 
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Function;
@@ -22,6 +24,7 @@ public class GeometryFormatterFactory {
 
     public static final String GEOJSON = "geojson";
     public static final String WKT = "wkt";
+    public static final String WKB = "wkb";
 
     /**
      * Returns a formatter by name
@@ -38,6 +41,11 @@ public class GeometryFormatterFactory {
                 geometries.forEach((shape) -> objects.add(WellKnownText.toWKT(toGeometry.apply(shape))));
                 return objects;
             };
+            case WKB -> geometries -> {
+                final List<Object> objects = new ArrayList<>(geometries.size());
+                geometries.forEach((shape) -> objects.add(WellKnownBinary.toWKB(toGeometry.apply(shape), ByteOrder.LITTLE_ENDIAN)));
+                return objects;
+            };
             default -> throw new IllegalArgumentException("Unrecognized geometry format [" + name + "].");
         };
     }

server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java

@@ -11,15 +11,19 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.tests.geo.GeoTestUtil;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.SimpleFeatureFactory;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.script.ScriptCompiler;
-import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class GeoPointFieldTypeTests extends FieldTypeTestCase {
 
     public void testFetchSourceValue() throws IOException {
@@ -36,31 +40,50 @@ public class GeoPointFieldTypeTests extends FieldTypeTestCase {
         Map<String, Object> otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0));
         String wktPoint = "POINT (42.0 27.1)";
         String otherWktPoint = "POINT (30.0 50.0)";
+        byte[] wkbPoint = WellKnownBinary.toWKB(new Point(42.0, 27.1), ByteOrder.LITTLE_ENDIAN);
+        byte[] otherWkbPoint = WellKnownBinary.toWKB(new Point(30.0, 50.0), ByteOrder.LITTLE_ENDIAN);
 
         // Test a single point in [lon, lat] array format.
         Object sourceValue = List.of(42.0, 27.1);
         assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        List<?> wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
 
         // Test a single point in "lat, lon" string format.
         sourceValue = "27.1,42.0";
         assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
 
         // Test a list of points in [lon, lat] array format.
         sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0));
         assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
+        assertThat(wkb.get(1), equalTo(otherWkbPoint));
 
         // Test a list of points in [lat,lon] array format with one malformed
         sourceValue = List.of(List.of(42.0, 27.1), List.of("a", "b"), List.of(30.0, 50.0));
         assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
+        assertThat(wkb.get(1), equalTo(otherWkbPoint));
 
         // Test a single point in well-known text format.
         sourceValue = "POINT (42.0 27.1)";
         assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
 
         // Test a malformed value
         sourceValue = "malformed";
@@ -71,9 +94,13 @@ public class GeoPointFieldTypeTests extends FieldTypeTestCase {
         if (ignoreMalformed) {
             assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
             assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+            wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+            assertThat(wkb.size(), equalTo(1));
+            assertThat(wkb.get(0), equalTo(wkbPoint));
         } else {
             assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, null));
             assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkt"));
+            assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkb"));
         }
 
         // test single point in GeoJSON format
@@ -110,13 +137,13 @@ public class GeoPointFieldTypeTests extends FieldTypeTestCase {
             final double lat = GeoTestUtil.nextLatitude();
             final double lon = GeoTestUtil.nextLongitude();
             List<?> sourceValue = fetchSourceValue(mapper, List.of(lon, lat), mvtString);
-            assertThat(sourceValue.size(), Matchers.equalTo(1));
-            assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.point(lon, lat)));
+            assertThat(sourceValue.size(), equalTo(1));
+            assertThat(sourceValue.get(0), equalTo(featureFactory.point(lon, lat)));
             geoPoints.add(new GeoPoint(lat, lon));
             values.add(List.of(lon, lat));
         }
         List<?> sourceValue = fetchSourceValue(mapper, values, mvtString);
-        assertThat(sourceValue.size(), Matchers.equalTo(1));
-        assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.points(geoPoints)));
+        assertThat(sourceValue.size(), equalTo(1));
+        assertThat(sourceValue.get(0), equalTo(featureFactory.points(geoPoints)));
     }
 }

x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldTypeTests.java

@@ -25,16 +25,18 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.xpack.vectortile.SpatialGeometryFormatterExtension;
 import org.elasticsearch.xpack.vectortile.feature.FeatureFactory;
-import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.nio.ByteOrder;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
 public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase {
 
-    public void testFetchSourceValue() throws IOException {
+    public void testFetchSourceValue() throws Exception {
         final GeoFormatterFactory<Geometry> geoFormatterFactory = new GeoFormatterFactory<>(
             new SpatialGeometryFormatterExtension().getGeometryFormatterFactories()
         );
@@ -53,26 +55,43 @@ public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase {
         String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)";
         String wktPoint = "POINT (14.0 15.0)";
         String wktMalformed = "POINT foo";
+        byte[] wkbLine = WellKnownBinary.toWKB(
+            WellKnownText.fromWKT(StandardValidator.NOOP, false, wktLineString),
+            ByteOrder.LITTLE_ENDIAN
+        );
+        byte[] wkbPoint = WellKnownBinary.toWKB(WellKnownText.fromWKT(StandardValidator.NOOP, false, wktPoint), ByteOrder.LITTLE_ENDIAN);
 
         // Test a single shape in geojson format.
         Object sourceValue = jsonLineString;
         assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt"));
+        List<?> wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbLine));
 
         // Test a malformed single shape in geojson format
         sourceValue = jsonMalformed;
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkt"));
+        assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkb"));
 
         // Test a list of shapes in geojson format.
         sourceValue = List.of(jsonLineString, jsonPoint);
         assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbLine));
+        assertThat(wkb.get(1), equalTo(wkbPoint));
 
         // Test a list of shapes including one malformed in geojson format
         sourceValue = List.of(jsonLineString, jsonMalformed, jsonPoint);
         assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbLine));
+        assertThat(wkb.get(1), equalTo(wkbPoint));
 
         // Test a single shape in wkt format.
         sourceValue = wktLineString;
@@ -109,26 +128,31 @@ public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase {
             geoFormatterFactory
         ).setStored(true).build(MapperBuilderContext.root(randomBoolean(), false)).fieldType();
 
-        ByteOrder byteOrder = randomBoolean() ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
-
         Map<String, Object> jsonLineString = Map.of("type", "LineString", "coordinates", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)));
         Map<String, Object> jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.0, 15.0));
         String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)";
         String wktPoint = "POINT (14.0 15.0)";
 
         BytesRef wkbLineString = new BytesRef(
-            WellKnownBinary.toWKB(new Line(new double[] { 42.0, 30.0 }, new double[] { 27.1, 50.0 }), byteOrder)
+            WellKnownBinary.toWKB(new Line(new double[] { 42.0, 30.0 }, new double[] { 27.1, 50.0 }), ByteOrder.LITTLE_ENDIAN)
         );
-        BytesRef wkbPoint = new BytesRef(WellKnownBinary.toWKB(new Point(14.0, 15.0), byteOrder));
+        BytesRef wkbPoint = new BytesRef(WellKnownBinary.toWKB(new Point(14.0, 15.0), ByteOrder.LITTLE_ENDIAN));
         // Test a single shape in wkb format.
         List<Object> storedValues = List.of(wkbLineString);
         assertEquals(List.of(jsonLineString), fetchStoredValue(mapper, storedValues, null));
         assertEquals(List.of(wktLineString), fetchStoredValue(mapper, storedValues, "wkt"));
+        List<?> wkb = fetchStoredValue(mapper, storedValues, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbLineString.bytes));
 
         // Test a list of shapes in wkb format.
         storedValues = List.of(wkbLineString, wkbPoint);
         assertEquals(List.of(jsonLineString, jsonPoint), fetchStoredValue(mapper, storedValues, null));
         assertEquals(List.of(wktLineString, wktPoint), fetchStoredValue(mapper, storedValues, "wkt"));
+        wkb = fetchStoredValue(mapper, storedValues, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbLineString.bytes));
+        assertThat(wkb.get(1), equalTo(wkbPoint.bytes));
     }
 
     public void testFetchVectorTile() throws IOException {
@@ -180,9 +204,9 @@ public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase {
             // happen that the geometry is out of range (close to the poles).
             features = List.of();
         }
-        assertThat(features.size(), Matchers.equalTo(sourceValue.size()));
+        assertThat(features.size(), equalTo(sourceValue.size()));
         for (int i = 0; i < features.size(); i++) {
-            assertThat(sourceValue.get(i), Matchers.equalTo(features.get(i)));
+            assertThat(sourceValue.get(i), equalTo(features.get(i)));
         }
     }
 
@@ -308,10 +332,10 @@ public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase {
         final int extent = randomIntBetween(256, 4096);
         List<?> mvtExpected = fetchSourceValue(mapper, mvtEquivalentAsWKT, "mvt(0/0/0@" + extent + ")");
         List<?> mvt = fetchSourceValue(mapper, sourceValue, "mvt(0/0/0@" + extent + ")");
-        assertThat(mvt.size(), Matchers.equalTo(1));
-        assertThat(mvt.size(), Matchers.equalTo(mvtExpected.size()));
-        assertThat(mvtExpected.get(0), Matchers.instanceOf(byte[].class));
-        assertThat(mvt.get(0), Matchers.instanceOf(byte[].class));
-        assertThat((byte[]) mvt.get(0), Matchers.equalTo((byte[]) mvtExpected.get(0)));
+        assertThat(mvt.size(), equalTo(1));
+        assertThat(mvt.size(), equalTo(mvtExpected.size()));
+        assertThat(mvtExpected.get(0), instanceOf(byte[].class));
+        assertThat(mvt.get(0), instanceOf(byte[].class));
+        assertThat((byte[]) mvt.get(0), equalTo((byte[]) mvtExpected.get(0)));
     }
 }

x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java

@@ -7,14 +7,19 @@
 
 package org.elasticsearch.xpack.spatial.index.mapper;
 
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 
 import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class PointFieldTypeTests extends FieldTypeTestCase {
 
     public void testFetchSourceValue() throws IOException {
@@ -24,26 +29,39 @@ public class PointFieldTypeTests extends FieldTypeTestCase {
         String wktPoint = "POINT (42.0 27.1)";
         Map<String, Object> otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0));
         String otherWktPoint = "POINT (30.0 50.0)";
+        byte[] wkbPoint = WellKnownBinary.toWKB(new Point(42.0, 27.1), ByteOrder.LITTLE_ENDIAN);
+        byte[] otherWkbPoint = WellKnownBinary.toWKB(new Point(30.0, 50.0), ByteOrder.LITTLE_ENDIAN);
 
         // Test a single point in [x, y] array format.
         Object sourceValue = List.of(42.0, 27.1);
         assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        List<?> wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
 
         // Test a single point in "x, y" string format.
         sourceValue = "42.0,27.1";
         assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
 
         // Test a malformed single point
         sourceValue = "foo";
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkt"));
+        assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkb"));
 
         // Test a list of points in [x, y] array format.
         sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0));
         assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbPoint));
+        assertThat(wkb.get(1), equalTo(otherWkbPoint));
 
         // Test a single point in well-known text format.
         sourceValue = "POINT (42.0 27.1)";

x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldTypeTests.java

@@ -7,18 +7,23 @@
 
 package org.elasticsearch.xpack.spatial.index.mapper;
 
+import org.elasticsearch.geometry.utils.StandardValidator;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 
-import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class ShapeFieldTypeTests extends FieldTypeTestCase {
 
-    public void testFetchSourceValue() throws IOException {
+    public void testFetchSourceValue() throws Exception {
         MappedFieldType mapper = new ShapeFieldMapper.Builder("field", IndexVersion.current(), false, true).build(
             MapperBuilderContext.root(false, false)
         ).fieldType();
@@ -29,26 +34,43 @@ public class ShapeFieldTypeTests extends FieldTypeTestCase {
         String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)";
         String wktPoint = "POINT (14.3 15.0)";
         String wktMalformed = "POINT foo";
+        byte[] wkbLine = WellKnownBinary.toWKB(
+            WellKnownText.fromWKT(StandardValidator.NOOP, false, wktLineString),
+            ByteOrder.LITTLE_ENDIAN
+        );
+        byte[] wkbPoint = WellKnownBinary.toWKB(WellKnownText.fromWKT(StandardValidator.NOOP, false, wktPoint), ByteOrder.LITTLE_ENDIAN);
 
         // Test a single shape in geojson format.
         Object sourceValue = jsonLineString;
         assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt"));
+        List<?> wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(1));
+        assertThat(wkb.get(0), equalTo(wkbLine));
 
         // Test a malformed single shape in geojson format
         sourceValue = jsonMalformed;
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkt"));
+        assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, "wkb"));
 
         // Test a list of shapes in geojson format.
         sourceValue = List.of(jsonLineString, jsonPoint);
         assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbLine));
+        assertThat(wkb.get(1), equalTo(wkbPoint));
 
         // Test a list of shapes including one malformed in geojson format
         sourceValue = List.of(jsonLineString, jsonMalformed, jsonPoint);
         assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null));
         assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt"));
+        wkb = fetchSourceValue(mapper, sourceValue, "wkb");
+        assertThat(wkb.size(), equalTo(2));
+        assertThat(wkb.get(0), equalTo(wkbLine));
+        assertThat(wkb.get(1), equalTo(wkbPoint));
 
         // Test a single shape in wkt format.
         sourceValue = wktLineString;