Add back range support to `ip` fields. #17777

`ip` fields currently fail range queries when either bound is inclusive. This
commit makes ranges also work in the exclusive case to be consistent with other
data types.

buildSrc/src/main/resources/forbidden/es-all-signatures.txt

@@ -31,3 +31,5 @@ org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
 
 @defaultMessage Soon to be removed
 org.apache.lucene.document.FieldType#numericType()
+
+org.apache.lucene.document.InetAddressPoint#newPrefixQuery(java.lang.String, java.net.InetAddress, int) @LUCENE-7232

core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.SuppressForbidden;
+
+/**
+ * Forked utility methods from Lucene's InetAddressPoint until LUCENE-7232 and
+ * LUCENE-7234 are released.
+ */
+// TODO: remove me when we upgrade to Lucene 6.1
+@SuppressForbidden(reason="uses InetAddress.getHostAddress")
+public final class XInetAddressPoint {
+
+    private XInetAddressPoint() {}
+
+    /** The minimum value that an ip address can hold. */
+    public static final InetAddress MIN_VALUE;
+    /** The maximum value that an ip address can hold. */
+    public static final InetAddress MAX_VALUE;
+    static {
+      MIN_VALUE = InetAddressPoint.decode(new byte[InetAddressPoint.BYTES]);
+      byte[] maxValueBytes = new byte[InetAddressPoint.BYTES];
+      Arrays.fill(maxValueBytes, (byte) 0xFF);
+      MAX_VALUE = InetAddressPoint.decode(maxValueBytes);
+    }
+
+    /**
+     * Return the {@link InetAddress} that compares immediately greater than
+     * {@code address}.
+     * @throws ArithmeticException if the provided address is the
+     *              {@link #MAX_VALUE maximum ip address}
+     */
+    public static InetAddress nextUp(InetAddress address) {
+      if (address.equals(MAX_VALUE)) {
+        throw new ArithmeticException("Overflow: there is no greater InetAddress than "
+            + address.getHostAddress());
+      }
+      byte[] delta = new byte[InetAddressPoint.BYTES];
+      delta[InetAddressPoint.BYTES-1] = 1;
+      byte[] nextUpBytes = new byte[InetAddressPoint.BYTES];
+      NumericUtils.add(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextUpBytes);
+      return InetAddressPoint.decode(nextUpBytes);
+    }
+
+    /**
+     * Return the {@link InetAddress} that compares immediately less than
+     * {@code address}.
+     * @throws ArithmeticException if the provided address is the
+     *              {@link #MIN_VALUE minimum ip address}
+     */
+    public static InetAddress nextDown(InetAddress address) {
+      if (address.equals(MIN_VALUE)) {
+        throw new ArithmeticException("Underflow: there is no smaller InetAddress than "
+            + address.getHostAddress());
+      }
+      byte[] delta = new byte[InetAddressPoint.BYTES];
+      delta[InetAddressPoint.BYTES-1] = 1;
+      byte[] nextDownBytes = new byte[InetAddressPoint.BYTES];
+      NumericUtils.subtract(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextDownBytes);
+      return InetAddressPoint.decode(nextDownBytes);
+    }
+
+    /** 
+     * Create a prefix query for matching a CIDR network range.
+     *
+     * @param field field name. must not be {@code null}.
+     * @param value any host address
+     * @param prefixLength the network prefix length for this address. This is also known as the subnet mask in the context of IPv4
+     * addresses.
+     * @throws IllegalArgumentException if {@code field} is null, or prefixLength is invalid.
+     * @return a query matching documents with addresses contained within this network
+     */
+    // TODO: remove me when we upgrade to Lucene 6.0.1
+    public static Query newPrefixQuery(String field, InetAddress value, int prefixLength) {
+      if (value == null) {
+        throw new IllegalArgumentException("InetAddress must not be null");
+      }
+      if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) {
+        throw new IllegalArgumentException("illegal prefixLength '" + prefixLength
+                + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges");
+      }
+      // create the lower value by zeroing out the host portion, upper value by filling it with all ones.
+      byte lower[] = value.getAddress();
+      byte upper[] = value.getAddress();
+      for (int i = prefixLength; i < 8 * lower.length; i++) {
+        int m = 1 << (7 - (i & 7));
+        lower[i >> 3] &= ~m;
+        upper[i >> 3] |= m;
+      }
+      try {
+        return InetAddressPoint.newRangeQuery(field, InetAddress.getByAddress(lower), InetAddress.getByAddress(upper));
+      } catch (UnknownHostException e) {
+        throw new AssertionError(e); // values are coming from InetAddress
+      }
+    }
+}

core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java

@@ -23,9 +23,11 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.XInetAddressPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -176,7 +178,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
                     if (fields.length == 2) {
                         InetAddress address = InetAddresses.forString(fields[0]);
                         int prefixLength = Integer.parseInt(fields[1]);
-                        return InetAddressPoint.newPrefixQuery(name(), address, prefixLength);
+                        return XInetAddressPoint.newPrefixQuery(name(), address, prefixLength);
                     } else {
                         throw new IllegalArgumentException("Expected [ip/prefix] but was [" + term + "]");
                     }
@@ -188,24 +190,30 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
 
         @Override
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
-            if (includeLower == false || includeUpper == false) {
-                // TODO: should we drop range support entirely
-                throw new IllegalArgumentException("range on ip addresses only supports inclusive bounds");
-            }
             InetAddress lower;
             if (lowerTerm == null) {
-                lower = InetAddressPoint.decode(new byte[16]);
+                lower = XInetAddressPoint.MIN_VALUE;
             } else {
                 lower = parse(lowerTerm);
+                if (includeLower == false) {
+                    if (lower.equals(XInetAddressPoint.MAX_VALUE)) {
+                        return new MatchNoDocsQuery();
+                    }
+                    lower = XInetAddressPoint.nextUp(lower);
+                }
             }
 
             InetAddress upper;
             if (upperTerm == null) {
-                byte[] bytes = new byte[16];
-                Arrays.fill(bytes, (byte) 255); 
-                upper = InetAddressPoint.decode(bytes);
+                upper = XInetAddressPoint.MAX_VALUE;
             } else {
                 upper = parse(upperTerm);
+                if (includeUpper == false) {
+                    if (upper.equals(XInetAddressPoint.MIN_VALUE)) {
+                        return new MatchNoDocsQuery();
+                    }
+                    upper = XInetAddressPoint.nextDown(upper);
+                }
             }
 
             return InetAddressPoint.newRangeQuery(name(), lower, upper);
@@ -215,7 +223,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
         public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
             InetAddress base = parse(value);
             int mask = fuzziness.asInt();
-            return InetAddressPoint.newPrefixQuery(name(), base, mask);
+            return XInetAddressPoint.newPrefixQuery(name(), base, mask);
         }
 
         @Override

core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java

@@ -21,6 +21,8 @@ package org.elasticsearch.index.mapper.ip;
 import java.net.InetAddress;
 
 import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.XInetAddressPoint;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
@@ -66,10 +68,93 @@ public class IpFieldTypeTests extends FieldTypeTestCase {
 
         ip = "2001:db8::2:1";
         String prefix = ip + "/64";
-        assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null));
+        assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null));
 
         ip = "192.168.1.7";
         prefix = ip + "/16";
-        assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null));
+        assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null));
+    }
+
+    public void testRangeQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("::"),
+                        XInetAddressPoint.MAX_VALUE),
+                ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("::"),
+                        InetAddresses.forString("192.168.2.0")),
+                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("::"),
+                        InetAddresses.forString("192.168.1.255")),
+                ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("2001:db8::"),
+                        XInetAddressPoint.MAX_VALUE),
+                ft.rangeQuery("2001:db8::", null, true, randomBoolean()));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("2001:db8::1"),
+                        XInetAddressPoint.MAX_VALUE),
+                ft.rangeQuery("2001:db8::", null, false, randomBoolean()));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("2001:db8::"),
+                        InetAddresses.forString("2001:db8::ffff")),
+                ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("2001:db8::1"),
+                        InetAddresses.forString("2001:db8::fffe")),
+                ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("2001:db8::2"),
+                        InetAddresses.forString("2001:db8::")),
+                // same lo/hi values but inclusive=false so this won't match anything
+                ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false));
+
+        // Upper bound is the min IP and is not inclusive
+        assertEquals(new MatchNoDocsQuery(),
+                ft.rangeQuery("::", "::", true, false));
+
+        // Lower bound is the max IP and is not inclusive
+        assertEquals(new MatchNoDocsQuery(),
+                ft.rangeQuery("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("::"),
+                        InetAddresses.forString("::fffe:ffff:ffff")),
+                // same lo/hi values but inclusive=false so this won't match anything
+                ft.rangeQuery("::", "0.0.0.0", true, false));
+
+        assertEquals(
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("::1:0:0:0"),
+                        XInetAddressPoint.MAX_VALUE),
+                // same lo/hi values but inclusive=false so this won't match anything
+                ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true));
+
+        assertEquals(
+                // lower bound is ipv4, upper bound is ipv6
+                InetAddressPoint.newRangeQuery("field",
+                        InetAddresses.forString("192.168.1.7"),
+                        InetAddresses.forString("2001:db8::")),
+                ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true));
     }
 }