Add support for .empty to expressions, and some docs improvements

Closes #18077

docs/reference/modules/scripting/scripting.asciidoc

@@ -455,11 +455,25 @@ for details on what operators and functions are available.
 
 Variables in `expression` scripts are available to access:
 
-* Single valued document fields, e.g. `doc['myfield'].value`
-* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']`
+* document fields, e.g. `doc['myfield'].value` or just `doc['myfield']`.
+* whether the field is empty, e.g. `doc['myfield'].empty`
 * Parameters passed into the script, e.g. `mymodifier`
 * The current document's score, `_score` (only available when used in a `script_score`)
 
+When a document is missing the field completely, by default the value will be treated as `0`.
+You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value`
+
+When a document has multiple values for the field, by default the minimum value is returned.
+You can choose a different value instead, e.g. `doc['myfield'].sum()`. The following methods are available
+for any field:
+
+* min()
+* max()
+* avg()
+* median()
+* sum()
+* count() 
+
 Variables in `expression` scripts that are of type `date` may use the following member methods:
 
 * getYear()
@@ -477,7 +491,6 @@ There are a few limitations relative to other script languages:
 
 * Only numeric fields may be accessed
 * Stored fields are not available
-* If a field is sparse (only some documents contain a value), documents missing the field will have a value of `0`
 
 [float]
 === Score

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java

@@ -63,7 +63,7 @@ public class CountMethodValueSource extends ValueSource {
 
     @Override
     public int hashCode() {
-        return fieldData.hashCode();
+        return 31 * getClass().hashCode() + fieldData.hashCode();
     }
 
     @Override

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+
+/**
+ * ValueSource to return non-zero if a field is missing.
+ * <p>
+ * This is essentially sugar over !count()
+ */
+public class EmptyMemberValueSource extends ValueSource {
+    protected IndexFieldData<?> fieldData;
+
+    protected EmptyMemberValueSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        final SortedNumericDoubleValues values = leafData.getDoubleValues();
+        return new DoubleDocValues(this) {
+            @Override
+            public double doubleVal(int doc) {
+                values.setDocument(doc);
+                if (values.count() == 0) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        EmptyMemberValueSource other = (EmptyMemberValueSource) obj;
+        if (!fieldData.equals(other.fieldData)) return false;
+        return true;
+    }
+
+    @Override
+    public String description() {
+        return "empty: field(" + fieldData.getFieldName() + ")";
+    }
+}

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java

@@ -65,6 +65,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final List<String> TYPES = Collections.singletonList(NAME);
 
+    // these methods only work on dates, e.g. doc['datefield'].getYear()
     protected static final String GET_YEAR_METHOD         = "getYear";
     protected static final String GET_MONTH_METHOD        = "getMonth";
     protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
@@ -72,6 +73,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     protected static final String GET_MINUTES_METHOD      = "getMinutes";
     protected static final String GET_SECONDS_METHOD      = "getSeconds";
 
+    // these methods work on any field, e.g. doc['field'].sum()
     protected static final String MINIMUM_METHOD          = "min";
     protected static final String MAXIMUM_METHOD          = "max";
     protected static final String AVERAGE_METHOD          = "avg";
@@ -79,6 +81,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     protected static final String SUM_METHOD              = "sum";
     protected static final String COUNT_METHOD            = "count";
 
+    // these variables work on any field, e.g. doc['field'].value
+    protected static final String VALUE_VARIABLE          = "value";
+    protected static final String EMPTY_VARIABLE          = "empty";
+
     @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
@@ -169,6 +175,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 } else {
                     String fieldname = null;
                     String methodname = null;
+                    String variablename = VALUE_VARIABLE; // .value is the default for doc['field'], its optional.
                     VariableContext[] parts = VariableContext.parse(variable);
                     if (parts[0].text.equals("doc") == false) {
                         throw new ScriptException("Unknown variable [" + parts[0].text + "] in expression");
@@ -181,8 +188,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     if (parts.length == 3) {
                         if (parts[2].type == VariableContext.Type.METHOD) {
                             methodname = parts[2].text;
-                        } else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) {
-                            throw new ScriptException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly");
+                        } else if (parts[2].type == VariableContext.Type.MEMBER) {
+                            variablename = parts[2].text;
+                        } else {
+                            throw new ScriptException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
                         }
                     }
                     if (parts.length > 3) {
@@ -201,7 +210,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric");
                     }
                     if (methodname == null) {
-                        bindings.add(variable, new FieldDataValueSource(fieldData, MultiValueMode.MIN));
+                        bindings.add(variable, getVariableValueSource(fieldType, fieldData, fieldname, variablename));
                     } else {
                         bindings.add(variable, getMethodValueSource(fieldType, fieldData, fieldname, methodname));
                     }
@@ -245,6 +254,17 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 throw new IllegalArgumentException("Member method [" + methodName + "] does not exist.");
         }
     }
+    
+    protected ValueSource getVariableValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String memberName) {
+        switch (memberName) {
+            case VALUE_VARIABLE:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case EMPTY_VARIABLE:
+                return new EmptyMemberValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member variable [" + memberName + "] does not exist.");
+        }
+    }
 
     protected ValueSource getDateMethodValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String methodName, int calendarType) {
         if (fieldType instanceof LegacyDateFieldMapper.DateFieldType == false

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java

@@ -164,10 +164,10 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testMultiValueMethods() throws Exception {
-        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double"));
+        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double", "double2", "type=double"));
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4"),
+                client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4", "double2", "3.0"),
                 client().prepareIndex("test", "doc", "2").setSource("double0", "5.0", "double1", "3.0"),
                 client().prepareIndex("test", "doc", "3").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double0", "-1.5", "double1", "4.0"));
 
@@ -227,6 +227,24 @@ public class MoreExpressionTests extends ESIntegTestCase {
         assertEquals(2.5, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         assertEquals(1.5, hits.getAt(2).field("foo").getValue(), 0.0D);
+        
+        // make sure count() works for missing
+        rsp = buildRequest("doc['double2'].count()").get();
+        assertSearchResponse(rsp);
+        hits = rsp.getHits();
+        assertEquals(3, hits.getTotalHits());
+        assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(0.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        assertEquals(0.0, hits.getAt(2).field("foo").getValue(), 0.0D);
+        
+        // make sure .empty works in the same way
+        rsp = buildRequest("doc['double2'].empty ? 5.0 : 2.0").get();
+        assertSearchResponse(rsp);
+        hits = rsp.getHits();
+        assertEquals(3, hits.getTotalHits());
+        assertEquals(2.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D);
     }
 
     public void testInvalidDateMethodCall() throws Exception {
@@ -363,8 +381,8 @@ public class MoreExpressionTests extends ESIntegTestCase {
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained ScriptException",
                     e.toString().contains("ScriptException"), equalTo(true));
-            assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed",
-                    e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true));
+            assertThat(e.toString() + "should have contained member variable [bogus] does not exist",
+                    e.toString().contains("Member variable [bogus] does not exist"), equalTo(true));
         }
     }