Script: Write Field API with basic path resolution (#89738)

Adds `WriteField` to the ingest context via `field(<path>)`.

`WriteField` implements APIs:

* `String` `getName()`: The path
 * `boolean` `exists()`: Does the path exist in the document?
 * `WriteField` `set(def)`: Set the value at the path, creates nested path elements if necessary
 * `WriteField` `append(def)`: Appends value to the path, creates nested path elements if necessary, the value at path is always a List after this call?
 * `boolean` `isEmpty()`: Does the path contain a value?
 * `int` `size()`: How many elements does the path contain?
 * `Iterator` `iterator()`: Iterate over all elements at the path.
 * `def` `get(def`): Get the value at the path if it exists, otherwise return the given default
 * `def` `get(int, def)`: Get the value at the path and index if it exists, otherwise return the given default
 * `boolean` `hasValue(Predicate)`: Is there a value at the path that passes the filter?
 * `WriteField` `transform(Function)`: Change all values at the path
 * `WriteField` `deduplicate()`: Remove duplicates from the path
 * `WriteField` `removeValuesIf(Predicate)`: Remove a values from the path that pass the filter
 * `WriteField` `removeValue(int)`: Remove the index from the path, if it exists

Some APIs remain unimplemented:
 * `void` `move(String)`
 * `void` `overwrite(String)`
 * `void` `remove()`

This change does not handle equivalent paths, which are paths that differ in the source but flatten to the same field in the indexed document.

Path resolution is basic, each path element is assumed to be a key in the current container, starting with the root.  If there is not an entry in the map at a given level, the algorithm checks to see if the remaining path exists as a flat key.  The nested then flat nature algorithm handles the common case of some or none nesting followed by flat keys.

Refs: #79155

docs/changelog/89738.yaml

@@ -0,0 +1,5 @@
+pr: 89738
+summary: "Script: Write Field API with basic path resolution"
+area: Infra/Scripting
+type: enhancement
+issues: []

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml

@@ -357,3 +357,83 @@ teardown:
   - match: { _source.source_field: "bazqux" }
   - match: { _version: 5 }
   - match: { _routing: "myRouting" }
+
+---
+"Write Fields API":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "fields api",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "field('create.depth').set('works');"
+                }
+              },
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "field('append').append('another')"
+                }
+              },
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "def a = field('append'); int i = 0; for (def value : a) { i += value.length(); } field('size').set(100 * a.size() + i);"
+                }
+              },
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "Collections.sort(field('duplicate').deduplicate().get(new ArrayList()));"
+                }
+              },
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "field('numbers').removeValuesIf(x -> x == null).transform(x -> x instanceof String ? Integer.parseInt(x) : x);"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            properties:
+              append:
+                type: keyword
+              duplicate:
+                type: keyword
+              numbers:
+                type: integer
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body:
+          append: ["one", "again", "another", "again"]
+          duplicate: ["aa", "aa", "bb"]
+          numbers: [1, null, "2"]
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.create.depth: "works" }
+  - match: { _source.size: 527 }
+  - match: { _source.duplicate: ["aa", "bb"] }
+  - match: { _source.numbers: [1, 2] }

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt

@@ -40,4 +40,25 @@ class org.elasticsearch.script.Metadata {
 
 class org.elasticsearch.script.IngestScript {
     Metadata metadata()
+    WriteField field(String)
+}
+
+class org.elasticsearch.script.field.WriteField {
+    String getName()
+    boolean exists()
+    void move(String)
+    void overwrite(String)
+    void remove()
+    WriteField set(def)
+    WriteField append(def)
+    boolean isEmpty()
+    int size()
+    Iterator iterator()
+    def get(def)
+    def get(int, def)
+    boolean hasValue(Predicate)
+    WriteField transform(Function)
+    WriteField deduplicate()
+    WriteField removeValuesIf(Predicate)
+    WriteField removeValue(int)
 }

server/src/main/java/org/elasticsearch/script/WriteScript.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.script.field.WriteField;
+
 import java.util.Map;
 
 /**
@@ -30,4 +32,8 @@ public abstract class WriteScript {
     public Metadata metadata() {
         return ctxMap.getMetadata();
     }
+
+    public WriteField field(String path) {
+        return new WriteField(path, ctxMap::getSource);
+    }
 }

server/src/main/java/org/elasticsearch/script/field/WriteField.java

@@ -0,0 +1,403 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field;
+
+import org.elasticsearch.common.util.Maps;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+public class WriteField implements Field<Object> {
+    protected String path;
+    protected final Supplier<Map<String, Object>> rootSupplier;
+
+    protected Map<String, Object> container;
+    protected String leaf;
+
+    private static final Object MISSING = new Object();
+
+    public WriteField(String path, Supplier<Map<String, Object>> rootSupplier) {
+        this.path = path;
+        this.rootSupplier = rootSupplier;
+        resolveDepthFlat();
+    }
+
+    // Path Read
+
+    /**
+     * Get the path represented by this Field
+     */
+    public String getName() {
+        return path;
+    }
+
+    /**
+     * Does the path exist?
+     */
+    public boolean exists() {
+        return leaf != null && container.containsKey(leaf);
+    }
+
+    // Path Update
+
+    /**
+     * Move this path to another path in the map.
+     *
+     * @throws IllegalArgumentException if the other path has contents
+     * @throws UnsupportedOperationException
+     */
+    public void move(String path) {
+        throw new UnsupportedOperationException("unimplemented");
+    }
+
+    /**
+     * Move this path to another path in the map, overwriting the destination path if it exists
+     *
+     * @throws UnsupportedOperationException
+     */
+    public void overwrite(String path) {
+        throw new UnsupportedOperationException("unimplemented");
+    }
+
+    // Path Delete
+
+    /**
+     * Removes this path from the map.
+     *
+     * @throws UnsupportedOperationException
+     */
+    public void remove() {
+        throw new UnsupportedOperationException("unimplemented");
+    }
+
+    // Value Create
+
+    /**
+     * Sets the value for this path.  Creates nested path if necessary.
+     */
+    public WriteField set(Object value) {
+        setLeaf();
+        container.put(leaf, value);
+        return this;
+    }
+
+    /**
+     * Appends a value to this path.  Creates the path and the List at the leaf if necessary.
+     */
+    @SuppressWarnings("unchecked")
+    public WriteField append(Object value) {
+        setLeaf();
+
+        container.compute(leaf, (k, v) -> {
+            List<Object> values;
+            if (v == null) {
+                values = new ArrayList<>(4);
+            } else if (v instanceof List<?> list) {
+                values = (List<Object>) list;
+            } else {
+                values = new ArrayList<>(4);
+                values.add(v);
+            }
+            values.add(value);
+            return values;
+        });
+        return this;
+    }
+
+    // Value Read
+
+    /**
+     * Is this path associated with any values?
+     */
+    @Override
+    public boolean isEmpty() {
+        return size() == 0;
+    }
+
+    /**
+     * How many elements are at the leaf of this path?
+     */
+    @Override
+    public int size() {
+        if (leaf == null) {
+            return 0;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return 0;
+        }
+
+        if (value instanceof List<?> list) {
+            return list.size();
+        }
+        return 1;
+    }
+
+    /**
+     * Iterate through all elements of this path
+     */
+    @Override
+    @SuppressWarnings("unchecked")
+    public Iterator<Object> iterator() {
+        if (leaf == null) {
+            return Collections.emptyIterator();
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return Collections.emptyIterator();
+        }
+
+        if (value instanceof List<?> list) {
+            return (Iterator<Object>) list.iterator();
+        }
+        return Collections.singleton(value).iterator();
+    }
+
+    /**
+     * Get the value at this path, if there is no value then get the provided {@param defaultValue}
+     */
+    public Object get(Object defaultValue) {
+        if (leaf == null) {
+            return defaultValue;
+        }
+
+        return container.getOrDefault(leaf, defaultValue);
+    }
+
+    /**
+     * Get the value at the given index at this path or {@param defaultValue} if there is no such value.
+     */
+    public Object get(int index, Object defaultValue) {
+        if (leaf == null) {
+            return defaultValue;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value instanceof List<?> list) {
+            if (index < list.size()) {
+                return list.get(index);
+            }
+        } else if (value != MISSING && index == 0) {
+            return value;
+        }
+
+        return defaultValue;
+    }
+
+    /**
+     * Is there any value matching {@param predicate} at this path?
+     */
+    public boolean hasValue(Predicate<Object> predicate) {
+        if (leaf == null) {
+            return false;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return false;
+        }
+
+        if (value instanceof List<?> list) {
+            return list.stream().anyMatch(predicate);
+        }
+
+        return predicate.test(value);
+    }
+
+    // Value Update
+
+    /**
+     * Update each value at this path with the {@param transformer} {@link Function}.
+     */
+    @SuppressWarnings("unchecked")
+    public WriteField transform(Function<Object, Object> transformer) {
+        if (leaf == null) {
+            return this;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return this;
+        }
+
+        if (value instanceof List<?> list) {
+            ((List<Object>) list).replaceAll(transformer::apply);
+        } else {
+            container.put(leaf, transformer.apply(value));
+        }
+
+        return this;
+    }
+
+    // Value Delete
+
+    /**
+     * Remove all duplicate values from this path.  List order is not preserved.
+     */
+    @SuppressWarnings("unchecked")
+    public WriteField deduplicate() {
+        if (leaf == null) {
+            return this;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return this;
+        }
+
+        if (value instanceof List<?> list) {
+            // Assume modifiable list
+            Set<Object> set = new HashSet<>(list);
+            list.clear();
+            ((List<Object>) list).addAll(set);
+        }
+
+        return this;
+    }
+
+    /**
+     * Remove all values at this path that match {@param filter}.  If there is only one value and it matches {@param filter},
+     * the mapping is removed, however empty Lists are retained.
+     */
+    public WriteField removeValuesIf(Predicate<Object> filter) {
+        if (leaf == null) {
+            return this;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return this;
+        }
+
+        if (value instanceof List<?> list) {
+            list.removeIf(filter);
+        } else if (filter.test(value)) {
+            container.remove(leaf);
+        }
+
+        return this;
+    }
+
+    /**
+     * Remove the value at {@param index}, if it exists.  If there is only one value and {@param index} is zero, remove the
+     * mapping.
+     */
+    public WriteField removeValue(int index) {
+        if (leaf == null) {
+            return this;
+        }
+
+        Object value = container.getOrDefault(leaf, MISSING);
+        if (value == MISSING) {
+            return this;
+        }
+
+        if (value instanceof List<?> list) {
+            if (index < list.size()) {
+                list.remove(index);
+            }
+        } else if (index == 0) {
+            container.remove(leaf);
+        }
+
+        return this;
+    }
+
+    /**
+     * Get the path to a leaf or create it if one does not exist.
+     */
+    protected void setLeaf() {
+        if (leaf == null) {
+            resolveDepthFlat();
+        }
+        if (leaf == null) {
+            createDepth();
+        }
+    }
+
+    /**
+     * Resolve {@link #path} from the root.
+     *
+     * Tries to resolve the path one segment at a time, if the segment is not mapped to a Java Map, then
+     * treats that segment and the rest as the leaf if it resolves.
+     *
+     * a.b.c could be resolved as
+     *  I)   ['a']['b']['c'] if 'a' is a Map at the root and 'b' is a Map in 'a', 'c' need not exist in 'b'.
+     *  II)  ['a']['b.c'] if 'a' is a Map at the root and 'b' does not exist in 'a's Map but 'b.c' does.
+     *  III) ['a.b.c'] if 'a' doesn't exist at the root but 'a.b.c' does.
+     *
+     * {@link #container} and {@link #leaf} and non-null if resolved.
+     */
+    @SuppressWarnings("unchecked")
+    protected void resolveDepthFlat() {
+        container = rootSupplier.get();
+
+        int index = path.indexOf('.');
+        int lastIndex = 0;
+        String segment;
+
+        while (index != -1) {
+            segment = path.substring(lastIndex, index);
+            Object value = container.get(segment);
+            if (value instanceof Map<?, ?> map) {
+                container = (Map<String, Object>) map;
+                lastIndex = index + 1;
+                index = path.indexOf('.', lastIndex);
+            } else {
+                // Check rest of segments as a single key
+                String rest = path.substring(lastIndex);
+                if (container.containsKey(rest)) {
+                    leaf = rest;
+                } else {
+                    leaf = null;
+                }
+                return;
+            }
+        }
+        leaf = path.substring(lastIndex);
+    }
+
+    /**
+     * Create a new Map for each segment in path, if that segment is unmapped or mapped to null.
+     *
+     * @throws IllegalArgumentException if a non-leaf segment maps to a non-Map Object.
+     */
+    @SuppressWarnings("unchecked")
+    protected void createDepth() {
+        container = rootSupplier.get();
+
+        String[] segments = path.split("\\.");
+        for (int i = 0; i < segments.length - 1; i++) {
+            String segment = segments[i];
+            Object value = container.get(segment);
+            if (value instanceof Map<?, ?> map) {
+                container = (Map<String, Object>) map;
+            } else if (value == null) {
+                Map<String, Object> next = Maps.newHashMapWithExpectedSize(4);
+                container.put(segment, next);
+                container = next;
+            } else {
+                throw new IllegalArgumentException(
+                    "Segment [" + i + ":'" + segment + "'] has value [" + value + "] of type [" + value.getClass().getName() + "]"
+                );
+            }
+        }
+        leaf = segments[segments.length - 1];
+    }
+}

server/src/test/java/org/elasticsearch/script/field/WriteFieldTests.java

@@ -0,0 +1,332 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.hamcrest.Matchers.contains;
+
+public class WriteFieldTests extends ESTestCase {
+
+    public void testResolveDepthFlat() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("abc.d.ef", "flat");
+
+        Map<String, Object> abc = new HashMap<>();
+        map.put("abc", abc);
+        abc.put("d.ef", "mixed");
+
+        Map<String, Object> d = new HashMap<>();
+        abc.put("d", d);
+        d.put("ef", "nested");
+
+        // { "abc.d.ef", "flat", "abc": { "d.ef": "mixed", "d": { "ef": "nested" } } }
+        WriteField wf = new WriteField("abc.d.ef", () -> map);
+        assertTrue(wf.exists());
+
+        assertEquals("nested", wf.get("missing"));
+        // { "abc.d.ef", "flat", "abc": { "d.ef": "mixed", "d": { } } }
+        d.remove("ef");
+        assertEquals("missing", wf.get("missing"));
+        // { "abc.d.ef", "flat", "abc": { "d.ef": "mixed" }
+        // TODO(stu): this should be inaccessible
+        abc.remove("d");
+        assertEquals("missing", wf.get("missing"));
+
+        // resolution at construction time
+        wf = new WriteField("abc.d.ef", () -> map);
+        assertEquals("mixed", wf.get("missing"));
+        abc.remove("d.ef");
+        assertEquals("missing", wf.get("missing"));
+
+        wf = new WriteField("abc.d.ef", () -> map);
+        // abc is still there
+        assertEquals("missing", wf.get("missing"));
+        map.remove("abc");
+        assertEquals("missing", wf.get("missing"));
+
+        wf = new WriteField("abc.d.ef", () -> map);
+        assertEquals("flat", wf.get("missing"));
+    }
+
+    public void testExists() {
+        Map<String, Object> a = new HashMap<>();
+        a.put("b.c", null);
+        assertTrue(new WriteField("a.b.c", () -> Map.of("a", a)).exists());
+
+        a.clear();
+        Map<String, Object> level1 = new HashMap<>();
+        level1.put("null", null);
+        a.put("level1", level1);
+        a.put("null", null);
+        // WriteField.leaf is null
+        assertFalse(new WriteField("missing.leaf", () -> a).exists());
+
+        // WriteField.leaf non-null but missing
+        assertFalse(new WriteField("missing", () -> a).exists());
+
+        // Check mappings with null values exist
+        assertTrue(new WriteField("level1.null", () -> a).exists());
+        assertTrue(new WriteField("null", () -> a).exists());
+    }
+
+    public void testMove() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a.b.c", "foo");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        UnsupportedOperationException err = expectThrows(UnsupportedOperationException.class, () -> wf.move("b.c.d"));
+        assertEquals("unimplemented", err.getMessage());
+    }
+
+    public void testOverwrite() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a.b.c", "foo");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        UnsupportedOperationException err = expectThrows(UnsupportedOperationException.class, () -> wf.overwrite("b.c.d"));
+        assertEquals("unimplemented", err.getMessage());
+    }
+
+    public void testRemove() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a.b.c", "foo");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        UnsupportedOperationException err = expectThrows(UnsupportedOperationException.class, wf::remove);
+        assertEquals("unimplemented", err.getMessage());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSet() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a.b.c", "foo");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        wf.set("bar");
+        assertEquals("bar", root.get("a.b.c"));
+
+        root.clear();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        wf = new WriteField("a.b.c", () -> root);
+        wf.set("bar");
+        assertEquals("bar", b.get("c"));
+
+        a.clear();
+        wf = new WriteField("a.b.c", () -> root);
+        wf.set("bar");
+        assertEquals("bar", ((Map<String, Object>) a.get("b")).get("c"));
+
+        a.clear();
+        a.put("b", "foo");
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> new WriteField("a.b.c", () -> root).set("bar"));
+        assertEquals("Segment [1:'b'] has value [foo] of type [java.lang.String]", err.getMessage());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSetCreate() {
+        Map<String, Object> root = new HashMap<>();
+        WriteField wf = new WriteField("a.b", () -> root);
+        wf.set("foo");
+        assertThat(root.keySet(), contains("a"));
+        assertThat(((Map<String, Object>) root.get("a")).keySet(), contains("b"));
+    }
+
+    public void testAppend() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a.b.c", "foo");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        wf.append("bar");
+        assertEquals(List.of("foo", "bar"), root.get("a.b.c"));
+
+        root.clear();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        wf = new WriteField("a.b.c", () -> root);
+        wf.append("bar");
+        assertEquals(new ArrayList<>(List.of("bar")), b.get("c"));
+    }
+
+    public void testSizeIsEmpty() {
+        Map<String, Object> root = new HashMap<>();
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        assertTrue(wf.isEmpty());
+        assertEquals(0, wf.size());
+
+        root.put("a.b.c", List.of(1, 2));
+        wf = new WriteField("a.b.c", () -> root);
+        assertFalse(wf.isEmpty());
+        assertEquals(2, wf.size());
+
+        Map<String, Object> d = new HashMap<>();
+        root.put("d", d);
+        wf = new WriteField("d.e", () -> root);
+        assertTrue(wf.isEmpty());
+        assertEquals(0, wf.size());
+        d.put("e", "foo");
+        assertFalse(wf.isEmpty());
+        assertEquals(1, wf.size());
+    }
+
+    public void testIterator() {
+        Map<String, Object> root = new HashMap<>();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        assertFalse(wf.iterator().hasNext());
+
+        b.put("c", "value");
+        Iterator<Object> it = wf.iterator();
+        assertTrue(it.hasNext());
+        assertEquals("value", it.next());
+        assertFalse(it.hasNext());
+
+        b.put("c", List.of(1, 2, 3));
+        it = wf.iterator();
+        assertTrue(it.hasNext());
+        assertEquals(1, it.next());
+        assertTrue(it.hasNext());
+        assertEquals(2, it.next());
+        assertTrue(it.hasNext());
+        assertEquals(3, it.next());
+        assertFalse(it.hasNext());
+
+        assertFalse(new WriteField("dne.dne", () -> root).iterator().hasNext());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testDeduplicate() {
+        Map<String, Object> root = new HashMap<>();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        b.put("c", new ArrayList<>(List.of(1, 1, 1, 2, 2, 2)));
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        assertEquals(6, wf.size());
+        wf.deduplicate();
+        assertEquals(2, wf.size());
+        List<Object> list = (List<Object>) wf.get(Collections.emptyList());
+        assertTrue(list.contains(1));
+        assertTrue(list.contains(2));
+
+        assertEquals("missing", new WriteField("d.e", () -> root).deduplicate().get("missing"));
+        assertEquals("missing", new WriteField("a.b.d", () -> root).deduplicate().get("missing"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testTransform() {
+        Map<String, Object> root = new HashMap<>();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        b.put("c", new ArrayList<>(List.of(1, 1, 1, 2, 2, 2)));
+        b.put("x", "Doctor");
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        wf.transform(v -> ((Integer) v) + 10);
+        List<Object> list = (List<Object>) wf.get(Collections.emptyList());
+        assertEquals(List.of(11, 11, 11, 12, 12, 12), list);
+
+        assertTrue(new WriteField("d.e", () -> root).transform(x -> x + ", I presume").isEmpty());
+        assertTrue(new WriteField("a.b.d", () -> root).transform(x -> x + ", I presume").isEmpty());
+        assertEquals("Doctor, I presume", new WriteField("a.b.x", () -> root).transform(x -> x + ", I presume").get(null));
+    }
+
+    public void testRemoveValuesIf() {
+        Map<String, Object> root = new HashMap<>();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        b.put("c", new ArrayList<>(List.of(10, 10, 10, 20, 20, 20)));
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        wf.removeValue(2);
+        assertEquals(20, wf.get(2, 1000));
+
+        wf.removeValuesIf(v -> (Integer) v > 10);
+        assertEquals(2, wf.size());
+        assertEquals(List.of(10, 10), wf.get(null));
+
+        b.clear();
+        wf = new WriteField("a.b.c", () -> root);
+        wf.removeValuesIf(v -> (Integer) v > 10);
+        assertNull(wf.get(null));
+        wf.removeValue(10);
+        assertNull(wf.get(null));
+
+        b.put("c", 11);
+        wf = new WriteField("a.b.c", () -> root);
+        wf.removeValuesIf(v -> (Integer) v > 10);
+        assertNull(wf.get(null));
+
+        b.put("c", 5);
+        wf.removeValuesIf(v -> (Integer) v > 10);
+        assertEquals(5, wf.get(null));
+        wf.removeValue(1);
+        assertEquals(5, wf.get(null));
+        wf.removeValue(0);
+        assertNull(wf.get(null));
+
+        root.clear();
+        wf = new WriteField("a.b.c", () -> root);
+        wf.removeValuesIf(v -> (Integer) v > 10);
+        assertNull(wf.get(null));
+        wf.removeValue(10);
+        assertNull(wf.get(null));
+    }
+
+    public void testHasValue() {
+        Map<String, Object> root = new HashMap<>();
+        Map<String, Object> a = new HashMap<>();
+        Map<String, Object> b = new HashMap<>();
+        a.put("b", b);
+        root.put("a", a);
+        b.put("c", new ArrayList<>(List.of(10, 11, 12)));
+        WriteField wf = new WriteField("a.b.c", () -> root);
+        assertFalse(wf.hasValue(v -> (Integer) v < 10));
+        assertTrue(wf.hasValue(v -> (Integer) v <= 10));
+        wf.append(9);
+        assertTrue(wf.hasValue(v -> (Integer) v < 10));
+
+        root.clear();
+        a.clear();
+        a.put("null", null);
+        a.put("b", List.of(1, 2, 3, 4));
+        root.put("a", a);
+        wf = new WriteField("a.b", () -> root);
+        assertTrue(wf.hasValue(x -> (Integer) x % 2 == 0));
+        assertFalse(wf.hasValue(x -> (Integer) x > 4));
+        assertFalse(new WriteField("d.e", () -> root).hasValue(Objects::isNull));
+        assertTrue(new WriteField("a.null", () -> root).hasValue(Objects::isNull));
+        assertFalse(new WriteField("a.null2", () -> root).hasValue(Objects::isNull));
+    }
+
+    public void testGetIndex() {
+        Map<String, Object> root = new HashMap<>();
+        root.put("a", Map.of("b", List.of(1, 2, 3, 5), "c", "foo"));
+        WriteField wf = new WriteField("a.b", () -> root);
+        assertEquals(5, wf.get(3, 100));
+        assertEquals(100, new WriteField("c.d", () -> root).get(3, 100));
+        assertEquals("bar", new WriteField("a.c", () -> root).get(1, "bar"));
+        assertEquals("foo", new WriteField("a.c", () -> root).get(0, "bar"));
+    }
+}