Script: Reindex & UpdateByQuery Metadata (#88665)

Adds metadata classes for Reindex and UpdateByQuery contexts.

For Reindex metadata:
 * _index can't be null
 * _id, _routing and _version are writable and nullable
 * _now is read-only
 * op is read-write must be 'noop', 'index' or 'delete'

Reindex metadata keeps the originx value for _index, _id, _routing and _version
so that `Reindexer` can see if they've changed.

If _version is null in the ctx map, or, equivalently, the augmentation
`setVersionToInternal()` was called by the script, `Reindexer` sets document
versioning to internal.  If `_version` is `null` in the ctx map, `getVersion`
returns `Long.MIN_VALUE`.

For UpdateByQuery metadata:
 * _index, _id, _version, _routing are all read-only
 * _routing is also nullable
 * _now is read-only
 * op is read-write and one of 'index', 'noop', 'delete'

Closes: #86472

docs/changelog/88665.yaml

@@ -0,0 +1,5 @@
+pr: 88665
+summary: "Script: Reindex & `UpdateByQuery` Metadata"
+area: Infra/Scripting
+type: enhancement
+issues: []

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.reindex.txt

@@ -18,3 +18,22 @@ class org.elasticsearch.painless.api.Json {
   String dump(def)
   String dump(def, boolean)
 }
+
+class org.elasticsearch.script.Metadata {
+    String getIndex()
+    void setIndex(String)
+    String getId()
+    void setId(String)
+    String getRouting()
+    void setRouting(String)
+    long getVersion()
+    void setVersion(long)
+    boolean org.elasticsearch.script.ReindexMetadata isVersionInternal()
+    void org.elasticsearch.script.ReindexMetadata setVersionToInternal()
+    String getOp()
+    void setOp(String)
+}
+
+class org.elasticsearch.script.ReindexScript {
+    Metadata metadata()
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update_by_query.txt

@@ -18,3 +18,16 @@ class org.elasticsearch.painless.api.Json {
   String dump(def)
   String dump(def, boolean)
 }
+
+class org.elasticsearch.script.Metadata {
+    String getIndex()
+    String getId()
+    String getRouting()
+    long getVersion()
+    String getOp()
+    void setOp(String)
+}
+
+class org.elasticsearch.script.UpdateByQueryScript {
+    Metadata metadata()
+}

modules/reindex/build.gradle

@@ -178,3 +178,12 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
 
     task.addAllowedWarningRegex("\\[types removal\\].*")
 }
+
+tasks.named("yamlRestTestV7CompatTest").configure {
+  systemProperty 'tests.rest.blacklist', [
+    'update_by_query/80_scripting/Can\'t change _id',
+    'update_by_query/80_scripting/Set unsupported operation type',
+    'update_by_query/80_scripting/Setting bogus context is an error',
+
+  ].join(',')
+}

modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java

@@ -30,11 +30,6 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.IndexFieldMapper;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
@@ -42,6 +37,8 @@ import org.elasticsearch.index.reindex.ClientScrollableHitSource;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState;
+import org.elasticsearch.script.CtxMap;
+import org.elasticsearch.script.Metadata;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.Scroll;
@@ -50,13 +47,10 @@ import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -64,6 +58,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiFunction;
+import java.util.function.LongSupplier;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -819,147 +814,73 @@ public abstract class AbstractAsyncBulkByScrollAction<
     /**
      * Apply a {@link Script} to a {@link RequestWrapper}
      */
-    public abstract static class ScriptApplier implements BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> {
+    public abstract static class ScriptApplier<T extends Metadata>
+        implements
+            BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> {
+
+        // "index" is the default operation
+        protected static final String INDEX = "index";
 
         private final WorkerBulkByScrollTaskState taskWorker;
         protected final ScriptService scriptService;
         protected final Script script;
         protected final Map<String, Object> params;
+        protected final LongSupplier nowInMillisSupplier;
 
         public ScriptApplier(
             WorkerBulkByScrollTaskState taskWorker,
             ScriptService scriptService,
             Script script,
-            Map<String, Object> params
+            Map<String, Object> params,
+            LongSupplier nowInMillisSupplier
         ) {
             this.taskWorker = taskWorker;
             this.scriptService = scriptService;
             this.script = script;
             this.params = params;
+            this.nowInMillisSupplier = nowInMillisSupplier;
         }
 
         @Override
-        @SuppressWarnings("unchecked")
         public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
             if (script == null) {
                 return request;
             }
 
-            Map<String, Object> context = new HashMap<>();
-            context.put(IndexFieldMapper.NAME, doc.getIndex());
-            context.put(IdFieldMapper.NAME, doc.getId());
-            Long oldVersion = doc.getVersion();
-            context.put(VersionFieldMapper.NAME, oldVersion);
-            String oldRouting = doc.getRouting();
-            context.put(RoutingFieldMapper.NAME, oldRouting);
-            context.put(SourceFieldMapper.NAME, request.getSource());
-
-            OpType oldOpType = OpType.INDEX;
-            context.put("op", oldOpType.toString());
+            CtxMap<T> ctxMap = execute(doc, request.getSource());
 
-            execute(context);
+            T metadata = ctxMap.getMetadata();
 
-            String newOp = (String) context.remove("op");
-            if (newOp == null) {
-                throw new IllegalArgumentException("Script cleared operation type");
-            }
+            request.setSource(ctxMap.getSource());
 
-            /*
-             * It'd be lovely to only set the source if we know its been modified
-             * but it isn't worth keeping two copies of it around just to check!
-             */
-            request.setSource((Map<String, Object>) context.remove(SourceFieldMapper.NAME));
+            updateRequest(request, metadata);
 
-            Object newValue = context.remove(IndexFieldMapper.NAME);
-            if (false == doc.getIndex().equals(newValue)) {
-                scriptChangedIndex(request, newValue);
-            }
-            newValue = context.remove(IdFieldMapper.NAME);
-            if (false == doc.getId().equals(newValue)) {
-                scriptChangedId(request, newValue);
-            }
-            newValue = context.remove(VersionFieldMapper.NAME);
-            if (false == Objects.equals(oldVersion, newValue)) {
-                scriptChangedVersion(request, newValue);
-            }
-            /*
-             * Its important that routing comes after parent in case you want to
-             * change them both.
-             */
-            newValue = context.remove(RoutingFieldMapper.NAME);
-            if (false == Objects.equals(oldRouting, newValue)) {
-                scriptChangedRouting(request, newValue);
-            }
+            return requestFromOp(request, metadata.getOp());
+        }
 
-            OpType newOpType = OpType.fromString(newOp);
-            if (newOpType != oldOpType) {
-                return scriptChangedOpType(request, oldOpType, newOpType);
-            }
+        protected abstract CtxMap<T> execute(ScrollableHitSource.Hit doc, Map<String, Object> source);
 
-            if (false == context.isEmpty()) {
-                throw new IllegalArgumentException("Invalid fields added to context [" + String.join(",", context.keySet()) + ']');
-            }
-            return request;
-        }
+        protected abstract void updateRequest(RequestWrapper<?> request, T metadata);
 
-        protected RequestWrapper<?> scriptChangedOpType(RequestWrapper<?> request, OpType oldOpType, OpType newOpType) {
-            switch (newOpType) {
-                case NOOP -> {
+        protected RequestWrapper<?> requestFromOp(RequestWrapper<?> request, String op) {
+            switch (op) {
+                case "noop" -> {
                     taskWorker.countNoop();
                     return null;
                 }
-                case DELETE -> {
+                case "delete" -> {
                     RequestWrapper<DeleteRequest> delete = wrap(new DeleteRequest(request.getIndex(), request.getId()));
                     delete.setVersion(request.getVersion());
                     delete.setVersionType(VersionType.INTERNAL);
                     delete.setRouting(request.getRouting());
                     return delete;
                 }
-                default -> throw new IllegalArgumentException(
-                    "Unsupported operation type change from [" + oldOpType + "] to [" + newOpType + "]"
-                );
+                case INDEX -> {
+                    return request;
+                }
+                default -> throw new IllegalArgumentException("Unsupported operation type change from [" + INDEX + "] to [" + op + "]");
             }
         }
-
-        protected abstract void scriptChangedIndex(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedId(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedVersion(RequestWrapper<?> request, Object to);
-
-        protected abstract void scriptChangedRouting(RequestWrapper<?> request, Object to);
-
-        protected abstract void execute(Map<String, Object> ctx);
-    }
-
-    public enum OpType {
-
-        NOOP("noop"),
-        INDEX("index"),
-        DELETE("delete");
-
-        private final String id;
-
-        OpType(String id) {
-            this.id = id;
-        }
-
-        public static OpType fromString(String opType) {
-            String lowerOpType = opType.toLowerCase(Locale.ROOT);
-            return switch (lowerOpType) {
-                case "noop" -> OpType.NOOP;
-                case "index" -> OpType.INDEX;
-                case "delete" -> OpType.DELETE;
-                default -> throw new IllegalArgumentException(
-                    "Operation type [" + lowerOpType + "] not allowed, only " + Arrays.toString(values()) + " are allowed"
-                );
-            };
-        }
-
-        @Override
-        public String toString() {
-            return id.toLowerCase(Locale.ROOT);
-        }
     }
 
     static class ScrollConsumableHitsResponse {

modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java

@@ -40,7 +40,6 @@ import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.ReindexAction;
@@ -49,6 +48,8 @@ import org.elasticsearch.index.reindex.RemoteInfo;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState;
 import org.elasticsearch.reindex.remote.RemoteScrollableHitSource;
+import org.elasticsearch.script.CtxMap;
+import org.elasticsearch.script.ReindexMetadata;
 import org.elasticsearch.script.ReindexScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
@@ -65,13 +66,12 @@ import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
+import java.util.function.LongSupplier;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.synchronizedList;
-import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
 public class Reindexer {
@@ -289,7 +289,7 @@ public class Reindexer {
             Script script = mainRequest.getScript();
             if (script != null) {
                 assert scriptService != null : "Script service must be set";
-                return new ReindexScriptApplier(worker, scriptService, script, script.getParams());
+                return new ReindexScriptApplier(worker, scriptService, script, script.getParams(), threadPool::absoluteTimeInMillis);
             }
             return super.buildScriptApplier();
         }
@@ -374,75 +374,62 @@ public class Reindexer {
             }
         }
 
-        class ReindexScriptApplier extends ScriptApplier {
+        static class ReindexScriptApplier extends ScriptApplier<ReindexMetadata> {
             private ReindexScript.Factory reindex;
 
             ReindexScriptApplier(
                 WorkerBulkByScrollTaskState taskWorker,
                 ScriptService scriptService,
                 Script script,
-                Map<String, Object> params
+                Map<String, Object> params,
+                LongSupplier nowInMillisSupplier
             ) {
-                super(taskWorker, scriptService, script, params);
+                super(taskWorker, scriptService, script, params, nowInMillisSupplier);
             }
 
             @Override
-            protected void execute(Map<String, Object> ctx) {
+            protected CtxMap<ReindexMetadata> execute(ScrollableHitSource.Hit doc, Map<String, Object> source) {
                 if (reindex == null) {
                     reindex = scriptService.compile(script, ReindexScript.CONTEXT);
                 }
-                reindex.newInstance(params, ctx).execute();
-            }
-
-            /*
-             * Methods below here handle script updating the index request. They try
-             * to be pretty liberal with regards to types because script are often
-             * dynamically typed.
-             */
-
-            @Override
-            protected void scriptChangedIndex(RequestWrapper<?> request, Object to) {
-                requireNonNull(to, "Can't reindex without a destination index!");
-                request.setIndex(to.toString());
-            }
-
-            @Override
-            protected void scriptChangedId(RequestWrapper<?> request, Object to) {
-                request.setId(Objects.toString(to, null));
+                CtxMap<ReindexMetadata> ctxMap = new CtxMap<>(
+                    source,
+                    new ReindexMetadata(
+                        doc.getIndex(),
+                        doc.getId(),
+                        doc.getVersion(),
+                        doc.getRouting(),
+                        INDEX,
+                        nowInMillisSupplier.getAsLong()
+                    )
+                );
+                reindex.newInstance(params, ctxMap).execute();
+                return ctxMap;
             }
 
             @Override
-            protected void scriptChangedVersion(RequestWrapper<?> request, Object to) {
-                if (to == null) {
-                    request.setVersion(Versions.MATCH_ANY);
-                    request.setVersionType(INTERNAL);
-                } else {
-                    request.setVersion(asLong(to, VersionFieldMapper.NAME));
+            protected void updateRequest(RequestWrapper<?> request, ReindexMetadata metadata) {
+                if (metadata.indexChanged()) {
+                    request.setIndex(metadata.getIndex());
                 }
-            }
-
-            @Override
-            protected void scriptChangedRouting(RequestWrapper<?> request, Object to) {
-                request.setRouting(Objects.toString(to, null));
-            }
-
-            private long asLong(Object from, String name) {
-                /*
-                 * Stuffing a number into the map will have converted it to
-                 * some Number.
-                 * */
-                Number fromNumber;
-                try {
-                    fromNumber = (Number) from;
-                } catch (ClassCastException e) {
-                    throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]", e);
+                if (metadata.idChanged()) {
+                    request.setId(metadata.getId());
+                }
+                if (metadata.versionChanged()) {
+                    if (metadata.isVersionInternal()) {
+                        request.setVersion(Versions.MATCH_ANY);
+                        request.setVersionType(INTERNAL);
+                    } else {
+                        request.setVersion(metadata.getVersion());
+                    }
                 }
-                long l = fromNumber.longValue();
-                // Check that we didn't round when we fetched the value.
-                if (fromNumber.doubleValue() != l) {
-                    throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]");
+                /*
+                 * Its important that routing comes after parent in case you want to
+                 * change them both.
+                 */
+                if (metadata.routingChanged()) {
+                    request.setRouting(metadata.getRouting());
                 }
-                return l;
             }
         }
     }

modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java

@@ -18,17 +18,16 @@ import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.IndexFieldMapper;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.index.reindex.UpdateByQueryAction;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState;
+import org.elasticsearch.script.CtxMap;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.UpdateByQueryMetadata;
 import org.elasticsearch.script.UpdateByQueryScript;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -36,6 +35,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.util.Map;
 import java.util.function.BiFunction;
+import java.util.function.LongSupplier;
 
 public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkByScrollResponse> {
 
@@ -117,7 +117,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
         public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
-                return new UpdateByQueryScriptApplier(worker, scriptService, script, script.getParams());
+                return new UpdateByQueryScriptApplier(worker, scriptService, script, script.getParams(), threadPool::absoluteTimeInMillis);
             }
             return super.buildScriptApplier();
         }
@@ -134,44 +134,42 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
             return wrap(index);
         }
 
-        class UpdateByQueryScriptApplier extends ScriptApplier {
+        static class UpdateByQueryScriptApplier extends ScriptApplier<UpdateByQueryMetadata> {
             private UpdateByQueryScript.Factory update = null;
 
             UpdateByQueryScriptApplier(
                 WorkerBulkByScrollTaskState taskWorker,
                 ScriptService scriptService,
                 Script script,
-                Map<String, Object> params
+                Map<String, Object> params,
+                LongSupplier nowInMillisSupplier
             ) {
-                super(taskWorker, scriptService, script, params);
+                super(taskWorker, scriptService, script, params, nowInMillisSupplier);
             }
 
             @Override
-            protected void scriptChangedIndex(RequestWrapper<?> request, Object to) {
-                throw new IllegalArgumentException("Modifying [" + IndexFieldMapper.NAME + "] not allowed");
-            }
-
-            @Override
-            protected void scriptChangedId(RequestWrapper<?> request, Object to) {
-                throw new IllegalArgumentException("Modifying [" + IdFieldMapper.NAME + "] not allowed");
-            }
-
-            @Override
-            protected void scriptChangedVersion(RequestWrapper<?> request, Object to) {
-                throw new IllegalArgumentException("Modifying [_version] not allowed");
-            }
-
-            @Override
-            protected void scriptChangedRouting(RequestWrapper<?> request, Object to) {
-                throw new IllegalArgumentException("Modifying [" + RoutingFieldMapper.NAME + "] not allowed");
-            }
-
-            @Override
-            protected void execute(Map<String, Object> ctx) {
+            protected CtxMap<UpdateByQueryMetadata> execute(ScrollableHitSource.Hit doc, Map<String, Object> source) {
                 if (update == null) {
                     update = scriptService.compile(script, UpdateByQueryScript.CONTEXT);
                 }
-                update.newInstance(params, ctx).execute();
+                CtxMap<UpdateByQueryMetadata> ctxMap = new CtxMap<>(
+                    source,
+                    new UpdateByQueryMetadata(
+                        doc.getIndex(),
+                        doc.getId(),
+                        doc.getVersion(),
+                        doc.getRouting(),
+                        INDEX,
+                        nowInMillisSupplier.getAsLong()
+                    )
+                );
+                update.newInstance(params, ctxMap).execute();
+                return ctxMap;
+            }
+
+            @Override
+            protected void updateRequest(RequestWrapper<?> request, UpdateByQueryMetadata metadata) {
+                // do nothing
             }
         }
     }

modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java

@@ -15,7 +15,6 @@ import org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollActionTestCase;
 import org.elasticsearch.index.reindex.AbstractBulkIndexByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
-import org.elasticsearch.reindex.AbstractAsyncBulkByScrollAction.OpType;
 import org.elasticsearch.reindex.AbstractAsyncBulkByScrollAction.RequestWrapper;
 import org.elasticsearch.script.ReindexScript;
 import org.elasticsearch.script.ScriptService;
@@ -83,7 +82,7 @@ public abstract class AbstractAsyncBulkByScrollActionScriptTestCase<
             applyScript((Map<String, Object> ctx) -> ctx.put("junk", "junk"));
             fail("Expected error");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Invalid fields added to context [junk]"));
+            assertThat(e.getMessage(), equalTo("Cannot put key [junk] with value [junk] into ctx"));
         }
     }
 
@@ -97,7 +96,7 @@ public abstract class AbstractAsyncBulkByScrollActionScriptTestCase<
     }
 
     public void testSetOpTypeDelete() throws Exception {
-        DeleteRequest delete = applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.DELETE.toString()));
+        DeleteRequest delete = applyScript((Map<String, Object> ctx) -> ctx.put("op", "delete"));
         assertThat(delete.index(), equalTo("index"));
         assertThat(delete.id(), equalTo("1"));
     }
@@ -107,7 +106,7 @@ public abstract class AbstractAsyncBulkByScrollActionScriptTestCase<
             IllegalArgumentException.class,
             () -> applyScript((Map<String, Object> ctx) -> ctx.put("op", "unknown"))
         );
-        assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed"));
+        assertThat(e.getMessage(), equalTo("[op] must be one of delete, index, noop, not [unknown]"));
     }
 
     protected abstract AbstractAsyncBulkByScrollAction<Request, ?> action(ScriptService scriptService, Request request);

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java

@@ -34,8 +34,8 @@ public class ReindexScriptTests extends AbstractAsyncBulkByScrollActionScriptTes
     public void testSettingIndexToNullIsError() throws Exception {
         try {
             applyScript((Map<String, Object> ctx) -> ctx.put("_index", null));
-        } catch (NullPointerException e) {
-            assertThat(e.getMessage(), containsString("Can't reindex without a destination index!"));
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("_index cannot be null"));
         }
     }
 
@@ -60,13 +60,17 @@ public class ReindexScriptTests extends AbstractAsyncBulkByScrollActionScriptTes
     }
 
     public void testSettingVersionToJunkIsAnError() throws Exception {
-        Object junkVersion = randomFrom(new Object[] { "junk", Math.PI });
-        try {
-            applyScript((Map<String, Object> ctx) -> ctx.put("_version", junkVersion));
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("_version may only be set to an int or a long but was ["));
-            assertThat(e.getMessage(), containsString(junkVersion.toString()));
-        }
+        IllegalArgumentException err = expectThrows(
+            IllegalArgumentException.class,
+            () -> applyScript((Map<String, Object> ctx) -> ctx.put("_version", "junk"))
+        );
+        assertEquals(err.getMessage(), "_version [junk] is wrong type, expected assignable to [java.lang.Number], not [java.lang.String]");
+
+        err = expectThrows(IllegalArgumentException.class, () -> applyScript((Map<String, Object> ctx) -> ctx.put("_version", Math.PI)));
+        assertEquals(
+            err.getMessage(),
+            "_version may only be set to an int or a long but was [3.141592653589793] with type [java.lang.Double]"
+        );
     }
 
     public void testSetRouting() throws Exception {

modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java

@@ -38,7 +38,7 @@ public class UpdateByQueryWithScriptTests extends AbstractAsyncBulkByScrollActio
             try {
                 applyScript((Map<String, Object> ctx) -> ctx.put(ctxVar, randomFrom(options)));
             } catch (IllegalArgumentException e) {
-                assertThat(e.getMessage(), containsString("Modifying [" + ctxVar + "] not allowed"));
+                assertThat(e.getMessage(), containsString(ctxVar + " cannot be updated"));
             }
         }
     }

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/80_scripting.yml

@@ -132,6 +132,9 @@
 
 ---
 "Setting bogus context is an error":
+  - skip:
+      version: " - 8.4.99"
+      reason: "error message changed in 8.5"
   - do:
       index:
         index:  twitter
@@ -141,7 +144,7 @@
       indices.refresh: {}
 
   - do:
-      catch: /Invalid fields added to context \[junk\]/
+      catch: /Cannot put key \[junk\] with value \[stuff\] into ctx/
       update_by_query:
         index: twitter
         body:
@@ -151,6 +154,9 @@
 
 ---
 "Can't change _id":
+  - skip:
+      version: " - 8.4.99"
+      reason: "error message changed in 8.5"
   - do:
       index:
         index:  twitter
@@ -160,7 +166,7 @@
       indices.refresh: {}
 
   - do:
-      catch: /Modifying \[_id\] not allowed/
+      catch: /_id cannot be updated/
       update_by_query:
         index: twitter
         body:
@@ -307,6 +313,9 @@
 
 ---
 "Set unsupported operation type":
+  - skip:
+      version: " - 8.4.99"
+      reason: "error message changed in 8.5"
   - do:
       index:
         index:  twitter
@@ -321,7 +330,7 @@
       indices.refresh: {}
 
   - do:
-      catch: bad_request
+      catch: /\[op\] must be one of delete, index, noop, not \[junk\]/
       update_by_query:
         refresh: true
         index:   twitter
@@ -330,8 +339,6 @@
             lang: painless
             source: if (ctx._source.user == "kimchy") {ctx.op = "index"} else {ctx.op = "junk"}
 
-  - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }
-
 ---
 "Update all docs with one deletion and one noop using a stored script":
   - do:

server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java

@@ -10,7 +10,6 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.script.CtxMap;
-import org.elasticsearch.script.Metadata;
 
 import java.time.ZonedDateTime;
 import java.util.HashMap;
@@ -29,7 +28,7 @@ import java.util.Map;
  *
  * The map is expected to be used by processors, server code should the typed getter and setters where possible.
  */
-class IngestCtxMap extends CtxMap {
+class IngestCtxMap extends CtxMap<IngestDocMetadata> {
 
     /**
      * Create an IngestCtxMap with the given metadata, source and default validators
@@ -52,7 +51,7 @@ class IngestCtxMap extends CtxMap {
      * @param source the source document map
      * @param metadata the metadata map
      */
-    IngestCtxMap(Map<String, Object> source, Metadata metadata) {
+    IngestCtxMap(Map<String, Object> source, IngestDocMetadata metadata) {
         super(source, metadata);
     }
 

server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java

@@ -18,41 +18,26 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 class IngestDocMetadata extends Metadata {
-    private static final FieldProperty<String> UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null);
+
     static final Map<String, FieldProperty<?>> PROPERTIES = Map.of(
         INDEX,
-        UPDATABLE_STRING,
+        StringField.withWritable().withNullable(),
         ID,
-        UPDATABLE_STRING,
+        StringField.withWritable().withNullable(),
         ROUTING,
-        UPDATABLE_STRING,
+        StringField.withWritable().withNullable(),
         VERSION_TYPE,
-        new FieldProperty<>(String.class, true, true, (k, v) -> {
-            try {
-                VersionType.fromString(v);
-                return;
-            } catch (IllegalArgumentException ignored) {}
-            throw new IllegalArgumentException(
-                k
-                    + " must be a null or one of ["
-                    + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", "))
-                    + "] but was ["
-                    + v
-                    + "] with type ["
-                    + v.getClass().getName()
-                    + "]"
-            );
-        }),
+        StringField.withWritable().withNullable().withValidation(IngestDocMetadata::versionTypeValidator),
         VERSION,
-        new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER),
+        LongField.withWritable(),
         TYPE,
-        new FieldProperty<>(String.class, true, false, null),
+        StringField.withNullable(),
         IF_SEQ_NO,
-        new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER),
+        LongField.withWritable().withNullable(),
         IF_PRIMARY_TERM,
-        new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER),
+        LongField.withWritable().withNullable(),
         DYNAMIC_TEMPLATES,
-        new FieldProperty<>(Map.class, true, true, null)
+        new FieldProperty<>(Map.class).withWritable().withNullable()
     );
 
     protected final ZonedDateTime timestamp;
@@ -62,7 +47,11 @@ class IngestDocMetadata extends Metadata {
     }
 
     IngestDocMetadata(Map<String, Object> metadata, ZonedDateTime timestamp) {
-        super(metadata, PROPERTIES);
+        this(metadata, PROPERTIES, timestamp);
+    }
+
+    IngestDocMetadata(Map<String, Object> metadata, Map<String, FieldProperty<?>> properties, ZonedDateTime timestamp) {
+        super(metadata, properties);
         this.timestamp = timestamp;
     }
 
@@ -87,4 +76,26 @@ class IngestDocMetadata extends Metadata {
     public ZonedDateTime getNow() {
         return timestamp;
     }
+
+    @Override
+    public IngestDocMetadata clone() {
+        return new IngestDocMetadata(map, timestamp);
+    }
+
+    private static void versionTypeValidator(String key, String value) {
+        try {
+            VersionType.fromString(value);
+            return;
+        } catch (IllegalArgumentException ignored) {}
+        throw new IllegalArgumentException(
+            key
+                + " must be a null or one of ["
+                + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", "))
+                + "] but was ["
+                + value
+                + "] with type ["
+                + value.getClass().getName()
+                + "]"
+        );
+    }
 }

server/src/main/java/org/elasticsearch/script/CtxMap.java

@@ -26,10 +26,10 @@ import java.util.stream.Collectors;
  * all other updates to source.  Implements the {@link Map} interface for backwards compatibility while performing
  * validation via {@link Metadata}.
  */
-public class CtxMap extends AbstractMap<String, Object> {
+public class CtxMap<T extends Metadata> extends AbstractMap<String, Object> {
     protected static final String SOURCE = "_source";
     protected Map<String, Object> source;
-    protected final Metadata metadata;
+    protected final T metadata;
 
     /**
      * Create CtxMap from a source and metadata
@@ -37,7 +37,7 @@ public class CtxMap extends AbstractMap<String, Object> {
      * @param source the source document map
      * @param metadata the metadata map
      */
-    protected CtxMap(Map<String, Object> source, Metadata metadata) {
+    public CtxMap(Map<String, Object> source, T metadata) {
         this.source = source;
         this.metadata = metadata;
         Set<String> badKeys = Sets.intersection(this.metadata.keySet(), this.source.keySet());
@@ -70,7 +70,7 @@ public class CtxMap extends AbstractMap<String, Object> {
     /**
      * get the metadata map, if externally modified then the guarantees of this class are not enforced
      */
-    public Metadata getMetadata() {
+    public T getMetadata() {
         return metadata;
     }
 
@@ -337,7 +337,7 @@ public class CtxMap extends AbstractMap<String, Object> {
         if (this == o) return true;
         if ((o instanceof CtxMap) == false) return false;
         if (super.equals(o) == false) return false;
-        CtxMap ctxMap = (CtxMap) o;
+        CtxMap<?> ctxMap = (CtxMap<?>) o;
         return source.equals(ctxMap.source) && metadata.equals(ctxMap.metadata);
     }
 

server/src/main/java/org/elasticsearch/script/Metadata.java

@@ -50,6 +50,10 @@ public class Metadata {
     protected static final String IF_PRIMARY_TERM = "_if_primary_term";
     protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates";
 
+    public static FieldProperty<Object> ObjectField = new FieldProperty<>(Object.class);
+    public static FieldProperty<String> StringField = new FieldProperty<>(String.class);
+    public static FieldProperty<Number> LongField = new FieldProperty<>(Number.class).withValidation(FieldProperty.LONGABLE_NUMBER);
+
     protected final Map<String, Object> map;
     protected final Map<String, FieldProperty<?>> properties;
     protected static final FieldProperty<?> BAD_KEY = new FieldProperty<>(null, false, false, null);
@@ -280,6 +284,31 @@ public class Metadata {
      */
     public record FieldProperty<T> (Class<T> type, boolean nullable, boolean writable, BiConsumer<String, T> extendedValidation) {
 
+        public FieldProperty(Class<T> type) {
+            this(type, false, false, null);
+        }
+
+        public FieldProperty<T> withNullable() {
+            if (nullable) {
+                return this;
+            }
+            return new FieldProperty<>(type, true, writable, extendedValidation);
+        }
+
+        public FieldProperty<T> withWritable() {
+            if (writable) {
+                return this;
+            }
+            return new FieldProperty<>(type, nullable, true, extendedValidation);
+        }
+
+        public FieldProperty<T> withValidation(BiConsumer<String, T> extendedValidation) {
+            if (this.extendedValidation == extendedValidation) {
+                return this;
+            }
+            return new FieldProperty<>(type, nullable, writable, extendedValidation);
+        }
+
         public static BiConsumer<String, Number> LONGABLE_NUMBER = (k, v) -> {
             long version = v.longValue();
             // did we round?
@@ -346,4 +375,14 @@ public class Metadata {
             }
         }
     }
+
+    public static BiConsumer<String, String> stringSetValidator(Set<String> valid) {
+        return (k, v) -> {
+            if (valid.contains(v) == false) {
+                throw new IllegalArgumentException(
+                    "[" + k + "] must be one of " + valid.stream().sorted().collect(Collectors.joining(", ")) + ", not [" + v + "]"
+                );
+            }
+        };
+    }
 }

server/src/main/java/org/elasticsearch/script/ReindexMetadata.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.ingest.IngestDocument;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Metadata for the {@link ReindexScript} context.
+ * _index, _id, _version, _routing are all read-write.  _id, _version and _routing are also nullable.
+ * _now is millis since epoch and read-only
+ * op is read-write one of 'index', 'noop', 'delete'
+ *
+ * If _version is set to null in the ctx map, that is interpreted as using internal versioning.
+ *
+ * The {@link #setVersionToInternal(Metadata)} and {@link #isVersionInternal(Metadata)} augmentations
+ * are provided for users of this class.  These augmentations allow the class to appear as {@link Metadata}
+ * but handle the internal versioning scheme without scripts accessing the ctx map.
+ */
+public class ReindexMetadata extends Metadata {
+    static final Map<String, FieldProperty<?>> PROPERTIES = Map.of(
+        INDEX,
+        ObjectField.withWritable(),
+        ID,
+        ObjectField.withWritable().withNullable(),
+        VERSION,
+        LongField.withWritable().withNullable(),
+        ROUTING,
+        StringField.withWritable().withNullable(),
+        OP,
+        StringField.withWritable().withValidation(stringSetValidator(Set.of("noop", "index", "delete"))),
+        NOW,
+        LongField
+    );
+
+    protected final String index;
+    protected final String id;
+    protected final Long version;
+    protected final String routing;
+
+    public ReindexMetadata(String index, String id, Long version, String routing, String op, long timestamp) {
+        super(metadataMap(index, id, version, routing, op, timestamp), PROPERTIES);
+        this.index = index;
+        this.id = id;
+        this.version = version;
+        this.routing = routing;
+    }
+
+    /**
+     * Create the backing metadata map with the standard contents assuming default validators.
+     */
+    protected static Map<String, Object> metadataMap(String index, String id, Long version, String routing, String op, long timestamp) {
+        Map<String, Object> metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length);
+        metadata.put(INDEX, index);
+        metadata.put(ID, id);
+        metadata.put(VERSION, version);
+        metadata.put(ROUTING, routing);
+        metadata.put(OP, op);
+        metadata.put(NOW, timestamp);
+        return metadata;
+    }
+
+    /**
+     * Get version, if it's null, return sentinel value {@link Long#MIN_VALUE}
+     */
+    @Override
+    public long getVersion() {
+        Number version = getNumber(VERSION);
+        if (version == null) {
+            return Long.MIN_VALUE;
+        }
+        return version.longValue();
+    }
+
+    public boolean isVersionInternal() {
+        return get(VERSION) == null;
+    }
+
+    /**
+     * Augmentation to allow {@link ReindexScript}s to check if the version is set to "internal"
+     */
+    public static boolean isVersionInternal(Metadata receiver) {
+        return receiver.get(VERSION) == null;
+    }
+
+    /**
+     * Augmentation to allow {@link ReindexScript}s to set the version to "internal".
+     *
+     * This is necessary because {@link #setVersion(long)} takes a primitive long.
+     */
+    public static void setVersionToInternal(Metadata receiver) {
+        receiver.put(VERSION, null);
+    }
+
+    public boolean versionChanged() {
+        Number updated = getNumber(VERSION);
+        if (version == null || updated == null) {
+            return version != updated;
+        }
+        return version != updated.longValue();
+    }
+
+    public boolean indexChanged() {
+        return Objects.equals(index, getString(INDEX)) == false;
+    }
+
+    public boolean idChanged() {
+        return Objects.equals(id, getString(ID)) == false;
+    }
+
+    public boolean routingChanged() {
+        return Objects.equals(routing, getString(ROUTING)) == false;
+    }
+}

server/src/main/java/org/elasticsearch/script/ReindexScript.java

@@ -25,11 +25,17 @@ public abstract class ReindexScript {
     private final Map<String, Object> params;
 
     /** The context map for the script */
-    private final Map<String, Object> ctx;
-
-    public ReindexScript(Map<String, Object> params, Map<String, Object> ctx) {
+    private final CtxMap<ReindexMetadata> ctxMap;
+
+    /**
+     * Metadata available to the script
+     * _index can't be null
+     * _id, _routing and _version are writable and nullable
+     * op must be 'noop', 'index' or 'delete'
+     */
+    public ReindexScript(Map<String, Object> params, CtxMap<ReindexMetadata> ctxMap) {
         this.params = params;
-        this.ctx = ctx;
+        this.ctxMap = ctxMap;
     }
 
     /** Return the parameters for this script. */
@@ -39,12 +45,17 @@ public abstract class ReindexScript {
 
     /** Return the context map for this script */
     public Map<String, Object> getCtx() {
-        return ctx;
+        return ctxMap;
+    }
+
+    /** Return the update metadata for this script */
+    public Metadata metadata() {
+        return ctxMap.getMetadata();
     }
 
     public abstract void execute();
 
     public interface Factory {
-        ReindexScript newInstance(Map<String, Object> params, Map<String, Object> ctx);
+        ReindexScript newInstance(Map<String, Object> params, CtxMap<ReindexMetadata> ctxMap);
     }
 }

server/src/main/java/org/elasticsearch/script/UpdateByQueryMetadata.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.ingest.IngestDocument;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Metadata for the {@link UpdateByQueryMetadata} context.
+ * _index, _id, _version, _routing are all read-only.  _routing is also nullable.
+ * _now is millis since epoch and read-only.
+ * op is read-write and one of 'index', 'noop', 'delete'
+ */
+public class UpdateByQueryMetadata extends Metadata {
+    static final Map<String, FieldProperty<?>> PROPERTIES = Map.of(
+        INDEX,
+        StringField,
+        ID,
+        StringField,
+        VERSION,
+        LongField,
+        ROUTING,
+        StringField.withNullable(),
+        OP,
+        StringField.withWritable().withValidation(stringSetValidator(Set.of("noop", "index", "delete"))),
+        NOW,
+        LongField
+    );
+
+    public UpdateByQueryMetadata(String index, String id, long version, String routing, String op, long timestamp) {
+        super(metadataMap(index, id, version, routing, op, timestamp), PROPERTIES);
+    }
+
+    /**
+     * Create the backing metadata map with the standard contents assuming default validators.
+     */
+    protected static Map<String, Object> metadataMap(String index, String id, Long version, String routing, String op, long timestamp) {
+        Map<String, Object> metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length);
+        metadata.put(INDEX, index);
+        metadata.put(ID, id);
+        metadata.put(VERSION, version);
+        metadata.put(ROUTING, routing);
+        metadata.put(OP, op);
+        metadata.put(NOW, timestamp);
+        return metadata;
+    }
+}

server/src/main/java/org/elasticsearch/script/UpdateByQueryScript.java

@@ -25,11 +25,11 @@ public abstract class UpdateByQueryScript {
     private final Map<String, Object> params;
 
     /** The context map for the script */
-    private final Map<String, Object> ctx;
+    private final CtxMap<UpdateByQueryMetadata> ctxMap;
 
-    public UpdateByQueryScript(Map<String, Object> params, Map<String, Object> ctx) {
+    public UpdateByQueryScript(Map<String, Object> params, CtxMap<UpdateByQueryMetadata> ctxMap) {
         this.params = params;
-        this.ctx = ctx;
+        this.ctxMap = ctxMap;
     }
 
     /** Return the parameters for this script. */
@@ -39,12 +39,17 @@ public abstract class UpdateByQueryScript {
 
     /** Return the context map for this script */
     public Map<String, Object> getCtx() {
-        return ctx;
+        return ctxMap;
+    }
+
+    /** Return the update metadata for this script */
+    public Metadata metadata() {
+        return ctxMap.getMetadata();
     }
 
     public abstract void execute();
 
     public interface Factory {
-        UpdateByQueryScript newInstance(Map<String, Object> params, Map<String, Object> ctx);
+        UpdateByQueryScript newInstance(Map<String, Object> params, CtxMap<UpdateByQueryMetadata> ctx);
     }
 }

server/src/main/java/org/elasticsearch/script/UpdateCtxMap.java

@@ -13,7 +13,7 @@ import java.util.Map;
 /**
  * Source and metadata for update (as opposed to insert via upsert) in the Update context.
  */
-public class UpdateCtxMap extends CtxMap {
+public class UpdateCtxMap extends CtxMap<UpdateMetadata> {
 
     public UpdateCtxMap(
         String index,
@@ -28,7 +28,7 @@ public class UpdateCtxMap extends CtxMap {
         super(source, new UpdateMetadata(index, id, version, routing, type, op, now));
     }
 
-    protected UpdateCtxMap(Map<String, Object> source, Metadata metadata) {
+    protected UpdateCtxMap(Map<String, Object> source, UpdateMetadata metadata) {
         super(source, metadata);
     }
 }

server/src/main/java/org/elasticsearch/script/UpdateMetadata.java

@@ -22,33 +22,23 @@ import java.util.stream.Collectors;
 public class UpdateMetadata extends Metadata {
     // AbstractAsyncBulkByScrollAction.OpType uses 'noop' rather than 'none', so unify on 'noop' but allow 'none' in
     // the ctx map
-
     protected static final String LEGACY_NOOP_STRING = "none";
 
-    protected static final FieldProperty<String> SET_ONCE_STRING = new FieldProperty<>(String.class, true, false, null);
-
-    protected static final FieldProperty<Number> SET_ONCE_LONG = new FieldProperty<>(
-        Number.class,
-        false,
-        false,
-        FieldProperty.LONGABLE_NUMBER
-    );
-
     static final Map<String, FieldProperty<?>> PROPERTIES = Map.of(
         INDEX,
-        SET_ONCE_STRING,
+        StringField.withNullable(),
         ID,
-        SET_ONCE_STRING,
+        StringField.withNullable(),
         VERSION,
-        SET_ONCE_LONG,
+        LongField,
         ROUTING,
-        SET_ONCE_STRING,
+        StringField.withNullable(),
         TYPE,
-        SET_ONCE_STRING,
+        StringField.withNullable(),
         OP,
-        new FieldProperty<>(String.class, true, true, null),
+        StringField.withWritable().withNullable(),
         NOW,
-        SET_ONCE_LONG
+        LongField
     );
 
     protected final Set<String> validOps;

server/src/main/java/org/elasticsearch/script/UpsertMetadata.java

@@ -16,13 +16,13 @@ import java.util.Set;
 class UpsertMetadata extends UpdateMetadata {
     static final Map<String, FieldProperty<?>> PROPERTIES = Map.of(
         INDEX,
-        SET_ONCE_STRING,
+        StringField,
         ID,
-        SET_ONCE_STRING,
+        StringField,
         OP,
-        new FieldProperty<>(String.class, true, true, null),
+        StringField.withWritable().withNullable(),
         NOW,
-        SET_ONCE_LONG
+        LongField
     );
 
     UpsertMetadata(String index, String id, String op, long now) {

server/src/test/java/org/elasticsearch/script/CtxMapTests.java

@@ -19,13 +19,13 @@ import java.util.Map;
 import static org.hamcrest.Matchers.containsString;
 
 public class CtxMapTests extends ESTestCase {
-    CtxMap map;
+    CtxMap<?> map;
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        map = new CtxMap(new HashMap<>(), new Metadata(new HashMap<>(), new HashMap<>()));
+        map = new CtxMap<>(new HashMap<>(), new Metadata(new HashMap<>(), new HashMap<>()));
     }
 
     public void testAddingJunkToCtx() {

server/src/test/java/org/elasticsearch/script/ReindexMetadataTests.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class ReindexMetadataTests extends ESTestCase {
+    private static final String INDEX = "myIndex";
+    private static final String ID = "myId";
+    private static final long VERSION = 5;
+    private static final String ROUTING = "myRouting";
+    private static final String OP = "index";
+
+    private static final long TIMESTAMP = 1_658_000_000_000L;
+    private ReindexMetadata metadata;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        reset();
+    }
+
+    protected void reset() {
+        metadata = new ReindexMetadata(INDEX, ID, VERSION, ROUTING, OP, TIMESTAMP);
+    }
+
+    public void testIndex() {
+        assertFalse(metadata.indexChanged());
+
+        metadata.put("_index", INDEX);
+        assertFalse(metadata.indexChanged());
+
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> metadata.remove("_index"));
+        assertEquals("_index cannot be removed", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> metadata.put("_index", null));
+        assertEquals("_index cannot be null", err.getMessage());
+        assertFalse(metadata.indexChanged());
+
+        metadata.put("_index", "myIndex2");
+        assertTrue(metadata.indexChanged());
+
+        metadata.put("_index", INDEX);
+        assertFalse(metadata.indexChanged());
+
+        metadata.setIndex("myIndex3");
+        assertTrue(metadata.indexChanged());
+    }
+
+    public void testId() {
+        assertFalse(metadata.idChanged());
+
+        metadata.put("_id", ID);
+        assertFalse(metadata.idChanged());
+
+        metadata.remove("_id");
+        assertTrue(metadata.idChanged());
+        assertNull(metadata.getId());
+
+        metadata.put("_id", "myId2");
+        assertTrue(metadata.idChanged());
+
+        metadata.setId(ID);
+        assertFalse(metadata.idChanged());
+
+        metadata.setId("myId3");
+        assertTrue(metadata.idChanged());
+    }
+
+    public void testRouting() {
+        assertFalse(metadata.routingChanged());
+
+        metadata.put("_routing", ROUTING);
+        assertFalse(metadata.routingChanged());
+
+        metadata.remove("_routing");
+        assertTrue(metadata.routingChanged());
+        assertNull(metadata.getRouting());
+
+        metadata.put("_routing", "myRouting2");
+        assertTrue(metadata.routingChanged());
+
+        metadata.setRouting(ROUTING);
+        assertFalse(metadata.routingChanged());
+
+        metadata.setRouting("myRouting3");
+        assertTrue(metadata.routingChanged());
+    }
+
+    public void testVersion() {
+        assertFalse(metadata.versionChanged());
+
+        metadata.put("_version", VERSION);
+        assertFalse(metadata.versionChanged());
+
+        metadata.remove("_version");
+        assertTrue(metadata.versionChanged());
+        assertTrue(metadata.isVersionInternal());
+        assertEquals(Long.MIN_VALUE, metadata.getVersion());
+        assertNull(metadata.get("_version"));
+
+        metadata.put("_version", VERSION + 5);
+        assertTrue(metadata.versionChanged());
+
+        metadata.setVersion(VERSION);
+        assertFalse(metadata.versionChanged());
+
+        metadata.setVersion(VERSION + 10);
+        assertTrue(metadata.versionChanged());
+        assertEquals(VERSION + 10, metadata.getVersion());
+
+        ReindexMetadata.setVersionToInternal(metadata);
+        assertTrue(metadata.isVersionInternal());
+        assertEquals(Long.MIN_VALUE, metadata.getVersion());
+        assertNull(metadata.get("_version"));
+    }
+
+    public void testOp() {
+        assertEquals("index", metadata.getOp());
+        assertEquals("index", metadata.get("op"));
+
+        metadata.setOp("noop");
+        assertEquals("noop", metadata.getOp());
+        assertEquals("noop", metadata.get("op"));
+
+        metadata.put("op", "delete");
+        assertEquals("delete", metadata.getOp());
+
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> metadata.setOp("bad"));
+        assertEquals("[op] must be one of delete, index, noop, not [bad]", err.getMessage());
+
+        err = expectThrows(IllegalArgumentException.class, () -> metadata.put("op", "malo"));
+        assertEquals("[op] must be one of delete, index, noop, not [malo]", err.getMessage());
+    }
+}

server/src/test/java/org/elasticsearch/script/UpdateByQueryMetadataTests.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+public class UpdateByQueryMetadataTests extends ESTestCase {
+    UpdateByQueryMetadata meta;
+
+    public void testROFields() {
+        meta = new UpdateByQueryMetadata("myIndex", "myId", 5, "myRouting", "index", 12345000);
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> meta.put("_index", "something"));
+        assertEquals("_index cannot be updated", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> meta.put("_id", "something"));
+        assertEquals("_id cannot be updated", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> meta.put("_version", 600));
+        assertEquals("_version cannot be updated", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> meta.put("_routing", "something"));
+        assertEquals("_routing cannot be updated", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> meta.put("_now", 9000));
+        assertEquals("_now cannot be updated", err.getMessage());
+    }
+
+    public void testOpSet() {
+        meta = new UpdateByQueryMetadata("myIndex", "myId", 5, "myRouting", "index", 12345000);
+        for (String op : List.of("noop", "index", "delete")) {
+            meta.setOp(op);
+            meta.put("op", op);
+        }
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> meta.put("op", "bad"));
+        assertEquals("[op] must be one of delete, index, noop, not [bad]", err.getMessage());
+        err = expectThrows(IllegalArgumentException.class, () -> meta.setOp("bad"));
+        assertEquals("[op] must be one of delete, index, noop, not [bad]", err.getMessage());
+    }
+}

test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java

@@ -13,9 +13,9 @@ import org.elasticsearch.script.Metadata;
 import java.util.HashMap;
 import java.util.Map;
 
-public class TestIngestCtxMetadata extends Metadata {
+public class TestIngestCtxMetadata extends IngestDocMetadata {
     public TestIngestCtxMetadata(Map<String, Object> map, Map<String, FieldProperty<?>> properties) {
-        super(map, properties);
+        super(map, properties, null);
     }
 
     public static TestIngestCtxMetadata withNullableVersion(Map<String, Object> map) {