ESQL: Introduce language versioning to REST API (#106824)

For the _query endpoint, add a parameter for the ESQL language version to the JSON payload.
For now, it is optional and is only validated with no further action.

docs/changelog/106824.yaml

@@ -0,0 +1,5 @@
+pr: 106824
+summary: "ESQL: Introduce language versioning to REST API"
+area: ES|QL
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/esql/action/EsqlQueryRequest.java

@@ -21,6 +21,9 @@ public abstract class EsqlQueryRequest extends ActionRequest {
         super(in);
     }
 
+    // Use the unparsed version String, so we don't have to serialize a version object.
+    public abstract String esqlVersion();
+
     public abstract String query();
 
     public abstract QueryBuilder filter();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/esql/action/EsqlQueryRequestBuilder.java

@@ -35,6 +35,8 @@ public abstract class EsqlQueryRequestBuilder<Request extends EsqlQueryRequest,
         return action;
     }
 
+    public abstract EsqlQueryRequestBuilder<Request, Response> esqlVersion(String esqlVersion);
+
     public abstract EsqlQueryRequestBuilder<Request, Response> query(String query);
 
     public abstract EsqlQueryRequestBuilder<Request, Response> filter(QueryBuilder filter);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java

@@ -20,6 +20,7 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.version.EsqlVersion;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,6 +36,7 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
 
     private boolean async;
 
+    private String esqlVersion;
     private String query;
     private boolean columnar;
     private boolean profile;
@@ -45,6 +47,7 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
     private TimeValue waitForCompletionTimeout = DEFAULT_WAIT_FOR_COMPLETION;
     private TimeValue keepAlive = DEFAULT_KEEP_ALIVE;
     private boolean keepOnCompletion;
+    private boolean onSnapshotBuild = Build.current().isSnapshot();
 
     static EsqlQueryRequest syncEsqlQueryRequest() {
         return new EsqlQueryRequest(false);
@@ -65,17 +68,54 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
+        if (Strings.hasText(esqlVersion) == false) {
+            // TODO: make this required
+            // "https://github.com/elastic/elasticsearch/issues/104890"
+            // validationException = addValidationError(invalidVersion("is required"), validationException);
+        } else {
+            EsqlVersion version = EsqlVersion.parse(esqlVersion);
+            if (version == null) {
+                validationException = addValidationError(invalidVersion("has invalid value [" + esqlVersion + "]"), validationException);
+            } else if (version == EsqlVersion.SNAPSHOT && onSnapshotBuild == false) {
+                validationException = addValidationError(
+                    invalidVersion("with value [" + esqlVersion + "] only allowed in snapshot builds"),
+                    validationException
+                );
+            }
+        }
         if (Strings.hasText(query) == false) {
-            validationException = addValidationError("[query] is required", validationException);
+            validationException = addValidationError("[" + RequestXContent.QUERY_FIELD + "] is required", validationException);
         }
-        if (Build.current().isSnapshot() == false && pragmas.isEmpty() == false) {
-            validationException = addValidationError("[pragma] only allowed in snapshot builds", validationException);
+        if (onSnapshotBuild == false && pragmas.isEmpty() == false) {
+            validationException = addValidationError(
+                "[" + RequestXContent.PRAGMA_FIELD + "] only allowed in snapshot builds",
+                validationException
+            );
         }
         return validationException;
     }
 
+    private static String invalidVersion(String reason) {
+        return "["
+            + RequestXContent.ESQL_VERSION_FIELD
+            + "] "
+            + reason
+            + ", latest available version is ["
+            + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+            + "]";
+    }
+
     public EsqlQueryRequest() {}
 
+    public void esqlVersion(String esqlVersion) {
+        this.esqlVersion = esqlVersion;
+    }
+
+    @Override
+    public String esqlVersion() {
+        return esqlVersion;
+    }
+
     public void query(String query) {
         this.query = query;
     }
@@ -174,4 +214,9 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
         // Pass the query as the description
         return new CancellableTask(id, type, action, query, parentTaskId, headers);
     }
+
+    // Setter for tests
+    void onSnapshotBuild(boolean onSnapshotBuild) {
+        this.onSnapshotBuild = onSnapshotBuild;
+    }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java

@@ -29,6 +29,12 @@ public class EsqlQueryRequestBuilder extends org.elasticsearch.xpack.core.esql.a
         super(client, EsqlQueryAction.INSTANCE, request);
     }
 
+    @Override
+    public EsqlQueryRequestBuilder esqlVersion(String esqlVersion) {
+        request.esqlVersion(esqlVersion);
+        return this;
+    }
+
     @Override
     public EsqlQueryRequestBuilder query(String query) {
         request.query(query);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java

@@ -46,10 +46,11 @@ final class RequestXContent {
         PARAM_PARSER.declareString(constructorArg(), TYPE);
     }
 
-    private static final ParseField QUERY_FIELD = new ParseField("query");
+    static final ParseField ESQL_VERSION_FIELD = new ParseField("version");
+    static final ParseField QUERY_FIELD = new ParseField("query");
     private static final ParseField COLUMNAR_FIELD = new ParseField("columnar");
     private static final ParseField FILTER_FIELD = new ParseField("filter");
-    private static final ParseField PRAGMA_FIELD = new ParseField("pragma");
+    static final ParseField PRAGMA_FIELD = new ParseField("pragma");
     private static final ParseField PARAMS_FIELD = new ParseField("params");
     private static final ParseField LOCALE_FIELD = new ParseField("locale");
     private static final ParseField PROFILE_FIELD = new ParseField("profile");
@@ -72,6 +73,7 @@ final class RequestXContent {
     }
 
     private static void objectParserCommon(ObjectParser<EsqlQueryRequest, ?> parser) {
+        parser.declareString(EsqlQueryRequest::esqlVersion, ESQL_VERSION_FIELD);
         parser.declareString(EsqlQueryRequest::query, QUERY_FIELD);
         parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD);
         parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/version/EsqlVersion.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.version;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.VersionId;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public enum EsqlVersion implements VersionId<EsqlVersion> {
+    /**
+     * Breaking changes go here until the next version is released.
+     */
+    SNAPSHOT(Integer.MAX_VALUE, 12, 99, "📷"),
+    ROCKET(2024, 4, "🚀");
+
+    static final Map<String, EsqlVersion> VERSION_MAP_WITH_AND_WITHOUT_EMOJI = versionMapWithAndWithoutEmoji();
+
+    private static Map<String, EsqlVersion> versionMapWithAndWithoutEmoji() {
+        Map<String, EsqlVersion> stringToVersion = new LinkedHashMap<>(EsqlVersion.values().length * 2);
+
+        for (EsqlVersion version : EsqlVersion.values()) {
+            putVersionCheckNoDups(stringToVersion, version.versionStringWithoutEmoji(), version);
+            putVersionCheckNoDups(stringToVersion, version.toString(), version);
+        }
+
+        return stringToVersion;
+    }
+
+    private static void putVersionCheckNoDups(Map<String, EsqlVersion> stringToVersion, String versionString, EsqlVersion version) {
+        EsqlVersion existingVersionForKey = stringToVersion.put(versionString, version);
+        if (existingVersionForKey != null) {
+            throw new IllegalArgumentException("Duplicate esql version with version string [" + versionString + "]");
+        }
+    }
+
+    /**
+     * Accepts a version string with the emoji suffix or without it.
+     * E.g. both "2024.04.01.🚀" and "2024.04.01" will be interpreted as {@link EsqlVersion#ROCKET}.
+     */
+    public static EsqlVersion parse(String versionString) {
+        return VERSION_MAP_WITH_AND_WITHOUT_EMOJI.get(versionString);
+    }
+
+    public static EsqlVersion latestReleased() {
+        return Arrays.stream(EsqlVersion.values()).filter(v -> v != SNAPSHOT).max(Comparator.comparingInt(EsqlVersion::id)).get();
+    }
+
+    private int year;
+    private byte month;
+    private byte revision;
+    private String emoji;
+
+    EsqlVersion(int year, int month, String emoji) {
+        this(year, month, 1, emoji);
+    }
+
+    EsqlVersion(int year, int month, int revision, String emoji) {
+        if ((1 <= revision && revision <= 99) == false) {
+            throw new IllegalArgumentException("Version revision number must be between 1 and 99 but was [" + revision + "]");
+        }
+        if ((1 <= month && month <= 12) == false) {
+            throw new IllegalArgumentException("Version month must be between 1 and 12 but was [" + month + "]");
+        }
+        if ((emoji.codePointCount(0, emoji.length()) == 1) == false) {
+            throw new IllegalArgumentException("Version emoji must be a single unicode character but was [" + emoji + "]");
+        }
+        this.year = year;
+        this.month = (byte) month;
+        this.revision = (byte) revision;
+        this.emoji = emoji;
+    }
+
+    public int year() {
+        return year;
+    }
+
+    public byte month() {
+        return month;
+    }
+
+    public byte revision() {
+        return revision;
+    }
+
+    public String emoji() {
+        return emoji;
+    }
+
+    public String versionStringWithoutEmoji() {
+        return this == SNAPSHOT ? "snapshot" : Strings.format("%d.%02d.%02d", year, month, revision);
+    }
+
+    @Override
+    public String toString() {
+        return versionStringWithoutEmoji() + "." + emoji;
+    }
+
+    @Override
+    public int id() {
+        return this == SNAPSHOT ? Integer.MAX_VALUE : (10000 * year + 100 * month + revision);
+    }
+}

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java

@@ -24,6 +24,8 @@ import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
+import org.elasticsearch.xpack.esql.version.EsqlVersion;
+import org.elasticsearch.xpack.esql.version.EsqlVersionTests;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,20 +46,23 @@ public class EsqlQueryRequestTests extends ESTestCase {
         boolean columnar = randomBoolean();
         Locale locale = randomLocale(random());
         QueryBuilder filter = randomQueryBuilder();
+        EsqlVersion esqlVersion = randomFrom(EsqlVersion.values());
 
         List<TypedParamValue> params = randomParameters();
         boolean hasParams = params.isEmpty() == false;
         StringBuilder paramsString = paramsString(params, hasParams);
         String json = String.format(Locale.ROOT, """
             {
+                "version": "%s",
                 "query": "%s",
                 "columnar": %s,
                 "locale": "%s",
                 "filter": %s
-                %s""", query, columnar, locale.toLanguageTag(), filter, paramsString);
+                %s""", esqlVersion, query, columnar, locale.toLanguageTag(), filter, paramsString);
 
         EsqlQueryRequest request = parseEsqlQueryRequestSync(json);
 
+        assertEquals(esqlVersion.toString(), request.esqlVersion());
         assertEquals(query, request.query());
         assertEquals(columnar, request.columnar());
         assertEquals(locale.toLanguageTag(), request.locale().toLanguageTag());
@@ -75,6 +80,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
         boolean columnar = randomBoolean();
         Locale locale = randomLocale(random());
         QueryBuilder filter = randomQueryBuilder();
+        EsqlVersion esqlVersion = randomFrom(EsqlVersion.values());
 
         List<TypedParamValue> params = randomParameters();
         boolean hasParams = params.isEmpty() == false;
@@ -86,6 +92,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
             Locale.ROOT,
             """
                 {
+                    "version": "%s",
                     "query": "%s",
                     "columnar": %s,
                     "locale": "%s",
@@ -94,6 +101,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
                     "wait_for_completion_timeout": "%s",
                     "keep_alive": "%s"
                     %s""",
+            esqlVersion,
             query,
             columnar,
             locale.toLanguageTag(),
@@ -106,6 +114,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
 
         EsqlQueryRequest request = parseEsqlQueryRequestAsync(json);
 
+        assertEquals(esqlVersion.toString(), request.esqlVersion());
         assertEquals(query, request.query());
         assertEquals(columnar, request.columnar());
         assertEquals(locale.toLanguageTag(), request.locale().toLanguageTag());
@@ -149,18 +158,123 @@ public class EsqlQueryRequestTests extends ESTestCase {
             }""", "unknown field [asdf]");
     }
 
-    public void testMissingQueryIsNotValidation() throws IOException {
+    public void testKnownVersionIsValid() throws IOException {
+        for (EsqlVersion version : EsqlVersion.values()) {
+            String validVersionString = randomBoolean() ? version.versionStringWithoutEmoji() : version.toString();
+
+            String json = String.format(Locale.ROOT, """
+                {
+                    "version": "%s",
+                    "query": "ROW x = 1"
+                }
+                """, validVersionString);
+
+            EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+            assertNull(request.validate());
+
+            request = parseEsqlQueryRequestAsync(json);
+            assertNull(request.validate());
+        }
+    }
+
+    public void testUnknownVersionIsNotValid() throws IOException {
+        String invalidVersionString = EsqlVersionTests.randomInvalidVersionString();
+
+        String json = String.format(Locale.ROOT, """
+            {
+                "version": "%s",
+                "query": "ROW x = 1"
+            }
+            """, invalidVersionString);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] has invalid value ["
+                    + invalidVersionString
+                    + "], latest available version is ["
+                    + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+                    + "]"
+            )
+        );
+    }
+
+    public void testSnapshotVersionIsOnlyValidOnSnapshot() throws IOException {
+        String esqlVersion = randomBoolean() ? "snapshot" : "snapshot.📷";
+        String json = String.format(Locale.ROOT, """
+            {
+                "version": "%s",
+                "query": "ROW x = 1"
+            }
+            """, esqlVersion);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+        request.onSnapshotBuild(true);
+        assertNull(request.validate());
+
+        request.onSnapshotBuild(false);
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] with value ["
+                    + esqlVersion
+                    + "] only allowed in snapshot builds, latest available version is ["
+                    + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+                    + "]"
+            )
+        );
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104890")
+    public void testMissingVersionIsNotValid() throws IOException {
+        String missingVersion = randomBoolean() ? "" : ", \"version\": \"\"";
+        String json = String.format(Locale.ROOT, """
+            {
+                "columnar": true,
+                "query": "row x = 1"
+                %s
+            }""", missingVersion);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] is required, latest available version is [" + EsqlVersion.latestReleased().versionStringWithoutEmoji() + "]"
+            )
+        );
+    }
+
+    public void testMissingQueryIsNotValid() throws IOException {
         String json = """
             {
-                "columnar": true
+                "columnar": true,
+                "version": "snapshot"
             }""";
-        EsqlQueryRequest request = parseEsqlQueryRequestSync(json);
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         assertNotNull(request.validate());
         assertThat(request.validate().getMessage(), containsString("[query] is required"));
+    }
+
+    public void testPragmasOnlyValidOnSnapshot() throws IOException {
+        String json = """
+            {
+                "version": "2024.04.01",
+                "query": "ROW x = 1",
+                "pragma": {"foo": "bar"}
+            }
+            """;
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+        request.onSnapshotBuild(true);
+        assertNull(request.validate());
 
-        request = parseEsqlQueryRequestAsync(json);
+        request.onSnapshotBuild(false);
         assertNotNull(request.validate());
-        assertThat(request.validate().getMessage(), containsString("[query] is required"));
+        assertThat(request.validate().getMessage(), containsString("[pragma] only allowed in snapshot builds"));
     }
 
     public void testTask() throws IOException {
@@ -260,6 +374,10 @@ public class EsqlQueryRequestTests extends ESTestCase {
         assertThat(e.getMessage(), containsString(message));
     }
 
+    static EsqlQueryRequest parseEsqlQueryRequest(String json, boolean sync) throws IOException {
+        return sync ? parseEsqlQueryRequestSync(json) : parseEsqlQueryRequestAsync(json);
+    }
+
     static EsqlQueryRequest parseEsqlQueryRequestSync(String json) throws IOException {
         var request = parseEsqlQueryRequest(json, RequestXContent::parseSync);
         assertFalse(request.async());

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/version/EsqlVersionTests.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.version;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class EsqlVersionTests extends ESTestCase {
+    public void testLatestReleased() {
+        assertThat(EsqlVersion.latestReleased(), is(EsqlVersion.ROCKET));
+    }
+
+    public void testVersionString() {
+        assertThat(EsqlVersion.SNAPSHOT.toString(), equalTo("snapshot.📷"));
+        assertThat(EsqlVersion.ROCKET.toString(), equalTo("2024.04.01.🚀"));
+    }
+
+    public void testVersionId() {
+        assertThat(EsqlVersion.SNAPSHOT.id(), equalTo(Integer.MAX_VALUE));
+        assertThat(EsqlVersion.ROCKET.id(), equalTo(20240401));
+
+        for (EsqlVersion version : EsqlVersion.values()) {
+            assertTrue(EsqlVersion.SNAPSHOT.onOrAfter(version));
+            if (version != EsqlVersion.SNAPSHOT) {
+                assertTrue(version.before(EsqlVersion.SNAPSHOT));
+            } else {
+                assertTrue(version.onOrAfter(EsqlVersion.SNAPSHOT));
+            }
+        }
+
+        List<EsqlVersion> versionsSortedAsc = Arrays.stream(EsqlVersion.values())
+            .sorted(Comparator.comparing(EsqlVersion::year).thenComparing(EsqlVersion::month).thenComparing(EsqlVersion::revision))
+            .toList();
+        for (int i = 0; i < versionsSortedAsc.size() - 1; i++) {
+            assertTrue(versionsSortedAsc.get(i).before(versionsSortedAsc.get(i + 1)));
+        }
+    }
+
+    public void testVersionStringNoEmoji() {
+        for (EsqlVersion version : EsqlVersion.values()) {
+            String[] versionSegments = version.toString().split("\\.");
+            String[] parsingPrefixSegments = Arrays.copyOf(versionSegments, versionSegments.length - 1);
+
+            String expectedParsingPrefix = String.join(".", parsingPrefixSegments);
+            assertThat(version.versionStringWithoutEmoji(), equalTo(expectedParsingPrefix));
+        }
+    }
+
+    public void testParsing() {
+        for (EsqlVersion version : EsqlVersion.values()) {
+            String versionStringWithoutEmoji = version.versionStringWithoutEmoji();
+
+            assertThat(EsqlVersion.parse(versionStringWithoutEmoji), is(version));
+            assertThat(EsqlVersion.parse(versionStringWithoutEmoji + "." + version.emoji()), is(version));
+        }
+
+        assertNull(EsqlVersion.parse(randomInvalidVersionString()));
+    }
+
+    public static String randomInvalidVersionString() {
+        String[] invalidVersionString = new String[1];
+
+        do {
+            int length = randomIntBetween(1, 10);
+            invalidVersionString[0] = randomAlphaOfLength(length);
+        } while (EsqlVersion.VERSION_MAP_WITH_AND_WITHOUT_EMOJI.containsKey(invalidVersionString[0]));
+
+        return invalidVersionString[0];
+    }
+}