ESQL: allow DATE_PARSE to read the timezones (#118603) (#119118)

This just removes fixing a formatter to a timezone (UTC), allowing
`DATE_PARSE` to correctly read timezones.

Fixes #117680.

docs/changelog/118603.yaml

@@ -0,0 +1,6 @@
+pr: 118603
+summary: Allow DATE_PARSE to read the timezones
+area: ES|QL
+type: bug
+issues:
+ - 117680

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

@@ -495,6 +495,60 @@ b:datetime
 null
 ;
 
+evalDateParseWithTimezone
+required_capability: date_parse_tz
+row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+
+s:keyword                  | d:datetime
+12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
+;
+
+evalDateParseWithTimezoneCrossingDayBoundary
+required_capability: date_parse_tz
+row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+
+s:keyword                  | d:datetime
+12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
+;
+
+evalDateParseWithTimezone2
+required_capability: date_parse_tz
+row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
+| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
+| eval eq = d1 == d2
+| keep d1, eq
+;
+
+d1:datetime              | eq:boolean
+2022-07-12T01:24:10.000Z | true
+;
+
+evalDateParseWithAndWithoutTimezone
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval no_tz = date_parse(format, s)
+| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| keep s, no_tz, with_tz
+;
+
+s:keyword           | no_tz:datetime           | with_tz:datetime
+2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
+evalDateParseWithOtherTimezoneSpecifiers
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
+| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
+| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
+| keep s, with_tz*
+;
+
+s:keyword           | with_tz1:datetime        | with_tz2:datetime        | with_tz3:datetime        | with_tz4:datetime        
+2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
 evalDateParseDynamic
 from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no 
 | eval birth_date_string = date_format("yyyy-MM-dd", birth_date)

x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java

@@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import java.time.ZoneId;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -31,18 +30,15 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
 
   private final EvalOperator.ExpressionEvaluator formatter;
 
-  private final ZoneId zoneId;
-
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public DateParseEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
-      EvalOperator.ExpressionEvaluator formatter, ZoneId zoneId, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator formatter, DriverContext driverContext) {
     this.source = source;
     this.val = val;
     this.formatter = formatter;
-    this.zoneId = zoneId;
     this.driverContext = driverContext;
   }
 
@@ -91,7 +87,7 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
           continue position;
         }
         try {
-          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.zoneId));
+          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch)));
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -108,7 +104,7 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
       BytesRef formatterScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch), this.zoneId));
+          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch)));
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -120,7 +116,7 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
 
   @Override
   public String toString() {
-    return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]";
+    return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]";
   }
 
   @Override
@@ -147,24 +143,21 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
 
     private final EvalOperator.ExpressionEvaluator.Factory formatter;
 
-    private final ZoneId zoneId;
-
     public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val,
-        EvalOperator.ExpressionEvaluator.Factory formatter, ZoneId zoneId) {
+        EvalOperator.ExpressionEvaluator.Factory formatter) {
       this.source = source;
       this.val = val;
       this.formatter = formatter;
-      this.zoneId = zoneId;
     }
 
     @Override
     public DateParseEvaluator get(DriverContext context) {
-      return new DateParseEvaluator(source, val.get(context), formatter.get(context), zoneId, context);
+      return new DateParseEvaluator(source, val.get(context), formatter.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]";
+      return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]";
     }
   }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -373,6 +373,11 @@ public class EsqlCapabilities {
          */
         DATE_NANOS_AGGREGATIONS(),
 
+        /**
+         * DATE_PARSE supports reading timezones
+         */
+        DATE_PARSE_TZ(),
+
         /**
          * Support for datetime in least and greatest functions
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java

@@ -28,14 +28,12 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunctio
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
-import java.time.ZoneId;
 import java.util.List;
 
 import static org.elasticsearch.common.time.DateFormatter.forPattern;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
@@ -130,13 +128,12 @@ public class DateParse extends EsqlScalarFunction implements OptionalArgument {
     }
 
     @Evaluator(warnExceptions = { IllegalArgumentException.class })
-    static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
-        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
+    static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
+        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
     }
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        ZoneId zone = UTC; // TODO session timezone?
         ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
         if (format == null) {
             return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
@@ -146,18 +143,18 @@ public class DateParse extends EsqlScalarFunction implements OptionalArgument {
         }
         if (format.foldable()) {
             try {
-                DateFormatter formatter = toFormatter(format.fold(), zone);
+                DateFormatter formatter = toFormatter(format.fold());
                 return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
             } catch (IllegalArgumentException e) {
                 throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
             }
         }
         ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
-        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
+        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
     }
 
-    private static DateFormatter toFormatter(Object format, ZoneId zone) {
-        return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
+    private static DateFormatter toFormatter(Object format) {
+        return forPattern(((BytesRef) format).utf8ToString());
     }
 
     @Override

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTe
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -46,11 +47,26 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
                 ),
+                new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+                    long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
+                    int hours = randomIntBetween(0, 23);
+                    String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
+                    long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
+                    return new TestCaseSupplier.TestCase(
+                        List.of(
+                            new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
+                            new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
+                        ),
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
+                        DataType.DATETIME,
+                        equalTo(expected_ts)
+                    );
+                }),
                 new TestCaseSupplier(
                     "With Text",
                     List.of(DataType.KEYWORD, DataType.TEXT),
@@ -59,7 +75,7 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -72,7 +88,7 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -85,7 +101,7 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -98,7 +114,7 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -118,7 +134,7 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
                             new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")