ESQL - date nanos range bug? (#125345)

Fixes #125439

We were incorrectly formatting nanosecond dates when building lucene queries. We had missed this in our testing because none of the CSV tests were running against Lucene. This happened because the date nanos test data includes multivalue fields. Our warning behavior for multivalue fields is inconsistent between queries run in Lucene and queries run in pure ES|QL without pushdown. Our warning tests, however, require that the specified warnings be present in all execution paths. When we first built the date nanos CSV tests, we worked around this by always using an MV function to unpack the multivalue fields. But we forgot that using an MV function prevents the entire query from being pushed down to Lucene, and thus that path wasn't being tested.

In this PR, I've duplicated many of the tests to have a version that doesn't use the MV function, and uses warningRegex instead of warning. The regex version does not fail if the header is absent, so it's safe to use in both modes. Rewriting the tests this way revealed several situations in which this bug can manifest, all of which are fixed in this PR. I cannot be confidant that there aren't more paths that can trigger this bug and aren't covered by these tests, but I haven't found any yet.

I've left some trace level logging that I found helpful while debugging this.

---------

Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>

docs/changelog/125345.yaml

@@ -0,0 +1,6 @@
+pr: 125345
+summary: ESQL - date nanos range bug?
+area: ES|QL
+type: bug
+issues:
+ - 125439

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec

@@ -216,6 +216,23 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
 ;
 
+date nanos greater than, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos > TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos > TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+;
+
 implicit casting to nanos, date only
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -236,6 +253,29 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
 ;
 
+implicit casting to nanos, date only, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos > "2023-10-23" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
 implicit casting to nanos, date only, equality test
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -248,6 +288,21 @@ FROM date_nanos
 millis:date             | nanos:date_nanos              
 ;
 
+implicit casting to nanos, date only, equality test, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos == "2023-10-23" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+;
+
 
 implicit casting to nanos, date plus time to seconds
 required_capability: date_nanos_type
@@ -269,6 +324,29 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
 ;
 
+implicit casting to nanos, date plus time to seconds, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos > "2023-10-23T00:00:00" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
 implicit casting to nanos, date plus time to seconds, equality test
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -281,6 +359,21 @@ FROM date_nanos
 millis:date             | nanos:date_nanos              
 ;
 
+implicit casting to nanos, date plus time to seconds, equality test, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos == "2023-10-23T12:27:28" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+;
+
 implicit casting to nanos, date plus time to millis
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -301,6 +394,29 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
 ;
 
+implicit casting to nanos, date plus time to millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos > "2023-10-23T00:00:00.000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\.000\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
 implicit casting to nanos, date plus time to millis, equality test
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -314,6 +430,22 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
 ;
 
+implicit casting to nanos, date plus time to millis, equality test, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos == "2023-10-23T12:27:28.948" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\.948\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+;
+
 implicit casting to nanos, date plus time to nanos
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -334,6 +466,29 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
 ;
 
+implicit casting to nanos, date plus time to nanos, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos > "2023-10-23T00:00:00.000000000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\.000000000\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
 implicit casting to nanos, date plus time to nanos, equality test
 required_capability: date_nanos_type
 required_capability: date_nanos_implicit_casting
@@ -347,6 +502,22 @@ millis:date             | nanos:date_nanos
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
 ;
 
+implicit casting to nanos, date plus time to nanos, equality test, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos == "2023-10-23T12:27:28.948000000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\.948000000\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date             | nanos:date_nanos              
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+;
+
 date nanos greater than millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -361,6 +532,23 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
 ;
 
+date nanos greater than millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+;
+
 date nanos greater than or equal
 required_capability: to_date_nanos
 required_capability: date_nanos_binary_comparison
@@ -376,6 +564,24 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+date nanos greater than or equal, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos >= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos >= TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
 date nanos greater than or equal millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -391,6 +597,23 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+date nanos greater than or equal millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos >= TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos >= TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
 
 date nanos less than
 required_capability: to_date_nanos
@@ -411,12 +634,39 @@ millis:date              | nanos:date_nanos               | num:long
 // end::to_date_nanos-result[]
 ;
 
+date nanos less than, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos < TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos < TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
+
 date nanos less than millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
 
 FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
 
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
+date nanos less than millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos < TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos < TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
 millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
@@ -434,6 +684,21 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
 ;
 
+date nanos less than equal, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos <= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos <= TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
+
 date nanos less than equal millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -446,6 +711,20 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
 ;
 
+date nanos less than equal millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos <= TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos <= TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
 
 date nanos equals
 required_capability: to_date_nanos
@@ -457,6 +736,19 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+date nanos equals, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos == TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z");
+warningRegex:Line 1:25: evaluation of \[nanos == TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
 date nanos equals millis exact match
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -467,6 +759,19 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+date nanos equals millis exact match, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos == TO_DATETIME("2023-10-23T12:27:28.948Z");
+warningRegex:Line 1:25: evaluation of \[nanos == TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
 date nanos equals millis without exact match
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -476,6 +781,18 @@ FROM date_nanos | WHERE MV_MIN(nanos) == TO_DATETIME("2023-10-23T13:33:34.937");
 millis:date              | nanos:date_nanos               | num:long
 ;
 
+date nanos equals millis without exact match, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos == TO_DATETIME("2023-10-23T13:33:34.937");
+warningRegex:Line 1:25: evaluation of \[nanos == TO_DATETIME\(\\\"2023-10-23T13:33:34\.937\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+;
+
 date nanos not equals
 required_capability: to_date_nanos
 required_capability: date_nanos_binary_comparison
@@ -492,6 +809,25 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
 ;
 
+date nanos not equals, no mv min
+required_capability: to_date_nanos
+required_capability: date_nanos_binary_comparison
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos != TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos != TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
+
 date nanos not equals millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -508,6 +844,25 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
 ;
 
+date nanos not equals millis, no mv min
+required_capability: date_nanos_type
+required_capability: date_nanos_compare_to_millis
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos | WHERE nanos != TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC;
+warningRegex:Line 1:25: evaluation of \[nanos != TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847
+;
+
 Date nanos date extract
 required_capability: date_nanos_date_extract
 
@@ -530,6 +885,34 @@ nn:date_nanos                  | year:long | ns:long
 2023-03-23T12:15:03.360103847Z | 2023      | 360103847
 ;
 
+Date nanos date extract, no mv max
+required_capability: date_nanos_date_extract
+
+FROM date_nanos
+| EVAL nn = nanos
+| EVAL year = DATE_EXTRACT("year", nn), ns = DATE_EXTRACT("nano_of_second", nn)
+| KEEP nn, year, ns
+| SORT nn DESC;
+# NB: sorting of the MV results isn't deterministic, so we ignore ordering here
+ignoreOrder:true
+warning:Line 3:15: evaluation of [DATE_EXTRACT(\"year\", nn)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 3:15: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warning:Line 3:46: evaluation of [DATE_EXTRACT(\"nano_of_second\", nn)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 3:46: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+nn:date_nanos                  | year:long | ns:long
+2023-10-23T13:55:01.543123456Z | 2023      | 543123456
+2023-10-23T13:53:55.832987654Z | 2023      | 832987654
+2023-10-23T13:52:55.015787878Z | 2023      | 015787878
+2023-10-23T13:51:54.732102837Z | 2023      | 732102837
+2023-10-23T13:33:34.937193000Z | 2023      | 937193000
+2023-10-23T12:27:28.948000000Z | 2023      | 948000000
+2023-10-23T12:15:03.360103847Z | 2023      | 360103847
+2023-10-23T12:15:03.360103847Z | 2023      | 360103847
+[2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z]    | null      | null     
+[2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | null      | null     
+;
+
 date nanos to long, index version
 required_capability: to_date_nanos
 
@@ -1206,6 +1589,22 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+Date Nanos IN constant date nanos, no mv first
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z"));
+ignoreOrder:true
+warningRegex:Line 2:9: evaluation of \[nanos IN \(TO_DATE_NANOS\(\\\"2023-10-23T13:55:01\.543123456Z\\\"\), TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\), TO_DATE_NANOS\(\\\"2017-10-23T13:53:55\.832987654Z\\\"\)\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
 Date Nanos IN constant date nanos, implicit casting
 required_capability: date_nanos_in_operator
 required_capability: to_date_nanos
@@ -1220,6 +1619,23 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
 ;
 
+Date Nanos IN constant date nanos, implicit casting, no mv function
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE nanos IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z");
+ignoreOrder:true
+warningRegex:Line 2:9: evaluation of \[nanos IN \(\\\"2023-10-23T13:55:01\.543123456Z\\\", \\\"2023-10-23T12:27:28\.948Z\\\", \\\"2017-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
 Date Nanos IN date nanos field, implicit casting
 required_capability: date_nanos_in_operator
 required_capability: to_date_nanos
@@ -1232,6 +1648,23 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
 ;
 
+Date Nanos IN date nanos field, implicit casting, no mv function 
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos 
+| WHERE "2023-10-23T13:55:01.543123456Z" IN (nanos);
+ignoreOrder:true
+warningRegex:Line 2:9: evaluation of \[\\\"2023-10-23T13:55:01\.543123456Z\\\" IN \(nanos\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+;
+
+# TODO: should this one work without the mv function? doesn't seem to.
 Date nanos IN millisecond date field
 required_capability: date_nanos_in_operator
 required_capability: to_date_nanos
@@ -1304,3 +1737,28 @@ diff_sec:integer | diff_sec_m:integer | n:date_nanos
 -18489600        | -18489599        | 2023-03-23T12:15:03.360103847Z
 -18489600        | -18489599        | 2023-03-23T12:15:03.360103847Z
 ;
+
+Regression out of bounds in where clause
+required_capability: to_date_nanos
+required_capability: fix_date_nanos_lucene_pushdown_bug
+
+FROM date_nanos
+| WHERE millis > "2020-01-01"
+| WHERE nanos >= to_datenanos("2010-03-20T15:08:25.608Z") AND nanos <= to_datenanos("2026-03-20T15:08:25.608Z")
+| KEEP nanos;
+ignoreOrder:true
+warningRegex:Line 3:63: evaluation of \[nanos <= to_datenanos\(\\\"2026-03-20T15:08:25\.608Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 3:63: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warningRegex:Line 3:9: evaluation of \[nanos >= to_datenanos\(\\\"2010-03-20T15:08:25\.608Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
+warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+nanos:date_nanos
+2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937193Z   
+2023-10-23T12:27:28.948Z      
+2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360103847Z
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -550,6 +550,10 @@ public class EsqlCapabilities {
          * support date diff function on date nanos type, and mixed nanos/millis
          */
         DATE_NANOS_DATE_DIFF(),
+        /**
+         * Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed
+         */
+        FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(),
         /**
          * DATE_PARSE supports reading timezones
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java

@@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.predicate;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@@ -31,18 +33,23 @@ import java.util.Objects;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
 // BETWEEN or range - is a mix of gt(e) AND lt(e)
 public class Range extends ScalarFunction implements TranslationAware.SingleValueTranslationAware {
+    private static final Logger logger = LogManager.getLogger(Range.class);
 
     private final Expression value, lower, upper;
     private final boolean includeLower, includeUpper;
@@ -210,12 +217,19 @@ public class Range extends ScalarFunction implements TranslationAware.SingleValu
         String format = null;
 
         DataType dataType = value.dataType();
-        if (DataType.isDateTime(dataType) && DataType.isDateTime(lower.dataType()) && DataType.isDateTime(upper.dataType())) {
+        logger.trace("Translating Range into lucene query.  dataType is [{}] upper is [{}]  lower is [{}]", dataType, lower, upper);
+        if (dataType == DataType.DATETIME && lower.dataType() == DATETIME && upper.dataType() == DATETIME) {
             l = dateTimeToString((Long) l);
             u = dateTimeToString((Long) u);
             format = DEFAULT_DATE_TIME_FORMATTER.pattern();
         }
 
+        if (dataType == DATE_NANOS && lower.dataType() == DATE_NANOS && upper.dataType() == DATE_NANOS) {
+            l = nanoTimeToString((Long) l);
+            u = nanoTimeToString((Long) u);
+            format = DEFAULT_DATE_NANOS_FORMATTER.pattern();
+        }
+
         if (dataType == IP) {
             if (l instanceof BytesRef bytesRef) {
                 l = ipToString(bytesRef);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java

@@ -13,6 +13,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
@@ -50,14 +52,17 @@ import java.util.Map;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.HOUR_MINUTE_SECOND;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
@@ -66,6 +71,8 @@ public abstract class EsqlBinaryComparison extends BinaryComparison
         EvaluatorMapper,
         TranslationAware.SingleValueTranslationAware {
 
+    private static final Logger logger = LogManager.getLogger(EsqlBinaryComparison.class);
+
     private final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap;
 
     private final BinaryComparisonOperation functionType;
@@ -375,6 +382,16 @@ public abstract class EsqlBinaryComparison extends BinaryComparison
         String format = null;
         boolean isDateLiteralComparison = false;
 
+        logger.trace(
+            "Translating binary comparison with right: [{}<{}>], left: [{}<{}>], attribute: [{}<{}>]",
+            right(),
+            right().dataType(),
+            left(),
+            left().dataType(),
+            attribute,
+            attribute.dataType()
+        );
+
         // TODO: This type coersion layer is copied directly from the QL counterpart code. It's probably not necessary or desireable
         // in the ESQL version. We should instead do the type conversions using our casting functions.
         // for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
@@ -382,7 +399,12 @@ public abstract class EsqlBinaryComparison extends BinaryComparison
         if (value instanceof ZonedDateTime || value instanceof OffsetTime) {
             DateFormatter formatter;
             if (value instanceof ZonedDateTime) {
-                formatter = DEFAULT_DATE_TIME_FORMATTER;
+                // NB: we check the data type of right here because value is the RHS value
+                formatter = switch (right().dataType()) {
+                    case DATETIME -> DEFAULT_DATE_TIME_FORMATTER;
+                    case DATE_NANOS -> DEFAULT_DATE_NANOS_FORMATTER;
+                    default -> throw new EsqlIllegalArgumentException("Found date value in non-date type comparison");
+                };
                 // RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance
                 // which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
                 // Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well.
@@ -408,10 +430,14 @@ public abstract class EsqlBinaryComparison extends BinaryComparison
         }
 
         ZoneId zoneId = null;
-        if (DataType.isDateTime(attribute.dataType())) {
+        if (attribute.dataType() == DATETIME) {
             zoneId = zoneId();
-            value = dateTimeToString((Long) value);
+            value = dateWithTypeToString((Long) value, right().dataType());
             format = DEFAULT_DATE_TIME_FORMATTER.pattern();
+        } else if (attribute.dataType() == DATE_NANOS) {
+            zoneId = zoneId();
+            value = dateWithTypeToString((Long) value, right().dataType());
+            format = DEFAULT_DATE_NANOS_FORMATTER.pattern();
         }
         if (this instanceof GreaterThan) {
             return new RangeQuery(source(), name, value, false, null, false, format, zoneId);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java

@@ -15,6 +15,8 @@ import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.Vector;
 import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -115,6 +117,7 @@ import static org.elasticsearch.xpack.esql.core.util.StringUtils.ordinal;
  */
 public class In extends EsqlScalarFunction implements TranslationAware.SingleValueTranslationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "In", In::new);
+    private static final Logger logger = LogManager.getLogger(In.class);
 
     private final Expression value;
     private final List<Expression> list;
@@ -468,6 +471,7 @@ public class In extends EsqlScalarFunction implements TranslationAware.SingleVal
     }
 
     private Query translate(TranslatorHandler handler) {
+        logger.trace("Attempting to generate lucene query for IN expression");
         TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value());
 
         Set<Object> terms = new LinkedHashSet<>();
@@ -501,7 +505,7 @@ public class In extends EsqlScalarFunction implements TranslationAware.SingleVal
     }
 
     private static boolean needsTypeSpecificValueHandling(DataType fieldType) {
-        return DataType.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG;
+        return fieldType == DATETIME || fieldType == DATE_NANOS || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG;
     }
 
     private static Query or(Source source, Query left, Query right) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java

@@ -40,6 +40,8 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.search.NestedHelper;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -76,6 +78,8 @@ import static org.elasticsearch.compute.lucene.LuceneSourceOperator.NO_LIMIT;
 import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE;
 
 public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProviders {
+    private static final Logger logger = LogManager.getLogger(EsPhysicalOperationProviders.class);
+
     /**
      * Context of each shard we're operating against.
      */
@@ -180,6 +184,7 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi
     @Override
     public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, LocalExecutionPlannerContext context) {
         final LuceneOperator.Factory luceneFactory;
+        logger.trace("Query Exec is {}", esQueryExec);
 
         List<Sort> sorts = esQueryExec.sorts();
         assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized";

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java

@@ -557,6 +557,16 @@ public class EsqlDataTypeConverter {
         return DateUtils.toLong(parsed);
     }
 
+    public static String dateWithTypeToString(long dateTime, DataType type) {
+        if (type == DATETIME) {
+            return dateTimeToString(dateTime);
+        }
+        if (type == DATE_NANOS) {
+            return nanoTimeToString(dateTime);
+        }
+        throw new IllegalArgumentException("Unsupported data type [" + type + "]");
+    }
+
     public static String dateTimeToString(long dateTime) {
         return DEFAULT_DATE_TIME_FORMATTER.formatMillis(dateTime);
     }