DOC: Enhance SQL Functions documentation

Split function section into multiple chapters
Add String functions
Add (small) section on Conversion/Cast functions
Add missing aggregation functions
Enable documentation testing (was disabled by accident). While at it,
fix failing tests
Improve spec tests to allow multi-line queries (useful for docs)
Add ability to ignore a spec test (name should end with -Ignore)

docs/reference/sql/functions/aggs.asciidoc

@@ -0,0 +1,168 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-aggs]]
+=== Aggregate Functions
+
+Functions for computing a _single_ result from a set of input values.
+{es-sql} supports aggregate functions only alongside <<sql-syntax-group-by,grouping>> (implicit or explicit).
+
+==== General Purpose
+
+[[sql-functions-aggs-avg]]
+===== `AVG`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Arithmetic_mean[Average] (arithmetic mean) of input values.
+
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggAvg]
+----
+
+[[sql-functions-aggs-count]]
+===== `COUNT`
+
+*Input*: Any, *Output*: `bigint`
+
+Total number (count) of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggCountStar]
+----
+
+[[sql-functions-aggs-count-distinct]]
+===== `COUNT(DISTINCT)`
+
+*Input*: Any, *Output*: `bigint`
+
+Total number of _distinct_ values in input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggCountDistinct]
+----
+
+[[sql-functions-aggs-max]]
+===== `MAX`
+
+*Input*: Numeric, *Output*: Same as input
+
+Maximum value across input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggMax]
+----
+
+[[sql-functions-aggs-min]]
+===== `MIN`
+
+*Input*: Numeric, *Output*: Same as input
+
+Minimum value across input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggMin]
+----
+
+[[sql-functions-aggs-sum]]
+===== `SUM`
+
+*Input*: Numeric, *Output*: `bigint` for integer input, `double` for floating points
+
+Sum of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggSum]
+----
+
+==== Statistics
+
+[[sql-functions-aggs-kurtosis]]
+===== `KURTOSIS`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Kurtosis[Quantify] the shape of the distribution of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggKurtosis]
+----
+
+[[sql-functions-aggs-percentile]]
+===== `PERCENTILE`
+
+*Input*: Numeric, *Output*: `double`
+
+The nth https://en.wikipedia.org/wiki/Percentile[percentile] of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggPercentile]
+----
+
+[[sql-functions-aggs-percentile-rank]]
+===== `PERCENTILE_RANK`
+
+*Input*: Numeric, *Output*: `double`
+
+The https://en.wikipedia.org/wiki/Percentile_rank[percentile rank] of input values of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggPercentileRank]
+----
+
+[[sql-functions-aggs-skewness]]
+===== `SKEWNESS`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Skewness[Quantify] the asymmetric distribution of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggSkewness]
+----
+
+[[sql-functions-aggs-stddev-pop]]
+===== `STDDEV_POP`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Standard_deviations[Population standard deviation] of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggStddevPop]
+----
+
+[[sql-functions-aggs-sum-squares]]
+===== `SUM_OF_SQUARES`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Total_sum_of_squares[Sum of squares] of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggSumOfSquares]
+----
+
+[[sql-functions-aggs-var-pop]]
+===== `VAR_POP`
+
+*Input*: Numeric, *Output*: `double`
+
+https://en.wikipedia.org/wiki/Variance[Population] variance of input values.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[aggVarPop]
+----

docs/reference/sql/functions/date-time.asciidoc

@@ -0,0 +1,94 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-datetime]]
+=== Date and Time Functions
+
+* Extract the year from a date (`YEAR`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[year]
+--------------------------------------------------
+
+* Extract the month of the year from a date (`MONTH_OF_YEAR` or `MONTH`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[monthOfYear]
+--------------------------------------------------
+
+* Extract the week of the year from a date (`WEEK_OF_YEAR` or `WEEK`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[weekOfYear]
+--------------------------------------------------
+
+* Extract the day of the year from a date (`DAY_OF_YEAR` or `DOY`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[dayOfYear]
+--------------------------------------------------
+
+* Extract the day of the month from a date (`DAY_OF_MONTH`, `DOM`, or `DAY`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[dayOfMonth]
+--------------------------------------------------
+
+* Extract the day of the week from a date (`DAY_OF_WEEK` or `DOW`).
+Monday is `1`, Tuesday is `2`, etc.
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[dayOfWeek]
+--------------------------------------------------
+
+* Extract the hour of the day from a date (`HOUR_OF_DAY` or `HOUR`).
+Monday is `1`, Tuesday is `2`, etc.
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[hourOfDay]
+--------------------------------------------------
+
+* Extract the minute of the day from a date (`MINUTE_OF_DAY`).
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[minuteOfDay]
+--------------------------------------------------
+
+* Extract the minute of the hour from a date (`MINUTE_OF_HOUR`, `MINUTE`).
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[minuteOfHour]
+--------------------------------------------------
+
+* Extract the second of the minute from a date (`SECOND_OF_MINUTE`, `SECOND`).
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[secondOfMinute]
+--------------------------------------------------
+
+* Extract
+
+As an alternative, one can support `EXTRACT` to extract fields from datetimes.
+You can run any <<sql-functions-datetime,datetime function>>
+with `EXTRACT(<datetime_function> FROM <expression>)`. So
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[extractDayOfYear]
+--------------------------------------------------
+
+is the equivalent to
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/datetime.csv-spec[dayOfYear]
+--------------------------------------------------

docs/reference/sql/functions/index.asciidoc

@@ -3,416 +3,20 @@
 [[sql-functions]]
 == Functions and Operators
 
-{es-sql} provides a number of built-in operators and functions.
-
-=== Comparison Operators
-
-{es-sql} supports the following comparison operators:
-
-* Equality (`=`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality]
---------------------------------------------------
-
-* Inequality (`<>` or `!=` or `<=>`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldNonEquality]
---------------------------------------------------
-
-* Comparison (`<`, `<=`, `>`, `>=`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldLessThan]
---------------------------------------------------
-
-* `BETWEEN`
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereBetween]
---------------------------------------------------
-
-* `IS NULL`/`IS NOT NULL`
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull]
---------------------------------------------------
-
-
-=== Logical Operators
-
-{es-sql} supports the following logical operators:
-
-* `AND`
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldAndComparison]
---------------------------------------------------
-
-* `OR`
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldOrComparison]
---------------------------------------------------
-
-* `NOT`
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/filter.sql-spec[whereFieldEqualityNot]
---------------------------------------------------
-
-
-=== Math Operators
-
-{es-sql} supports the following math operators:
-
-* Add (`+`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[plus]
---------------------------------------------------
-
-* Subtract (infix `-`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[minus]
---------------------------------------------------
-
-* Negate (unary `-`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[unaryMinus]
---------------------------------------------------
-
-* Multiply (`*`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
---------------------------------------------------
-
-* Divide (`/`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Reminder(`%`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
---------------------------------------------------
-
-
-=== Math Functions
-
-All math and trigonometric functions require their input (where applicable)
-to be numeric.
-
-==== Generic
-
-* `ABS`
-
-https://en.wikipedia.org/wiki/Absolute_value[Absolute value], returns \[same type as input]
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[abs]
---------------------------------------------------
-
-* `CBRT`
-
-https://en.wikipedia.org/wiki/Cube_root[Cube root], returns `double`
-
-// TODO make the example in the tests presentable
-
-* `CEIL`
-
-https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling], returns `double`
-
-* `CEILING`
-
-Same as `CEIL`
-
-// TODO make the example in the tests presentable
-
-* `E`
-
-https://en.wikipedia.org/wiki/E_%28mathematical_constant%29[Euler's number], returns `2.7182818284590452354`
-
-
-* https://en.wikipedia.org/wiki/Rounding#Round_half_up[Round] (`ROUND`)
-
-// TODO make the example in the tests presentable
-
-NOTE: This rounds "half up" meaning that `ROUND(-1.5)` results in `-1`.
-
-
-* https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Floor] (`FLOOR`)
-
-// TODO make the example in the tests presentable
-
-* https://en.wikipedia.org/wiki/Natural_logarithm[Natural logarithm] (`LOG`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[log]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Logarithm[Logarithm] base 10 (`LOG10`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[log10]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Square_root[Square root] (`SQRT`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[sqrt]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Exponential_function[e^x^] (`EXP`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[exp]
---------------------------------------------------
-
-* https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#expm1-double-[e^x^ - 1] (`EXPM1`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[expm1]
---------------------------------------------------
-
-==== Trigonometric
-
-* Convert from https://en.wikipedia.org/wiki/Radian[radians]
-to https://en.wikipedia.org/wiki/Degree_(angle)[degrees] (`DEGREES`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[degrees]
---------------------------------------------------
-
-* Convert from https://en.wikipedia.org/wiki/Degree_(angle)[degrees]
-to https://en.wikipedia.org/wiki/Radian[radians] (`RADIANS`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[degrees]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Trigonometric_functions#sine[Sine] (`SIN`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[sin]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Trigonometric_functions#cosine[Cosine] (`COS`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[cos]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Trigonometric_functions#tangent[Tangent] (`TAN`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[tan]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc sine] (`ASIN`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[asin]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc cosine] (`ACOS`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[acos]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc tangent] (`ATAN`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[atan]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Hyperbolic_function[Hyperbolic sine] (`SINH`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[sinh]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Hyperbolic_function[Hyperbolic cosine] (`COSH`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/math.sql-spec[cosh]
---------------------------------------------------
-
-[[sql-functions-datetime]]
-=== Date and Time Functions
-
-* Extract the year from a date (`YEAR`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[year]
---------------------------------------------------
-
-* Extract the month of the year from a date (`MONTH_OF_YEAR` or `MONTH`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[monthOfYear]
---------------------------------------------------
-
-* Extract the week of the year from a date (`WEEK_OF_YEAR` or `WEEK`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[weekOfYear]
---------------------------------------------------
-
-* Extract the day of the year from a date (`DAY_OF_YEAR` or `DOY`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[dayOfYear]
---------------------------------------------------
-
-* Extract the day of the month from a date (`DAY_OF_MONTH`, `DOM`, or `DAY`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[dayOfMonth]
---------------------------------------------------
-
-* Extract the day of the week from a date (`DAY_OF_WEEK` or `DOW`).
-Monday is `1`, Tuesday is `2`, etc.
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[dayOfWeek]
---------------------------------------------------
-
-* Extract the hour of the day from a date (`HOUR_OF_DAY` or `HOUR`).
-Monday is `1`, Tuesday is `2`, etc.
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[hourOfDay]
---------------------------------------------------
-
-* Extract the minute of the day from a date (`MINUTE_OF_DAY`).
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[minuteOfDay]
---------------------------------------------------
-
-* Extract the minute of the hour from a date (`MINUTE_OF_HOUR`, `MINUTE`).
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[minuteOfHour]
---------------------------------------------------
-
-* Extract the second of the minute from a date (`SECOND_OF_MINUTE`, `SECOND`).
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[secondOfMinute]
---------------------------------------------------
-
-* Extract
-
-As an alternative, one can support `EXTRACT` to extract fields from datetimes.
-You can run any <<sql-functions-datetime,datetime function>>
-with `EXTRACT(<datetime_function> FROM <expression>)`. So
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[extractDayOfYear]
---------------------------------------------------
-
-is the equivalent to
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/datetime.csv-spec[dayOfYear]
---------------------------------------------------
-
-
-[[sql-functions-aggregate]]
-=== Aggregate Functions
-
-==== Basic
-
-* https://en.wikipedia.org/wiki/Arithmetic_mean[Average] (`AVG`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.sql-spec[avg]
---------------------------------------------------
-
-* Count the number of matching fields (`COUNT`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.sql-spec[countStar]
---------------------------------------------------
-
-* Count the number of distinct values in matching documents (`COUNT(DISTINCT`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.sql-spec[countDistinct]
---------------------------------------------------
-
-* Find the maximum value in matching documents (`MAX`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.sql-spec[max]
---------------------------------------------------
-
-* Find the minimum value in matching documents (`MIN`)
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.sql-spec[min]
---------------------------------------------------
-
-* https://en.wikipedia.org/wiki/Kahan_summation_algorithm[Sum]
-all values of matching documents (`SUM`).
-
-["source","sql",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{sql-specs}/agg.csv-spec[sum]
---------------------------------------------------
+{es-sql} provides a comprehensive set of built-in operators and functions:
+
+* <<sql-operators, Operators>>
+* <<sql-functions-aggs, Aggregate>>
+* <<sql-functions-datetime, Date-Time>>
+* <<sql-functions-search, Full-Text Search>>
+* <<sql-functions-math, Mathematical>>
+* <<sql-functions-string, String>>
+* <<sql-functions-type-conversion,Type Conversion>>
+
+include::operators.asciidoc[]
+include::aggs.asciidoc[]
+include::date-time.asciidoc[]
+include::search.asciidoc[]
+include::math.asciidoc[]
+include::string.asciidoc[]
+include::type-conversion.asciidoc[]

docs/reference/sql/functions/math.asciidoc

@@ -0,0 +1,159 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-math]]
+=== Math Functions
+
+All math and trigonometric functions require their input (where applicable)
+to be numeric.
+
+==== Generic
+
+* `ABS`
+
+https://en.wikipedia.org/wiki/Absolute_value[Absolute value], returns \[same type as input]
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[abs]
+--------------------------------------------------
+
+* `CBRT`
+
+https://en.wikipedia.org/wiki/Cube_root[Cube root], returns `double`
+
+// TODO make the example in the tests presentable
+
+* `CEIL`
+
+https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling], returns `double`
+
+* `CEILING`
+
+Same as `CEIL`
+
+// TODO make the example in the tests presentable
+
+* `E`
+
+https://en.wikipedia.org/wiki/E_%28mathematical_constant%29[Euler's number], returns `2.7182818284590452354`
+
+
+* https://en.wikipedia.org/wiki/Rounding#Round_half_up[Round] (`ROUND`)
+
+// TODO make the example in the tests presentable
+
+NOTE: This rounds "half up" meaning that `ROUND(-1.5)` results in `-1`.
+
+
+* https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Floor] (`FLOOR`)
+
+// TODO make the example in the tests presentable
+
+* https://en.wikipedia.org/wiki/Natural_logarithm[Natural logarithm] (`LOG`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[log]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Logarithm[Logarithm] base 10 (`LOG10`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[log10]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Square_root[Square root] (`SQRT`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[sqrt]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Exponential_function[e^x^] (`EXP`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[exp]
+--------------------------------------------------
+
+* https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#expm1-double-[e^x^ - 1] (`EXPM1`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[expm1]
+--------------------------------------------------
+
+==== Trigonometric
+
+* Convert from https://en.wikipedia.org/wiki/Radian[radians]
+to https://en.wikipedia.org/wiki/Degree_(angle)[degrees] (`DEGREES`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[degrees]
+--------------------------------------------------
+
+* Convert from https://en.wikipedia.org/wiki/Degree_(angle)[degrees]
+to https://en.wikipedia.org/wiki/Radian[radians] (`RADIANS`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[degrees]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Trigonometric_functions#sine[Sine] (`SIN`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[sin]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Trigonometric_functions#cosine[Cosine] (`COS`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[cos]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Trigonometric_functions#tangent[Tangent] (`TAN`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[tan]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc sine] (`ASIN`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[asin]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc cosine] (`ACOS`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[acos]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Inverse_trigonometric_functions[Arc tangent] (`ATAN`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[atan]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Hyperbolic_function[Hyperbolic sine] (`SINH`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[sinh]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Hyperbolic_function[Hyperbolic cosine] (`COSH`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/math.sql-spec[cosh]
+--------------------------------------------------

docs/reference/sql/functions/operators.asciidoc

@@ -0,0 +1,115 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-operators]]
+=== Comparison Operators
+
+Boolean operator for comparing one or two expressions.
+
+* Equality (`=`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality]
+--------------------------------------------------
+
+* Inequality (`<>` or `!=` or `<=>`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldNonEquality]
+--------------------------------------------------
+
+* Comparison (`<`, `<=`, `>`, `>=`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldLessThan]
+--------------------------------------------------
+
+* `BETWEEN`
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereBetween]
+--------------------------------------------------
+
+* `IS NULL`/`IS NOT NULL`
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull]
+--------------------------------------------------
+
+[[sql-operators-logical]]
+=== Logical Operators
+
+Boolean operator for evaluating one or two expressions.
+
+* `AND`
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldAndComparison]
+--------------------------------------------------
+
+* `OR`
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldOrComparison]
+--------------------------------------------------
+
+* `NOT`
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/filter.sql-spec[whereFieldEqualityNot]
+--------------------------------------------------
+
+[[sql-operators-math]]
+=== Math Operators
+
+Perform mathematical operations affecting one or two values.
+The result is a value of numeric type.
+
+* Add (`+`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[plus]
+--------------------------------------------------
+
+* Subtract (infix `-`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[minus]
+--------------------------------------------------
+
+* Negate (unary `-`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[unaryMinus]
+--------------------------------------------------
+
+* Multiply (`*`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
+--------------------------------------------------
+
+* Divide (`/`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
+--------------------------------------------------
+
+* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Reminder(`%`)
+
+["source","sql",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
+--------------------------------------------------

docs/reference/sql/functions/search.asciidoc

@@ -0,0 +1,35 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-search]]
+=== Full-Text Search Functions
+
+Search functions should be used when performing full-text search, namely
+when the `MATCH` or `QUERY` predicates are being used.
+Outside a, so-called, search context, these functions will return default values
+such as `0` or `NULL`.
+
+[[sql-functions-search-score]]
+==== `SCORE`
+
+*Input*: None, *Output*: `double`
+
+Returns the {defguide}/relevance-intro.html[relevance] of a given input to the executed query. 
+The higher score, the more relevant the data.
+
+NOTE: When doing multiple text queries in the `WHERE` clause then, their scores will be
+combined using the same rules as {es}'s
+<<query-dsl-bool-query,bool query>>. 
+
+Typically `SCORE` is used for ordering the results of a query based on their relevance:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[orderByScore]
+----
+
+However, it is perfectly fine to return the score without sorting by it:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[scoreWithMatch]
+----

docs/reference/sql/functions/string.asciidoc

@@ -0,0 +1,240 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-string]]
+=== String Functions
+
+Functions for performing string manipulation.
+
+[[sql-functions-string-ascii]]
+==== `ASCII`
+
+*Input*: `string`, *Output*: `integer`
+
+Returns the ASCII code value of the leftmost character of string_exp as an integer.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringAscii]
+----
+
+[[sql-functions-string-bit-length]]
+==== `BIT_LENGTH`
+
+*Input*: `string`, *Output*: `integer`
+
+Returns the length in bits of the input.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringBitLength]
+----
+
+[[sql-functions-string-char]]
+==== `CHAR`
+
+*Input*: `numeric`, *Output*: `string`
+
+Returns the character that has the ASCII code value specified by the numeric input. The value should be between 0 and 255; otherwise, the return value is data source–dependent.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringChar]
+----
+
+[[sql-functions-string-char-length]]
+==== `CHAR_LENGTH`
+
+*Input*: `string`, *Output*: `integer`
+
+Returns the length in characters of the input, if the string expression is of a character data type; otherwise, returns the length in bytes of the string expression (the smallest integer not less than the number of bits divided by 8).
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringCharLength]
+----
+
+[[sql-functions-string-concat]]
+==== `CONCAT`
+
+*Input*: `string1`, `string2`, *Output*: `string`
+
+turns a character string that is the result of concatenating string1 to string2. If one of the string is `NULL`,
+the other string will be returned.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringConcat]
+----
+
+[[sql-functions-string-insert]]
+==== `INSERT`
+
+*Input*: `string1`, `start`, `length`, `string2`, *Output*: `string`
+
+Returns a string where length characters have been deleted from string1, beginning at start, and where string2 has been inserted into string1, beginning at start.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringInsert]
+----
+
+[[sql-functions-string-lcase]]
+==== `LCASE`
+
+*Input*: `string`, *Output*: `string`
+
+Returns a string equal to that in string, with all uppercase characters converted to lowercase.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLCase]
+----
+
+[[sql-functions-string-left]]
+==== `LEFT`
+
+*Input*: `string`, *Output*: `string`
+
+Returns the leftmost count characters of string.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLeft]
+----
+
+[[sql-functions-string-length]]
+==== `LENGTH`
+
+*Input*: `string`, *Output*: `integer`
+
+Returns the number of characters in string, excluding trailing blanks.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLength]
+----
+
+[[sql-functions-string-locate]]
+==== `LOCATE`
+
+*Input*: `string1`, `string2`[, `start`]`, *Output*: `integer`
+
+Returns the starting position of the first occurrence of string1 within string2. The search for the first occurrence of string1 begins with the first character position in string2 unless the optional argument, start, is specified. If start is specified, the search begins with the character position indicated by the value of start. The first character position in string2 is indicated by the value 1. If string1 is not found within string2, the value 0 is returned.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLocateWoStart]
+----
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLocateWithStart]
+----
+
+[[sql-functions-string-ltrim]]
+==== `LTRIM`
+
+*Input*: `string`, *Output*: `string`
+
+Returns the characters of string_exp, with leading blanks removed.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringLTrim]
+----
+
+[[sql-functions-string-position]]
+==== `POSITION`
+
+*Input*: `string1`, `string2`, *Output*: `integer`
+
+Returns the position of the string1 in string2. The result is an exact numeric.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringPosition]
+----
+
+[[sql-functions-string-repeat]]
+==== `REPEAT`
+
+*Input*: `string`, `count`, *Output*: `string`
+
+Returns a character string composed of string1 repeated count times.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringRepeat]
+----
+
+[[sql-functions-string-replace]]
+==== `REPLACE`
+
+*Input*: `string1`, `string2`, `string3`, *Output*: `string`
+
+Search string1 for occurrences of string2, and replace with string3.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringReplace]
+----
+
+[[sql-functions-string-right]]
+==== `RIGHT`
+
+*Input*: `string`, `count`, *Output*: `string`
+
+Returns the rightmost count characters of string.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringRight]
+----
+
+[[sql-functions-string-rtrim]]
+==== `RTRIM`
+
+*Input*: `string`, *Output*: `string`
+
+Returns the characters of string with trailing blanks removed.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringRTrim]
+----
+
+[[sql-functions-string-space]]
+==== `SPACE`
+
+*Input*: `integer`, *Output*: `string`
+
+Returns a character string consisting of count spaces.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringSpace]
+----
+
+[[sql-functions-string-substring]]
+==== `SUBSTRING`
+
+*Input*: `string`, `start`, `length`, *Output*: `integer`
+
+Returns a character string that is derived from the string, beginning at the character position specified by `start` for `length` characters.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringSubString]
+----
+
+[[sql-functions-string-ucase]]
+==== `UCASE`
+
+*Input*: `string`, *Output*: `string`
+
+Returns a string equal to that of the input, with all lowercase characters converted to uppercase.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[stringUCase]
+----

docs/reference/sql/functions/type-conversion.asciidoc

@@ -0,0 +1,39 @@
+[role="xpack"]
+[testenv="basic"]
+[[sql-functions-type-conversion]]
+=== Type Conversion Functions
+
+Functions for converting an expression of one data type to another.
+
+[[sql-functions-type-conversion-cast]]
+==== `CAST`
+
+.Synopsis
+[source, sql]
+----
+CAST ( expression<1> AS data_type<2> )
+----
+
+<1> Expression to cast
+<2> Target data type to cast to
+
+.Description
+
+Casts the result of the given expression to the target type.
+If the cast is not possible (for example because of target type is too narrow or because
+the value itself cannot be converted), the query fails.
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[conversionStringToIntCast]
+----
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[conversionIntToStringCast]
+----
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[conversionStringToDateCast]
+----

docs/reference/sql/language/data-types.asciidoc

@@ -17,7 +17,7 @@ Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicat
 | <<number, `byte`>>          | `tinyint`   | 3
 | <<number, `short`>>         | `smallint`  | 5
 | <<number, `integer`>>       | `integer`   | 10
-| <<number, `long`>>          | `long`      | 19
+| <<number, `long`>>          | `bigint`    | 19
 | <<number, `double`>>        | `double`    | 15
 | <<number, `float`>>         | `real`      | 7
 | <<number, `half_float`>>    | `float`     | 16

x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDocCsvSpectIT.java → x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDocCsvSpecIT.java

@@ -11,7 +11,7 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
 import org.elasticsearch.xpack.qa.sql.jdbc.DataLoader;
-import org.elasticsearch.xpack.qa.sql.jdbc.JdbcTestUtils;
+import org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert;
 import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
 import org.elasticsearch.xpack.qa.sql.jdbc.SqlSpecTestCase;
 
@@ -36,7 +36,7 @@ import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
  * That's not to say the two cannot be merged however that felt like too much of an effort
  * at this stage and, to not keep things stalling, started with this approach.
  */
-public class JdbcDocCsvSpectIT extends SpecBaseIntegrationTestCase {
+public class JdbcDocCsvSpecIT extends SpecBaseIntegrationTestCase {
 
     private final CsvTestCase testCase;
 
@@ -56,7 +56,7 @@ public class JdbcDocCsvSpectIT extends SpecBaseIntegrationTestCase {
         return readScriptSpec("/docs.csv-spec", parser);
     }
 
-    public JdbcDocCsvSpectIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+    public JdbcDocCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
         super(fileName, groupName, testName, lineNumber);
         this.testCase = testCase;
     }
@@ -68,8 +68,8 @@ public class JdbcDocCsvSpectIT extends SpecBaseIntegrationTestCase {
         //
         // uncomment this to printout the result set and create new CSV tests
         //
-        JdbcTestUtils.logLikeCLI(elastic, log);
-        //JdbcAssert.assertResultSets(expected, elastic, log, true);
+        //JdbcTestUtils.logLikeCLI(elastic, log);
+        JdbcAssert.assertResultSets(expected, elastic, log, true);
     }
 
     @Override

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java

@@ -160,17 +160,28 @@ public final class CsvTestUtils {
     }
 
     private static class CsvSpecParser implements SpecBaseIntegrationTestCase.Parser {
+        private final StringBuilder query = new StringBuilder();
         private final StringBuilder data = new StringBuilder();
         private CsvTestCase testCase;
 
         @Override
         public Object parse(String line) {
-            // beginning of the section
+            // read the query
             if (testCase == null) {
-                // pick up the query
-                testCase = new CsvTestCase();
-                testCase.query = line.endsWith(";") ? line.substring(0, line.length() - 1) : line;
+                if (line.endsWith(";")) {
+                    // pick up the query
+                    testCase = new CsvTestCase();
+                    query.append(line.substring(0, line.length() - 1).trim());
+                    testCase.query = query.toString();
+                    query.setLength(0);
+                }
+                // keep reading the query
+                else {
+                    query.append(line);
+                    query.append("\r\n");
+                }
             }
+            // read the results
             else {
                 // read data
                 if (line.startsWith(";")) {

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java

@@ -85,6 +85,7 @@ public abstract class SpecBaseIntegrationTestCase extends JdbcIntegrationTestCas
 
     public final void test() throws Throwable {
         try {
+            assumeFalse("Test marked as Ignored", testName.endsWith("-Ignore"));
             doTest();
         } catch (AssertionError ae) {
             throw reworkException(ae);

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java

@@ -25,13 +25,13 @@ public abstract class SqlSpecTestCase extends SpecBaseIntegrationTestCase {
     private String query;
 
     @ClassRule
-    public static LocalH2 H2 = new LocalH2((c) -> { 
+    public static LocalH2 H2 = new LocalH2((c) -> {
         c.createStatement().execute("RUNSCRIPT FROM 'classpath:/setup_test_emp.sql'");
         c.createStatement().execute("RUNSCRIPT FROM 'classpath:/setup_test_emp_with_nulls.sql'");
     });
 
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
-    public static List<Object[]> readScriptSpec() throws Exception { 
+    public static List<Object[]> readScriptSpec() throws Exception {
         Parser parser = specParser();
         List<Object[]> tests = new ArrayList<>();
         tests.addAll(readScriptSpec("/select.sql-spec", parser));
@@ -47,9 +47,22 @@ public abstract class SqlSpecTestCase extends SpecBaseIntegrationTestCase {
     }
 
     private static class SqlSpecParser implements Parser {
+        private final StringBuilder query = new StringBuilder();
+
         @Override
         public Object parse(String line) {
-            return line.endsWith(";") ? line.substring(0, line.length() - 1) : line;
+            // not initialized
+            String q = null;
+            if (line.endsWith(";")) {
+                query.append(line.substring(0, line.length() - 1));
+                q = query.toString();
+                query.setLength(0);
+            } else {
+                query.append(line);
+                query.append("\r\n");
+            }
+
+            return q;
         }
     }
 

x-pack/qa/sql/src/main/resources/agg.csv-spec

@@ -66,11 +66,9 @@ F                    | 10099.1936           | 10098.021            | 26.35135135
 M                    | 10095.6112           | 10090.846            | 23.41269841269841
 ;
 
-// Simple sum used in documentation
 sum
-// tag::sum
 SELECT SUM(salary) FROM test_emp;
-// end::sum
+
   SUM(salary)
 ---------------
 4824855

x-pack/qa/sql/src/main/resources/agg.sql-spec

@@ -87,9 +87,7 @@ SELECT (emp_no % 3) + 1 AS e, (languages % 3) + 1 AS l FROM test_emp GROUP BY e,
 
 // COUNT
 aggCountImplicit
-// tag::countStar
 SELECT COUNT(*) AS count FROM test_emp;
-// end::countStar
 aggCountImplicitWithCast
 SELECT CAST(COUNT(*) AS INT) c FROM "test_emp";
 aggCountImplicitWithConstant
@@ -105,9 +103,7 @@ SELECT gender g, CAST(COUNT(*) AS INT) c FROM "test_emp" WHERE emp_no < 10020 GR
 aggCountWithAlias
 SELECT gender g, COUNT(*) c FROM "test_emp" GROUP BY g ORDER BY gender;
 countDistinct
-// tag::countDistinct
 SELECT COUNT(DISTINCT hire_date) AS count FROM test_emp;
-// end::countDistinct
 
 aggCountAliasAndWhereClauseMultiGroupBy
 SELECT gender g, languages l, COUNT(*) c FROM "test_emp" WHERE emp_no < 10020 GROUP BY gender, languages ORDER BY gender, languages;
@@ -237,9 +233,7 @@ SELECT gender g, languages l, MIN(emp_no) m FROM "test_emp" GROUP BY g, l HAVING
 
 // MAX
 aggMaxImplicit
-// tag::max
 SELECT MAX(salary) AS max FROM test_emp;
-// end::max
 aggMaxImplicitWithCast
 SELECT CAST(MAX(emp_no) AS SMALLINT) c FROM "test_emp";
 aggMax
@@ -310,9 +304,7 @@ SELECT gender g, CAST(AVG(emp_no) AS FLOAT) a FROM "test_emp" GROUP BY gender OR
 aggAvgWithCastToDouble
 SELECT gender g, CAST(AVG(emp_no) AS DOUBLE) a FROM "test_emp" GROUP BY gender ORDER BY gender;
 aggAvg
-// tag::avg
 SELECT AVG(salary) AS avg FROM test_emp;
-// end::avg
 aggAvgWithCastAndCount
 SELECT gender g, CAST(AVG(emp_no) AS FLOAT) a, COUNT(1) c FROM "test_emp" GROUP BY gender ORDER BY gender;
 aggAvgWithCastAndCountWithFilter

x-pack/qa/sql/src/main/resources/docs.csv-spec

@@ -148,7 +148,7 @@ emp            |BASE TABLE
 // end::showTablesLikeMixed
 ;
 
-showTablesLikeEscape
+showTablesLikeEscape-Ignore
 // tag::showTablesLikeEscape
 SHOW TABLES LIKE 'emp!%' ESCAPE '!';
 
@@ -183,88 +183,88 @@ SHOW FUNCTIONS;
 
       name      |     type      
 ----------------+---------------
-AVG             |AGGREGATE      
-COUNT           |AGGREGATE      
-MAX             |AGGREGATE      
-MIN             |AGGREGATE      
-SUM             |AGGREGATE      
-STDDEV_POP      |AGGREGATE      
-VAR_POP         |AGGREGATE      
-PERCENTILE      |AGGREGATE      
-PERCENTILE_RANK |AGGREGATE      
-SUM_OF_SQUARES  |AGGREGATE      
-SKEWNESS        |AGGREGATE      
-KURTOSIS        |AGGREGATE      
-DAY_OF_MONTH    |SCALAR         
-DAY             |SCALAR         
-DOM             |SCALAR         
-DAY_OF_WEEK     |SCALAR         
-DOW             |SCALAR         
-DAY_OF_YEAR     |SCALAR         
-DOY             |SCALAR         
-HOUR_OF_DAY     |SCALAR         
-HOUR            |SCALAR         
-MINUTE_OF_DAY   |SCALAR         
-MINUTE_OF_HOUR  |SCALAR         
-MINUTE          |SCALAR         
-SECOND_OF_MINUTE|SCALAR         
-SECOND          |SCALAR         
-MONTH_OF_YEAR   |SCALAR         
-MONTH           |SCALAR         
-YEAR            |SCALAR         
-WEEK_OF_YEAR    |SCALAR         
-WEEK            |SCALAR         
-ABS             |SCALAR         
-ACOS            |SCALAR         
-ASIN            |SCALAR         
-ATAN            |SCALAR         
-ATAN2           |SCALAR         
-CBRT            |SCALAR         
-CEIL            |SCALAR         
-CEILING         |SCALAR         
-COS             |SCALAR         
-COSH            |SCALAR         
-COT             |SCALAR         
-DEGREES         |SCALAR         
-E               |SCALAR         
-EXP             |SCALAR         
-EXPM1           |SCALAR         
-FLOOR           |SCALAR         
-LOG             |SCALAR         
-LOG10           |SCALAR         
-MOD             |SCALAR         
-PI              |SCALAR         
-POWER           |SCALAR         
-RADIANS         |SCALAR         
-RANDOM          |SCALAR         
-RAND            |SCALAR         
-ROUND           |SCALAR         
-SIGN            |SCALAR         
-SIGNUM          |SCALAR         
-SIN             |SCALAR         
-SINH            |SCALAR         
-SQRT            |SCALAR         
-TAN             |SCALAR         
-ASCII           |SCALAR         
-CHAR            |SCALAR         
-BIT_LENGTH      |SCALAR         
-CHAR_LENGTH     |SCALAR         
-LCASE           |SCALAR         
-LENGTH          |SCALAR         
-LTRIM           |SCALAR         
-RTRIM           |SCALAR         
-SPACE           |SCALAR         
-UCASE           |SCALAR         
-CONCAT          |SCALAR         
-INSERT          |SCALAR         
-LEFT            |SCALAR         
-LOCATE          |SCALAR         
-POSITION        |SCALAR         
-REPEAT          |SCALAR         
-REPLACE         |SCALAR         
-RIGHT           |SCALAR         
-SUBSTRING       |SCALAR         
-SCORE           |SCORE   
+AVG             |AGGREGATE
+COUNT           |AGGREGATE
+MAX             |AGGREGATE
+MIN             |AGGREGATE
+SUM             |AGGREGATE
+STDDEV_POP      |AGGREGATE
+VAR_POP         |AGGREGATE
+PERCENTILE      |AGGREGATE
+PERCENTILE_RANK |AGGREGATE
+SUM_OF_SQUARES  |AGGREGATE
+SKEWNESS        |AGGREGATE
+KURTOSIS        |AGGREGATE
+DAY_OF_MONTH    |SCALAR
+DAY             |SCALAR
+DOM             |SCALAR
+DAY_OF_WEEK     |SCALAR
+DOW             |SCALAR
+DAY_OF_YEAR     |SCALAR
+DOY             |SCALAR
+HOUR_OF_DAY     |SCALAR
+HOUR            |SCALAR
+MINUTE_OF_DAY   |SCALAR
+MINUTE_OF_HOUR  |SCALAR
+MINUTE          |SCALAR
+SECOND_OF_MINUTE|SCALAR
+SECOND          |SCALAR
+MONTH_OF_YEAR   |SCALAR
+MONTH           |SCALAR
+YEAR            |SCALAR
+WEEK_OF_YEAR    |SCALAR
+WEEK            |SCALAR
+ABS             |SCALAR
+ACOS            |SCALAR
+ASIN            |SCALAR
+ATAN            |SCALAR
+ATAN2           |SCALAR
+CBRT            |SCALAR
+CEIL            |SCALAR
+CEILING         |SCALAR
+COS             |SCALAR
+COSH            |SCALAR
+COT             |SCALAR
+DEGREES         |SCALAR
+E               |SCALAR
+EXP             |SCALAR
+EXPM1           |SCALAR
+FLOOR           |SCALAR
+LOG             |SCALAR
+LOG10           |SCALAR
+MOD             |SCALAR
+PI              |SCALAR
+POWER           |SCALAR
+RADIANS         |SCALAR
+RANDOM          |SCALAR
+RAND            |SCALAR
+ROUND           |SCALAR
+SIGN            |SCALAR
+SIGNUM          |SCALAR
+SIN             |SCALAR
+SINH            |SCALAR
+SQRT            |SCALAR
+TAN             |SCALAR
+ASCII           |SCALAR
+CHAR            |SCALAR
+BIT_LENGTH      |SCALAR
+CHAR_LENGTH     |SCALAR
+LCASE           |SCALAR
+LENGTH          |SCALAR
+LTRIM           |SCALAR
+RTRIM           |SCALAR
+SPACE           |SCALAR
+CONCAT          |SCALAR
+INSERT          |SCALAR
+LEFT            |SCALAR
+LOCATE          |SCALAR
+POSITION        |SCALAR
+REPEAT          |SCALAR
+REPLACE         |SCALAR
+RIGHT           |SCALAR
+SUBSTRING       |SCALAR
+UCASE           |SCALAR
+SCORE           |SCORE
 
 // end::showFunctions
 ;
@@ -331,7 +331,7 @@ MINUTE_OF_DAY  |SCALAR
 
 selectColumnAlias
 // tag::selectColumnAlias
-SELECT 1 + 1 AS result
+SELECT 1 + 1 AS result;
 
     result     
 ---------------
@@ -503,18 +503,19 @@ groupByMulti
 // tag::groupByMulti
 SELECT gender g, languages l, COUNT(*) c FROM "emp" GROUP BY g, l ORDER BY languages ASC, gender DESC;
 
-       g       |       l       |       c       
+       g       |       l       |       c
 ---------------+---------------+---------------
-F              |2              |4              
-F              |3              |8              
-F              |4              |7              
-F              |5              |7              
-F              |6              |11             
-M              |2              |12             
-M              |3              |12             
-M              |4              |15             
-M              |5              |11             
-M              |6              |13       
+M              |1              |12
+F              |1              |4
+M              |2              |12
+F              |2              |8
+M              |3              |15
+F              |3              |7
+M              |4              |11
+F              |4              |7
+M              |5              |13
+F              |5              |11 
+
 
 // end::groupByMulti
 ;
@@ -658,7 +659,7 @@ James S.A. Corey |Leviathan Wakes     |561            |2011-06-02T00:00:00Z
 
 orderByScore
 // tag::orderByScore
-SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY SCORE() DESC;
+SELECT SCORE(), * FROM library WHERE MATCH(name, 'dune') ORDER BY SCORE() DESC;
 
     SCORE()    |    author     |       name        |  page_count   |    release_date    
 ---------------+---------------+-------------------+---------------+--------------------
@@ -672,7 +673,7 @@ SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY SCORE() DESC;
 
 orderByScoreWithMatch
 // tag::orderByScoreWithMatch
-SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY page_count DESC;
+SELECT SCORE(), * FROM library WHERE MATCH(name, 'dune') ORDER BY page_count DESC;
 
     SCORE()    |    author     |       name        |  page_count   |    release_date    
 ---------------+---------------+-------------------+---------------+--------------------
@@ -684,6 +685,19 @@ SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY page_count DES
 // end::orderByScoreWithMatch
 ;
 
+scoreWithMatch
+// tag::scoreWithMatch
+SELECT SCORE() AS score, name, release_date FROM library WHERE QUERY('dune') ORDER BY YEAR(release_date) DESC;
+
+     score     |       name        |    release_date    
+---------------+-------------------+--------------------
+1.4005898      |God Emperor of Dune|1981-05-28T00:00:00Z
+1.6086555      |Children of Dune   |1976-04-21T00:00:00Z
+1.8893257      |Dune Messiah       |1969-10-15T00:00:00Z
+2.288635       |Dune               |1965-06-01T00:00:00Z
+// end::scoreWithMatch
+;
+
 
 ///////////////////////////////
 //
@@ -701,3 +715,399 @@ Georgi         |Facello        |10001
 
 // end::limitBasic
 ;
+
+///////////////////////////////
+//
+// Aggregations
+//
+///////////////////////////////
+
+aggAvg
+// tag::aggAvg
+SELECT AVG(salary) AS avg FROM emp;
+
+      avg      
+---------------
+48248          
+// end::aggAvg
+;
+
+aggCountStar
+// tag::aggCountStar
+SELECT COUNT(*) AS count FROM emp;
+
+     count     
+---------------
+100               
+// end::aggCountStar
+;
+
+aggCountDistinct
+// tag::aggCountDistinct
+
+SELECT COUNT(DISTINCT hire_date) AS hires FROM emp;
+
+     hires      
+---------------
+99       
+// end::aggCountDistinct
+;
+
+aggMax
+// tag::aggMax
+SELECT MAX(salary) AS max FROM emp;
+
+      max     
+---------------
+74999               
+// end::aggMax
+;
+
+aggMin
+// tag::aggMin
+SELECT MIN(salary) AS min FROM emp;
+
+      min     
+---------------
+25324               
+// end::aggMin
+;
+
+aggSum
+// tag::aggSum
+SELECT SUM(salary) AS sum FROM emp;
+
+      sum
+---------------
+4824855
+// end::aggSum
+;
+
+aggKurtosis
+// tag::aggKurtosis
+SELECT MIN(salary) AS min, MAX(salary) AS max, KURTOSIS(salary) AS k FROM emp;
+
+      min      |      max      |        k         
+---------------+---------------+------------------
+25324          |74999          |2.0444718929142986
+// end::aggKurtosis
+;
+
+aggPercentile
+// tag::aggPercentile
+SELECT languages, PERCENTILE(salary, 95) AS "95th" FROM emp 
+       GROUP BY languages;
+
+   languages   |      95th       
+---------------+-----------------
+1              |72605.2          
+2              |71741.0          
+3              |74981.6          
+4              |72115.59999999999
+5              |68927.19999999998
+// end::aggPercentile
+;
+
+aggPercentileRank
+// tag::aggPercentileRank
+SELECT languages, PERCENTILE_RANK(salary, 65000) AS rank FROM emp GROUP BY languages;
+
+   languages   |      rank       
+---------------+-----------------
+1              |75.37108985853756
+2              |89.43605326660112
+3              |77.74873333978765
+4              |85.70446389643493
+5              |92.52677973666592
+// end::aggPercentileRank
+;
+
+aggSkewness
+// tag::aggSkewness
+SELECT MIN(salary) AS min, MAX(salary) AS max, SKEWNESS(salary) AS s FROM emp;
+
+      min      |      max      |        s         
+---------------+---------------+------------------
+25324          |74999          |0.2707722118423227
+// end::aggSkewness
+;
+
+aggStddevPop
+// tag::aggStddevPop
+SELECT MIN(salary) AS min, MAX(salary) AS max, STDDEV_POP(salary) AS stddev 
+       FROM emp;
+
+      min      |      max      |      stddev      
+---------------+---------------+------------------
+25324          |74999          |13765.125502787832
+// end::aggStddevPop
+;
+
+
+aggSumOfSquares
+// tag::aggSumOfSquares
+SELECT MIN(salary) AS min, MAX(salary) AS max, SUM_OF_SQUARES(salary) AS sumsq 
+       FROM emp;
+
+      min      |      max      |     sumsq      
+---------------+---------------+----------------
+25324          |74999          |2.51740125721E11
+// end::aggSumOfSquares
+;
+
+
+aggVarPop
+// tag::aggVarPop
+SELECT MIN(salary) AS min, MAX(salary) AS max, VAR_POP(salary) AS varpop FROM emp;
+
+      min      |      max      |     varpop     
+---------------+---------------+----------------
+25324          |74999          |1.894786801075E8
+// end::aggVarPop
+;
+
+
+///////////////////////////////
+//
+// String
+//
+///////////////////////////////
+
+stringAscii
+// tag::stringAscii
+SELECT ASCII('Elastic');
+
+ASCII(Elastic) 
+---------------
+69   
+// end::stringAscii
+;
+
+stringBitLength
+// tag::stringBitLength
+SELECT BIT_LENGTH('Elastic');
+
+BIT_LENGTH(Elastic)
+-------------------
+56  
+// end::stringBitLength
+;
+
+stringChar
+// tag::stringChar
+SELECT CHAR(69);
+
+   CHAR(69)    
+---------------
+E        
+// end::stringChar
+;
+
+stringCharLength
+// tag::stringCharLength
+SELECT CHAR_LENGTH('Elastic');
+
+CHAR_LENGTH(Elastic)
+--------------------
+7     
+// end::stringCharLength
+;
+
+stringConcat
+// tag::stringConcat
+SELECT CONCAT('Elasticsearch', ' SQL');
+
+CONCAT(Elasticsearch, SQL)
+--------------------------
+Elasticsearch SQL  
+// end::stringConcat
+;
+
+stringInsert
+// tag::stringInsert
+SELECT INSERT('Elastic ', 8, 1, 'search');
+
+INSERT(Elastic ,8,1,search)
+---------------------------
+Elasticsearch
+// end::stringInsert
+;
+
+stringLCase
+// tag::stringLCase
+SELECT LCASE('Elastic');
+
+LCASE(Elastic) 
+---------------
+elastic    
+// end::stringLCase
+;
+
+stringLeft
+// tag::stringLeft
+SELECT LEFT('Elastic',3);
+
+LEFT(Elastic,3)
+---------------
+Ela    
+// end::stringLeft
+;
+
+stringLength
+// tag::stringLength
+SELECT LENGTH('Elastic   ');
+
+LENGTH(Elastic   )
+------------------
+7     
+// end::stringLength
+;
+
+stringLocateWoStart
+// tag::stringLocateWoStart
+SELECT LOCATE('a', 'Elasticsearch');
+
+LOCATE(a,Elasticsearch)
+-----------------------
+3        
+// end::stringLocateWoStart
+;
+
+stringLocateWithStart
+// tag::stringLocateWithStart
+SELECT LOCATE('a', 'Elasticsearch', 5);
+
+LOCATE(a,Elasticsearch,5)
+-------------------------
+10
+// end::stringLocateWithStart
+;
+
+stringLTrim
+// tag::stringLTrim
+SELECT LTRIM('   Elastic');
+
+LTRIM(   Elastic)
+-----------------
+Elastic   
+// end::stringLTrim
+;
+
+stringPosition
+// tag::stringPosition
+SELECT POSITION('Elastic', 'Elasticsearch');
+
+POSITION(Elastic,Elasticsearch)
+-------------------------------
+1  
+// end::stringPosition
+;
+
+stringRepeat
+// tag::stringRepeat
+SELECT REPEAT('La', 3);
+
+ REPEAT(La,3)  
+---------------
+LaLaLa      
+// end::stringRepeat
+;
+
+stringReplace-Ignore
+// tag::stringReplace
+SELECT REPLACE('Elastic', 'El', 'Fant');
+
+  REPLACE(Elastic, El, Fant)
+-----------------------------
+Fantastic
+// end::stringReplace
+;
+
+stringRight
+// tag::stringRight
+SELECT RIGHT('Elastic',3);
+
+RIGHT(Elastic,3)
+----------------
+tic    
+// end::stringRight
+;
+
+stringRTrim
+// tag::stringRTrim
+SELECT RTRIM('Elastic   ');
+
+RTRIM(Elastic   )
+-----------------
+Elastic       
+// end::stringRTrim
+;
+
+stringSpace-Ignore
+// tag::stringSpace
+SELECT SPACE(3);
+
+   SPACE(3)    
+---------------
+               
+ 
+// end::stringSpace
+;
+
+stringSubString
+// tag::stringSubString
+SELECT SUBSTRING('Elasticsearch', 0, 7);
+
+SUBSTRING(Elasticsearch,0,7)
+----------------------------
+Elastic    
+// end::stringSubString
+;
+
+stringUCase
+// tag::stringUCase
+SELECT UCASE('Elastic');
+
+UCASE(Elastic) 
+---------------
+ELASTIC    
+// end::stringUCase
+;
+
+
+///////////////////////////////
+//
+// Cast
+//
+///////////////////////////////
+
+conversionStringToIntCast
+// tag::conversionStringToIntCast
+SELECT CAST('123' AS INT) AS int;
+
+      int      
+---------------
+123    
+// end::conversionStringToIntCast
+;
+
+conversionIntToStringCast-Ignore
+// tag::conversionIntToStringCast
+SELECT CAST(123 AS VARCHAR) AS string;
+
+    string     
+---------------
+123   
+
+// end::conversionIntToStringCast
+;
+
+conversionStringToDateCast
+// tag::conversionStringToDateCast
+SELECT YEAR(CAST('2018-05-19T11:23:45Z' AS TIMESTAMP)) AS year;
+
+     year
+---------------
+2018
+// end::conversionStringToDateCast
+;