NumberFieldMapper.java 92 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317
  1. /*
  2. * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  3. * or more contributor license agreements. Licensed under the "Elastic License
  4. * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
  5. * Public License v 1"; you may not use this file except in compliance with, at
  6. * your election, the "Elastic License 2.0", the "GNU Affero General Public
  7. * License v3.0 only", or the "Server Side Public License, v 1".
  8. */
  9. package org.elasticsearch.index.mapper;
  10. import org.apache.lucene.document.DoubleField;
  11. import org.apache.lucene.document.DoublePoint;
  12. import org.apache.lucene.document.Field;
  13. import org.apache.lucene.document.FloatField;
  14. import org.apache.lucene.document.FloatPoint;
  15. import org.apache.lucene.document.IntField;
  16. import org.apache.lucene.document.IntPoint;
  17. import org.apache.lucene.document.LongField;
  18. import org.apache.lucene.document.LongPoint;
  19. import org.apache.lucene.document.SortedNumericDocValuesField;
  20. import org.apache.lucene.document.StoredField;
  21. import org.apache.lucene.index.IndexableField;
  22. import org.apache.lucene.index.LeafReaderContext;
  23. import org.apache.lucene.sandbox.document.HalfFloatPoint;
  24. import org.apache.lucene.search.IndexOrDocValuesQuery;
  25. import org.apache.lucene.search.MatchNoDocsQuery;
  26. import org.apache.lucene.search.Query;
  27. import org.apache.lucene.util.BytesRef;
  28. import org.apache.lucene.util.NumericUtils;
  29. import org.elasticsearch.common.Explicit;
  30. import org.elasticsearch.common.Numbers;
  31. import org.elasticsearch.common.lucene.search.Queries;
  32. import org.elasticsearch.common.settings.Setting;
  33. import org.elasticsearch.common.settings.Setting.Property;
  34. import org.elasticsearch.common.settings.Settings;
  35. import org.elasticsearch.index.IndexMode;
  36. import org.elasticsearch.index.IndexVersion;
  37. import org.elasticsearch.index.IndexVersions;
  38. import org.elasticsearch.index.fielddata.FieldDataContext;
  39. import org.elasticsearch.index.fielddata.IndexFieldData;
  40. import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
  41. import org.elasticsearch.index.fielddata.SourceValueFetcherSortedDoubleIndexFieldData;
  42. import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData;
  43. import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData;
  44. import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
  45. import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType;
  46. import org.elasticsearch.index.query.SearchExecutionContext;
  47. import org.elasticsearch.lucene.document.NumericField;
  48. import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
  49. import org.elasticsearch.script.DoubleFieldScript;
  50. import org.elasticsearch.script.LongFieldScript;
  51. import org.elasticsearch.script.Script;
  52. import org.elasticsearch.script.ScriptCompiler;
  53. import org.elasticsearch.script.field.ByteDocValuesField;
  54. import org.elasticsearch.script.field.DoubleDocValuesField;
  55. import org.elasticsearch.script.field.FloatDocValuesField;
  56. import org.elasticsearch.script.field.HalfFloatDocValuesField;
  57. import org.elasticsearch.script.field.IntegerDocValuesField;
  58. import org.elasticsearch.script.field.LongDocValuesField;
  59. import org.elasticsearch.script.field.ShortDocValuesField;
  60. import org.elasticsearch.search.DocValueFormat;
  61. import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
  62. import org.elasticsearch.search.aggregations.support.ValuesSourceType;
  63. import org.elasticsearch.search.lookup.FieldValues;
  64. import org.elasticsearch.search.lookup.SearchLookup;
  65. import org.elasticsearch.search.lookup.SourceProvider;
  66. import org.elasticsearch.xcontent.XContentBuilder;
  67. import org.elasticsearch.xcontent.XContentParser;
  68. import org.elasticsearch.xcontent.XContentParser.Token;
  69. import java.io.IOException;
  70. import java.math.BigDecimal;
  71. import java.time.ZoneId;
  72. import java.util.ArrayList;
  73. import java.util.Arrays;
  74. import java.util.Collection;
  75. import java.util.Collections;
  76. import java.util.List;
  77. import java.util.Map;
  78. import java.util.Objects;
  79. import java.util.Set;
  80. import java.util.function.BiFunction;
  81. import java.util.function.Function;
  82. import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
  83. /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */
  84. public class NumberFieldMapper extends FieldMapper {
  85. public static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", true, Property.IndexScope);
  86. private static NumberFieldMapper toType(FieldMapper in) {
  87. return (NumberFieldMapper) in;
  88. }
  89. public static final class Builder extends FieldMapper.DimensionBuilder {
  90. private final Parameter<Boolean> indexed;
  91. private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
  92. private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
  93. private final Parameter<Explicit<Boolean>> ignoreMalformed;
  94. private final Parameter<Explicit<Boolean>> coerce;
  95. private final Parameter<Number> nullValue;
  96. private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
  97. private final Parameter<OnScriptError> onScriptErrorParam = Parameter.onScriptErrorParam(
  98. m -> toType(m).builderParams.onScriptError(),
  99. script
  100. );
  101. /**
  102. * Parameter that marks this field as a time series dimension.
  103. */
  104. private final Parameter<Boolean> dimension;
  105. /**
  106. * Parameter that marks this field as a time series metric defining its time series metric type.
  107. * For the numeric fields gauge and counter metric types are
  108. * supported
  109. */
  110. private final Parameter<MetricType> metric;
  111. private final Parameter<Map<String, String>> meta = Parameter.metaParam();
  112. private final ScriptCompiler scriptCompiler;
  113. private final NumberType type;
  114. private boolean allowMultipleValues = true;
  115. private final IndexVersion indexCreatedVersion;
  116. private final IndexMode indexMode;
  117. private final SourceKeepMode indexSourceKeepMode;
  118. public Builder(
  119. String name,
  120. NumberType type,
  121. ScriptCompiler compiler,
  122. Settings settings,
  123. IndexVersion indexCreatedVersion,
  124. IndexMode mode,
  125. SourceKeepMode indexSourceKeepMode
  126. ) {
  127. this(
  128. name,
  129. type,
  130. compiler,
  131. IGNORE_MALFORMED_SETTING.get(settings),
  132. COERCE_SETTING.get(settings),
  133. indexCreatedVersion,
  134. mode,
  135. indexSourceKeepMode
  136. );
  137. }
  138. public static Builder docValuesOnly(String name, NumberType type, IndexVersion indexCreatedVersion) {
  139. Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null, null);
  140. builder.indexed.setValue(false);
  141. builder.dimension.setValue(false);
  142. return builder;
  143. }
  144. public Builder(
  145. String name,
  146. NumberType type,
  147. ScriptCompiler compiler,
  148. boolean ignoreMalformedByDefault,
  149. boolean coerceByDefault,
  150. IndexVersion indexCreatedVersion,
  151. IndexMode mode,
  152. SourceKeepMode indexSourceKeepMode
  153. ) {
  154. super(name);
  155. this.type = type;
  156. this.scriptCompiler = Objects.requireNonNull(compiler);
  157. this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
  158. this.ignoreMalformed = Parameter.explicitBoolParam(
  159. "ignore_malformed",
  160. true,
  161. m -> toType(m).ignoreMalformed,
  162. ignoreMalformedByDefault
  163. );
  164. this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
  165. this.nullValue = new Parameter<>(
  166. "null_value",
  167. false,
  168. () -> null,
  169. (n, c, o) -> o == null ? null : type.parse(o, false),
  170. m -> toType(m).nullValue,
  171. XContentBuilder::field,
  172. Objects::toString
  173. ).acceptsNull();
  174. this.indexMode = mode;
  175. this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
  176. if (indexMode == IndexMode.TIME_SERIES) {
  177. var metricType = getMetric().getValue();
  178. return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE;
  179. } else {
  180. return true;
  181. }
  182. });
  183. this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> {
  184. if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) {
  185. throw new IllegalArgumentException(
  186. "Field ["
  187. + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM
  188. + "] requires that ["
  189. + indexed.name
  190. + "] and ["
  191. + hasDocValues.name
  192. + "] are true"
  193. );
  194. }
  195. });
  196. this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> {
  197. if (v != null && hasDocValues.getValue() == false) {
  198. throw new IllegalArgumentException(
  199. "Field [" + TimeSeriesParams.TIME_SERIES_METRIC_PARAM + "] requires that [" + hasDocValues.name + "] is true"
  200. );
  201. }
  202. }).precludesParameters(dimension);
  203. this.script.precludesParameters(ignoreMalformed, coerce, nullValue);
  204. addScriptValidation(script, indexed, hasDocValues);
  205. this.indexSourceKeepMode = indexSourceKeepMode;
  206. }
  207. Builder nullValue(Number number) {
  208. this.nullValue.setValue(number);
  209. return this;
  210. }
  211. public Builder docValues(boolean hasDocValues) {
  212. this.hasDocValues.setValue(hasDocValues);
  213. return this;
  214. }
  215. private FieldValues<Number> scriptValues() {
  216. if (this.script.get() == null) {
  217. return null;
  218. }
  219. return type.compile(leafName(), script.get(), scriptCompiler);
  220. }
  221. public Builder dimension(boolean dimension) {
  222. this.dimension.setValue(dimension);
  223. return this;
  224. }
  225. public Builder metric(MetricType metric) {
  226. this.metric.setValue(metric);
  227. return this;
  228. }
  229. private Parameter<MetricType> getMetric() {
  230. return metric;
  231. }
  232. public Builder allowMultipleValues(boolean allowMultipleValues) {
  233. this.allowMultipleValues = allowMultipleValues;
  234. return this;
  235. }
  236. @Override
  237. protected Parameter<?>[] getParameters() {
  238. return new Parameter<?>[] {
  239. indexed,
  240. hasDocValues,
  241. stored,
  242. ignoreMalformed,
  243. coerce,
  244. nullValue,
  245. script,
  246. onScriptErrorParam,
  247. meta,
  248. dimension,
  249. metric };
  250. }
  251. @Override
  252. public NumberFieldMapper build(MapperBuilderContext context) {
  253. if (inheritDimensionParameterFromParentObject(context)) {
  254. dimension.setValue(true);
  255. }
  256. MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic());
  257. hasScript = script.get() != null;
  258. onScriptError = onScriptErrorParam.getValue();
  259. String offsetsFieldName = getOffsetsFieldName(
  260. context,
  261. indexSourceKeepMode,
  262. hasDocValues.getValue(),
  263. stored.getValue(),
  264. this,
  265. indexCreatedVersion,
  266. IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER
  267. );
  268. return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this, offsetsFieldName);
  269. }
  270. }
  271. public enum NumberType {
  272. HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
  273. @Override
  274. public Float parse(Object value, boolean coerce) {
  275. final float result = parseToFloat(value);
  276. validateFiniteValue(result);
  277. // Reduce the precision to what we actually index
  278. return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(result));
  279. }
  280. @Override
  281. public double reduceToStoredPrecision(double value) {
  282. return parse(value, false).doubleValue();
  283. }
  284. /**
  285. * Parse a query parameter or {@code _source} value to a float,
  286. * keeping float precision. Used by queries which do need to validate
  287. * against infinite values, but need more precise control over their
  288. * rounding behavior that {@link #parse(Object, boolean)} provides.
  289. */
  290. private static float parseToFloat(Object value) {
  291. final float result;
  292. if (value instanceof Number) {
  293. result = ((Number) value).floatValue();
  294. } else {
  295. if (value instanceof BytesRef) {
  296. value = ((BytesRef) value).utf8ToString();
  297. }
  298. result = Float.parseFloat(value.toString());
  299. }
  300. return result;
  301. }
  302. @Override
  303. public Number parsePoint(byte[] value) {
  304. return HalfFloatPoint.decodeDimension(value, 0);
  305. }
  306. @Override
  307. public Float parse(XContentParser parser, boolean coerce) throws IOException {
  308. float parsed = parser.floatValue(coerce);
  309. validateFiniteValue(parsed);
  310. return parsed;
  311. }
  312. @Override
  313. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  314. float v = parseToFloat(value);
  315. if (Float.isFinite(HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(v))) == false) {
  316. return Queries.newMatchNoDocsQuery("Value [" + value + "] is out of range");
  317. }
  318. if (isIndexed) {
  319. if (hasDocValues) {
  320. return new IndexOrDocValuesQuery(
  321. HalfFloatPoint.newExactQuery(field, v),
  322. SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v))
  323. );
  324. }
  325. return HalfFloatPoint.newExactQuery(field, v);
  326. } else {
  327. return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v));
  328. }
  329. }
  330. @Override
  331. public Query termsQuery(String field, Collection<?> values) {
  332. float[] v = new float[values.size()];
  333. int pos = 0;
  334. for (Object value : values) {
  335. float float_value = parseToFloat(value);
  336. validateFiniteValue(float_value);
  337. v[pos++] = float_value;
  338. }
  339. return HalfFloatPoint.newSetQuery(field, v);
  340. }
  341. @Override
  342. public Query rangeQuery(
  343. String field,
  344. Object lowerTerm,
  345. Object upperTerm,
  346. boolean includeLower,
  347. boolean includeUpper,
  348. boolean hasDocValues,
  349. SearchExecutionContext context,
  350. boolean isIndexed
  351. ) {
  352. float l = Float.NEGATIVE_INFINITY;
  353. float u = Float.POSITIVE_INFINITY;
  354. if (lowerTerm != null) {
  355. l = parseToFloat(lowerTerm);
  356. if (includeLower) {
  357. l = HalfFloatPoint.nextDown(l);
  358. }
  359. l = HalfFloatPoint.nextUp(l);
  360. }
  361. if (upperTerm != null) {
  362. u = parseToFloat(upperTerm);
  363. if (includeUpper) {
  364. u = HalfFloatPoint.nextUp(u);
  365. }
  366. u = HalfFloatPoint.nextDown(u);
  367. }
  368. Query query;
  369. if (isIndexed) {
  370. query = HalfFloatPoint.newRangeQuery(field, l, u);
  371. if (hasDocValues) {
  372. Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
  373. field,
  374. HalfFloatPoint.halfFloatToSortableShort(l),
  375. HalfFloatPoint.halfFloatToSortableShort(u)
  376. );
  377. query = new IndexOrDocValuesQuery(query, dvQuery);
  378. }
  379. } else {
  380. query = SortedNumericDocValuesField.newSlowRangeQuery(
  381. field,
  382. HalfFloatPoint.halfFloatToSortableShort(l),
  383. HalfFloatPoint.halfFloatToSortableShort(u)
  384. );
  385. }
  386. return query;
  387. }
  388. @Override
  389. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  390. final float f = value.floatValue();
  391. if (indexed) {
  392. document.add(new HalfFloatPoint(name, f));
  393. }
  394. if (docValued) {
  395. document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f)));
  396. }
  397. if (stored) {
  398. document.add(new StoredField(name, f));
  399. }
  400. }
  401. @Override
  402. public long toSortableLong(Number value) {
  403. return HalfFloatPoint.halfFloatToSortableShort(value.floatValue());
  404. }
  405. @Override
  406. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  407. return new SortedDoublesIndexFieldData.Builder(
  408. ft.name(),
  409. numericType(),
  410. valuesSourceType,
  411. HalfFloatDocValuesField::new,
  412. ft.isIndexed()
  413. );
  414. }
  415. @Override
  416. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  417. String name,
  418. ValuesSourceType valuesSourceType,
  419. SourceProvider sourceProvider,
  420. ValueFetcher valueFetcher
  421. ) {
  422. return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
  423. name,
  424. valuesSourceType,
  425. valueFetcher,
  426. sourceProvider,
  427. HalfFloatDocValuesField::new
  428. );
  429. }
  430. private static void validateFiniteValue(float value) {
  431. if (Float.isFinite(HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value))) == false) {
  432. throw new IllegalArgumentException("[half_float] supports only finite values, but got [" + value + "]");
  433. }
  434. }
  435. @Override
  436. public void writeValue(XContentBuilder b, long value) throws IOException {
  437. b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
  438. }
  439. @Override
  440. BlockLoader blockLoaderFromDocValues(String fieldName) {
  441. return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> HalfFloatPoint.sortableShortToHalfFloat((short) l));
  442. }
  443. @Override
  444. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  445. return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
  446. }
  447. @Override
  448. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  449. return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  450. }
  451. },
  452. FLOAT("float", NumericType.FLOAT) {
  453. @Override
  454. public Float parse(Object value, boolean coerce) {
  455. final float result = parseToFloat(value);
  456. validateFiniteValue(result);
  457. return result;
  458. }
  459. /**
  460. * Parse a query parameter or {@code _source} value to a float,
  461. * keeping float precision. Used by queries which do need validate
  462. * against infinite values like {@link #parse(Object, boolean)} does.
  463. */
  464. private static float parseToFloat(Object value) {
  465. final float result;
  466. if (value instanceof Number) {
  467. result = ((Number) value).floatValue();
  468. } else {
  469. if (value instanceof BytesRef) {
  470. value = ((BytesRef) value).utf8ToString();
  471. }
  472. result = Float.parseFloat(value.toString());
  473. }
  474. return result;
  475. }
  476. @Override
  477. public double reduceToStoredPrecision(double value) {
  478. return parse(value, false).doubleValue();
  479. }
  480. @Override
  481. public Number parsePoint(byte[] value) {
  482. return FloatPoint.decodeDimension(value, 0);
  483. }
  484. @Override
  485. public Float parse(XContentParser parser, boolean coerce) throws IOException {
  486. float parsed = parser.floatValue(coerce);
  487. validateFiniteValue(parsed);
  488. return parsed;
  489. }
  490. @Override
  491. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  492. float v = parseToFloat(value);
  493. if (Float.isFinite(v) == false) {
  494. return new MatchNoDocsQuery("Value [" + value + "] is out of range");
  495. }
  496. if (isIndexed && hasDocValues) {
  497. return FloatField.newExactQuery(field, v);
  498. } else if (isIndexed) {
  499. return FloatPoint.newExactQuery(field, v);
  500. } else {
  501. return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v));
  502. }
  503. }
  504. @Override
  505. public Query termsQuery(String field, Collection<?> values) {
  506. float[] v = new float[values.size()];
  507. int pos = 0;
  508. for (Object value : values) {
  509. v[pos++] = parse(value, false);
  510. }
  511. return FloatPoint.newSetQuery(field, v);
  512. }
  513. @Override
  514. public Query rangeQuery(
  515. String field,
  516. Object lowerTerm,
  517. Object upperTerm,
  518. boolean includeLower,
  519. boolean includeUpper,
  520. boolean hasDocValues,
  521. SearchExecutionContext context,
  522. boolean isIndexed
  523. ) {
  524. float l = Float.NEGATIVE_INFINITY;
  525. float u = Float.POSITIVE_INFINITY;
  526. if (lowerTerm != null) {
  527. l = parseToFloat(lowerTerm);
  528. if (includeLower) {
  529. l = FloatPoint.nextDown(l);
  530. }
  531. l = FloatPoint.nextUp(l);
  532. }
  533. if (upperTerm != null) {
  534. u = parseToFloat(upperTerm);
  535. if (includeUpper) {
  536. u = FloatPoint.nextUp(u);
  537. }
  538. u = FloatPoint.nextDown(u);
  539. }
  540. Query query;
  541. if (isIndexed) {
  542. query = FloatPoint.newRangeQuery(field, l, u);
  543. if (hasDocValues) {
  544. Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
  545. field,
  546. NumericUtils.floatToSortableInt(l),
  547. NumericUtils.floatToSortableInt(u)
  548. );
  549. query = new IndexOrDocValuesQuery(query, dvQuery);
  550. }
  551. } else {
  552. query = SortedNumericDocValuesField.newSlowRangeQuery(
  553. field,
  554. NumericUtils.floatToSortableInt(l),
  555. NumericUtils.floatToSortableInt(u)
  556. );
  557. }
  558. return query;
  559. }
  560. @Override
  561. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  562. final float f = value.floatValue();
  563. if (indexed && docValued) {
  564. document.add(new FloatField(name, f, Field.Store.NO));
  565. } else if (docValued) {
  566. document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f)));
  567. } else if (indexed) {
  568. document.add(new FloatPoint(name, f));
  569. }
  570. if (stored) {
  571. document.add(new StoredField(name, f));
  572. }
  573. }
  574. @Override
  575. public long toSortableLong(Number value) {
  576. return NumericUtils.floatToSortableInt(value.floatValue());
  577. }
  578. @Override
  579. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  580. return new SortedDoublesIndexFieldData.Builder(
  581. ft.name(),
  582. numericType(),
  583. valuesSourceType,
  584. FloatDocValuesField::new,
  585. ft.isIndexed()
  586. );
  587. }
  588. @Override
  589. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  590. String name,
  591. ValuesSourceType valuesSourceType,
  592. SourceProvider sourceProvider,
  593. ValueFetcher valueFetcher
  594. ) {
  595. return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
  596. name,
  597. valuesSourceType,
  598. valueFetcher,
  599. sourceProvider,
  600. FloatDocValuesField::new
  601. );
  602. }
  603. private static void validateFiniteValue(float value) {
  604. if (Float.isFinite(value) == false) {
  605. throw new IllegalArgumentException("[float] supports only finite values, but got [" + value + "]");
  606. }
  607. }
  608. @Override
  609. public void writeValue(XContentBuilder b, long value) throws IOException {
  610. b.value(NumericUtils.sortableIntToFloat((int) value));
  611. }
  612. @Override
  613. BlockLoader blockLoaderFromDocValues(String fieldName) {
  614. return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> NumericUtils.sortableIntToFloat((int) l));
  615. }
  616. @Override
  617. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  618. return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
  619. }
  620. @Override
  621. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  622. return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  623. }
  624. },
  625. DOUBLE("double", NumericType.DOUBLE) {
  626. @Override
  627. public Double parse(Object value, boolean coerce) {
  628. double parsed = objectToDouble(value);
  629. validateParsed(parsed);
  630. return parsed;
  631. }
  632. @Override
  633. public Number parsePoint(byte[] value) {
  634. return DoublePoint.decodeDimension(value, 0);
  635. }
  636. @Override
  637. public Double parse(XContentParser parser, boolean coerce) throws IOException {
  638. double parsed = parser.doubleValue(coerce);
  639. validateParsed(parsed);
  640. return parsed;
  641. }
  642. @Override
  643. public FieldValues<Number> compile(String fieldName, Script script, ScriptCompiler compiler) {
  644. DoubleFieldScript.Factory scriptFactory = compiler.compile(script, DoubleFieldScript.CONTEXT);
  645. return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL)
  646. .newInstance(ctx)
  647. .runForDoc(doc, consumer::accept);
  648. }
  649. @Override
  650. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  651. double v = objectToDouble(value);
  652. if (Double.isFinite(v) == false) {
  653. return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
  654. }
  655. if (isIndexed && hasDocValues) {
  656. return DoubleField.newExactQuery(field, v);
  657. } else if (isIndexed) {
  658. return DoublePoint.newExactQuery(field, v);
  659. } else {
  660. return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
  661. }
  662. }
  663. @Override
  664. public Query termsQuery(String field, Collection<?> values) {
  665. double[] v = values.stream().mapToDouble(value -> parse(value, false)).toArray();
  666. return DoublePoint.newSetQuery(field, v);
  667. }
  668. @Override
  669. public Query rangeQuery(
  670. String field,
  671. Object lowerTerm,
  672. Object upperTerm,
  673. boolean includeLower,
  674. boolean includeUpper,
  675. boolean hasDocValues,
  676. SearchExecutionContext context,
  677. boolean isIndexed
  678. ) {
  679. return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
  680. Query query;
  681. if (isIndexed) {
  682. query = DoublePoint.newRangeQuery(field, l, u);
  683. if (hasDocValues) {
  684. Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
  685. field,
  686. NumericUtils.doubleToSortableLong(l),
  687. NumericUtils.doubleToSortableLong(u)
  688. );
  689. query = new IndexOrDocValuesQuery(query, dvQuery);
  690. }
  691. } else {
  692. query = SortedNumericDocValuesField.newSlowRangeQuery(
  693. field,
  694. NumericUtils.doubleToSortableLong(l),
  695. NumericUtils.doubleToSortableLong(u)
  696. );
  697. }
  698. return query;
  699. });
  700. }
  701. @Override
  702. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  703. final double d = value.doubleValue();
  704. if (indexed && docValued) {
  705. document.add(new DoubleField(name, d, Field.Store.NO));
  706. } else if (docValued) {
  707. document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d)));
  708. } else if (indexed) {
  709. document.add(new DoublePoint(name, d));
  710. }
  711. if (stored) {
  712. document.add(new StoredField(name, d));
  713. }
  714. }
  715. @Override
  716. public long toSortableLong(Number value) {
  717. return NumericUtils.doubleToSortableLong(value.doubleValue());
  718. }
  719. @Override
  720. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  721. return new SortedDoublesIndexFieldData.Builder(
  722. ft.name(),
  723. numericType(),
  724. valuesSourceType,
  725. DoubleDocValuesField::new,
  726. ft.isIndexed()
  727. );
  728. }
  729. @Override
  730. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  731. String name,
  732. ValuesSourceType valuesSourceType,
  733. SourceProvider sourceProvider,
  734. ValueFetcher valueFetcher
  735. ) {
  736. return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
  737. name,
  738. valuesSourceType,
  739. valueFetcher,
  740. sourceProvider,
  741. DoubleDocValuesField::new
  742. );
  743. }
  744. private static void validateParsed(double value) {
  745. if (Double.isFinite(value) == false) {
  746. throw new IllegalArgumentException("[double] supports only finite values, but got [" + value + "]");
  747. }
  748. }
  749. @Override
  750. public void writeValue(XContentBuilder b, long value) throws IOException {
  751. b.value(NumericUtils.sortableLongToDouble(value));
  752. }
  753. @Override
  754. BlockLoader blockLoaderFromDocValues(String fieldName) {
  755. return new BlockDocValuesReader.DoublesBlockLoader(fieldName, NumericUtils::sortableLongToDouble);
  756. }
  757. @Override
  758. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  759. return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
  760. }
  761. @Override
  762. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  763. return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  764. }
  765. },
  766. BYTE("byte", NumericType.BYTE) {
  767. @Override
  768. public Byte parse(Object value, boolean coerce) {
  769. double doubleValue = objectToDouble(value);
  770. if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) {
  771. throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
  772. }
  773. if (coerce == false && doubleValue % 1 != 0) {
  774. throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
  775. }
  776. if (value instanceof Number) {
  777. return ((Number) value).byteValue();
  778. }
  779. return (byte) doubleValue;
  780. }
  781. @Override
  782. public Number parsePoint(byte[] value) {
  783. return INTEGER.parsePoint(value).byteValue();
  784. }
  785. @Override
  786. public Byte parse(XContentParser parser, boolean coerce) throws IOException {
  787. int value = parser.intValue(coerce);
  788. if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
  789. throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
  790. }
  791. return (byte) value;
  792. }
  793. @Override
  794. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  795. if (isOutOfRange(value)) {
  796. return new MatchNoDocsQuery("Value [" + value + "] is out of range");
  797. }
  798. return INTEGER.termQuery(field, value, isIndexed, hasDocValues);
  799. }
  800. @Override
  801. public Query termsQuery(String field, Collection<?> values) {
  802. return INTEGER.termsQuery(field, values);
  803. }
  804. @Override
  805. public Query rangeQuery(
  806. String field,
  807. Object lowerTerm,
  808. Object upperTerm,
  809. boolean includeLower,
  810. boolean includeUpper,
  811. boolean hasDocValues,
  812. SearchExecutionContext context,
  813. boolean isIndexed
  814. ) {
  815. return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues, context, isIndexed);
  816. }
  817. @Override
  818. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  819. INTEGER.addFields(document, name, value, indexed, docValued, stored);
  820. }
  821. @Override
  822. public long toSortableLong(Number value) {
  823. return INTEGER.toSortableLong(value);
  824. }
  825. @Override
  826. Number valueForSearch(Number value) {
  827. return value.byteValue();
  828. }
  829. @Override
  830. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  831. return new SortedNumericIndexFieldData.Builder(
  832. ft.name(),
  833. numericType(),
  834. valuesSourceType,
  835. ByteDocValuesField::new,
  836. ft.isIndexed()
  837. );
  838. }
  839. @Override
  840. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  841. String name,
  842. ValuesSourceType valuesSourceType,
  843. SourceProvider sourceProvider,
  844. ValueFetcher valueFetcher
  845. ) {
  846. return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
  847. name,
  848. valuesSourceType,
  849. valueFetcher,
  850. sourceProvider,
  851. ByteDocValuesField::new
  852. );
  853. }
  854. @Override
  855. public void writeValue(XContentBuilder b, long value) throws IOException {
  856. b.value(value);
  857. }
  858. @Override
  859. BlockLoader blockLoaderFromDocValues(String fieldName) {
  860. return new BlockDocValuesReader.IntsBlockLoader(fieldName);
  861. }
  862. @Override
  863. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  864. return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
  865. }
  866. @Override
  867. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  868. return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  869. }
  870. private boolean isOutOfRange(Object value) {
  871. double doubleValue = objectToDouble(value);
  872. return doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE;
  873. }
  874. },
  875. SHORT("short", NumericType.SHORT) {
  876. @Override
  877. public Short parse(Object value, boolean coerce) {
  878. double doubleValue = objectToDouble(value);
  879. if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) {
  880. throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
  881. }
  882. if (coerce == false && doubleValue % 1 != 0) {
  883. throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
  884. }
  885. if (value instanceof Number) {
  886. return ((Number) value).shortValue();
  887. }
  888. return (short) doubleValue;
  889. }
  890. @Override
  891. public Number parsePoint(byte[] value) {
  892. return INTEGER.parsePoint(value).shortValue();
  893. }
  894. @Override
  895. public Short parse(XContentParser parser, boolean coerce) throws IOException {
  896. return parser.shortValue(coerce);
  897. }
  898. @Override
  899. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  900. if (isOutOfRange(value)) {
  901. return Queries.newMatchNoDocsQuery("Value [" + value + "] is out of range");
  902. }
  903. return INTEGER.termQuery(field, value, isIndexed, hasDocValues);
  904. }
  905. @Override
  906. public Query termsQuery(String field, Collection<?> values) {
  907. return INTEGER.termsQuery(field, values);
  908. }
  909. @Override
  910. public Query rangeQuery(
  911. String field,
  912. Object lowerTerm,
  913. Object upperTerm,
  914. boolean includeLower,
  915. boolean includeUpper,
  916. boolean hasDocValues,
  917. SearchExecutionContext context,
  918. boolean isIndexed
  919. ) {
  920. return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues, context, isIndexed);
  921. }
  922. @Override
  923. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  924. INTEGER.addFields(document, name, value, indexed, docValued, stored);
  925. }
  926. @Override
  927. public long toSortableLong(Number value) {
  928. return INTEGER.toSortableLong(value);
  929. }
  930. @Override
  931. Number valueForSearch(Number value) {
  932. return value.shortValue();
  933. }
  934. @Override
  935. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  936. return new SortedNumericIndexFieldData.Builder(
  937. ft.name(),
  938. numericType(),
  939. valuesSourceType,
  940. ShortDocValuesField::new,
  941. ft.isIndexed()
  942. );
  943. }
  944. @Override
  945. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  946. String name,
  947. ValuesSourceType valuesSourceType,
  948. SourceProvider sourceProvider,
  949. ValueFetcher valueFetcher
  950. ) {
  951. return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
  952. name,
  953. valuesSourceType,
  954. valueFetcher,
  955. sourceProvider,
  956. ShortDocValuesField::new
  957. );
  958. }
  959. @Override
  960. public void writeValue(XContentBuilder b, long value) throws IOException {
  961. b.value(value);
  962. }
  963. @Override
  964. BlockLoader blockLoaderFromDocValues(String fieldName) {
  965. return new BlockDocValuesReader.IntsBlockLoader(fieldName);
  966. }
  967. @Override
  968. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  969. return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
  970. }
  971. @Override
  972. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  973. return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  974. }
  975. private boolean isOutOfRange(Object value) {
  976. double doubleValue = objectToDouble(value);
  977. return doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE;
  978. }
  979. },
  980. INTEGER("integer", NumericType.INT) {
  981. @Override
  982. public Integer parse(Object value, boolean coerce) {
  983. double doubleValue = objectToDouble(value);
  984. if (isOutOfRange(doubleValue)) {
  985. throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer");
  986. }
  987. if (coerce == false && doubleValue % 1 != 0) {
  988. throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
  989. }
  990. if (value instanceof Number) {
  991. return ((Number) value).intValue();
  992. }
  993. return (int) doubleValue;
  994. }
  995. private boolean isOutOfRange(double value) {
  996. return value < Integer.MIN_VALUE || value > Integer.MAX_VALUE;
  997. }
  998. @Override
  999. public Number parsePoint(byte[] value) {
  1000. return IntPoint.decodeDimension(value, 0);
  1001. }
  1002. @Override
  1003. public Integer parse(XContentParser parser, boolean coerce) throws IOException {
  1004. return parser.intValue(coerce);
  1005. }
  1006. @Override
  1007. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  1008. if (hasDecimalPart(value)) {
  1009. return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
  1010. }
  1011. double doubleValue = objectToDouble(value);
  1012. if (isOutOfRange(doubleValue)) {
  1013. return Queries.newMatchNoDocsQuery("Value [" + value + "] is out of range");
  1014. }
  1015. int v = parse(value, true);
  1016. if (isIndexed && hasDocValues) {
  1017. return NumericField.newExactIntQuery(field, v);
  1018. } else if (isIndexed) {
  1019. return IntPoint.newExactQuery(field, v);
  1020. } else {
  1021. return SortedNumericDocValuesField.newSlowExactQuery(field, v);
  1022. }
  1023. }
  1024. @Override
  1025. public Query termsQuery(String field, Collection<?> values) {
  1026. int[] v = new int[values.size()];
  1027. int upTo = 0;
  1028. for (Object value : values) {
  1029. if (hasDecimalPart(value) == false) {
  1030. v[upTo++] = parse(value, true);
  1031. }
  1032. }
  1033. if (upTo == 0) {
  1034. return Queries.newMatchNoDocsQuery("All values have a decimal part");
  1035. }
  1036. if (upTo != v.length) {
  1037. v = Arrays.copyOf(v, upTo);
  1038. }
  1039. return IntPoint.newSetQuery(field, v);
  1040. }
  1041. @Override
  1042. public Query rangeQuery(
  1043. String field,
  1044. Object lowerTerm,
  1045. Object upperTerm,
  1046. boolean includeLower,
  1047. boolean includeUpper,
  1048. boolean hasDocValues,
  1049. SearchExecutionContext context,
  1050. boolean isIndexed
  1051. ) {
  1052. int l = Integer.MIN_VALUE;
  1053. int u = Integer.MAX_VALUE;
  1054. if (lowerTerm != null) {
  1055. l = parse(lowerTerm, true);
  1056. // if the lower bound is decimal:
  1057. // - if the bound is positive then we increment it:
  1058. // if lowerTerm=1.5 then the (inclusive) bound becomes 2
  1059. // - if the bound is negative then we leave it as is:
  1060. // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
  1061. boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
  1062. if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
  1063. if (l == Integer.MAX_VALUE) {
  1064. return new MatchNoDocsQuery();
  1065. }
  1066. ++l;
  1067. }
  1068. }
  1069. if (upperTerm != null) {
  1070. u = parse(upperTerm, true);
  1071. boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
  1072. if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
  1073. if (u == Integer.MIN_VALUE) {
  1074. return new MatchNoDocsQuery();
  1075. }
  1076. --u;
  1077. }
  1078. }
  1079. Query query;
  1080. if (isIndexed) {
  1081. query = IntPoint.newRangeQuery(field, l, u);
  1082. if (hasDocValues) {
  1083. Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
  1084. query = new IndexOrDocValuesQuery(query, dvQuery);
  1085. }
  1086. } else {
  1087. query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
  1088. }
  1089. if (hasDocValues && context.indexSortedOnField(field)) {
  1090. query = new XIndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
  1091. }
  1092. return query;
  1093. }
  1094. @Override
  1095. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  1096. final int i = value.intValue();
  1097. if (indexed && docValued) {
  1098. document.add(new IntField(name, i, Field.Store.NO));
  1099. } else if (docValued) {
  1100. document.add(new SortedNumericDocValuesField(name, i));
  1101. } else if (indexed) {
  1102. document.add(new IntPoint(name, i));
  1103. }
  1104. if (stored) {
  1105. document.add(new StoredField(name, i));
  1106. }
  1107. }
  1108. @Override
  1109. public long toSortableLong(Number value) {
  1110. return value.intValue();
  1111. }
  1112. @Override
  1113. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  1114. return new SortedNumericIndexFieldData.Builder(
  1115. ft.name(),
  1116. numericType(),
  1117. valuesSourceType,
  1118. IntegerDocValuesField::new,
  1119. ft.isIndexed()
  1120. );
  1121. }
  1122. @Override
  1123. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  1124. String name,
  1125. ValuesSourceType valuesSourceType,
  1126. SourceProvider sourceProvider,
  1127. ValueFetcher valueFetcher
  1128. ) {
  1129. return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
  1130. name,
  1131. valuesSourceType,
  1132. valueFetcher,
  1133. sourceProvider,
  1134. IntegerDocValuesField::new
  1135. );
  1136. }
  1137. @Override
  1138. public void writeValue(XContentBuilder b, long value) throws IOException {
  1139. b.value(value);
  1140. }
  1141. @Override
  1142. BlockLoader blockLoaderFromDocValues(String fieldName) {
  1143. return new BlockDocValuesReader.IntsBlockLoader(fieldName);
  1144. }
  1145. @Override
  1146. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  1147. return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
  1148. }
  1149. @Override
  1150. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  1151. return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
  1152. }
  1153. },
  1154. LONG("long", NumericType.LONG) {
  1155. @Override
  1156. public Long parse(Object value, boolean coerce) {
  1157. return objectToLong(value, coerce);
  1158. }
  1159. @Override
  1160. public Number parsePoint(byte[] value) {
  1161. return LongPoint.decodeDimension(value, 0);
  1162. }
  1163. @Override
  1164. public Long parse(XContentParser parser, boolean coerce) throws IOException {
  1165. return parser.longValue(coerce);
  1166. }
  1167. @Override
  1168. public FieldValues<Number> compile(String fieldName, Script script, ScriptCompiler compiler) {
  1169. final LongFieldScript.Factory scriptFactory = compiler.compile(script, LongFieldScript.CONTEXT);
  1170. return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL)
  1171. .newInstance(ctx)
  1172. .runForDoc(doc, consumer::accept);
  1173. }
  1174. @Override
  1175. public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
  1176. if (hasDecimalPart(value)) {
  1177. return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
  1178. }
  1179. if (isOutOfRange(value)) {
  1180. return Queries.newMatchNoDocsQuery("Value [" + value + "] is out of range");
  1181. }
  1182. long v = parse(value, true);
  1183. if (isIndexed && hasDocValues) {
  1184. return NumericField.newExactLongQuery(field, v);
  1185. } else if (isIndexed) {
  1186. return LongPoint.newExactQuery(field, v);
  1187. } else {
  1188. return SortedNumericDocValuesField.newSlowExactQuery(field, v);
  1189. }
  1190. }
  1191. @Override
  1192. public Query termsQuery(String field, Collection<?> values) {
  1193. long[] v = new long[values.size()];
  1194. int upTo = 0;
  1195. for (Object value : values) {
  1196. if (hasDecimalPart(value) == false) {
  1197. v[upTo++] = parse(value, true);
  1198. }
  1199. }
  1200. if (upTo == 0) {
  1201. return Queries.newMatchNoDocsQuery("All values have a decimal part");
  1202. }
  1203. if (upTo != v.length) {
  1204. v = Arrays.copyOf(v, upTo);
  1205. }
  1206. return LongPoint.newSetQuery(field, v);
  1207. }
  1208. @Override
  1209. public Query rangeQuery(
  1210. String field,
  1211. Object lowerTerm,
  1212. Object upperTerm,
  1213. boolean includeLower,
  1214. boolean includeUpper,
  1215. boolean hasDocValues,
  1216. SearchExecutionContext context,
  1217. boolean isIndexed
  1218. ) {
  1219. return longRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
  1220. Query query;
  1221. if (isIndexed) {
  1222. query = LongPoint.newRangeQuery(field, l, u);
  1223. if (hasDocValues) {
  1224. Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
  1225. query = new IndexOrDocValuesQuery(query, dvQuery);
  1226. }
  1227. } else {
  1228. query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
  1229. }
  1230. if (hasDocValues && context.indexSortedOnField(field)) {
  1231. query = new XIndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
  1232. }
  1233. return query;
  1234. });
  1235. }
  1236. @Override
  1237. public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
  1238. final long l = value.longValue();
  1239. if (indexed && docValued) {
  1240. document.add(new LongField(name, l, Field.Store.NO));
  1241. } else if (docValued) {
  1242. document.add(new SortedNumericDocValuesField(name, l));
  1243. } else if (indexed) {
  1244. document.add(new LongPoint(name, l));
  1245. }
  1246. if (stored) {
  1247. document.add(new StoredField(name, l));
  1248. }
  1249. }
  1250. @Override
  1251. public long toSortableLong(Number value) {
  1252. return value.longValue();
  1253. }
  1254. @Override
  1255. public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) {
  1256. return new SortedNumericIndexFieldData.Builder(
  1257. ft.name(),
  1258. numericType(),
  1259. valuesSourceType,
  1260. LongDocValuesField::new,
  1261. ft.isIndexed()
  1262. );
  1263. }
  1264. @Override
  1265. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  1266. String name,
  1267. ValuesSourceType valuesSourceType,
  1268. SourceProvider sourceProvider,
  1269. ValueFetcher valueFetcher
  1270. ) {
  1271. return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
  1272. name,
  1273. valuesSourceType,
  1274. valueFetcher,
  1275. sourceProvider,
  1276. LongDocValuesField::new
  1277. );
  1278. }
  1279. @Override
  1280. public void writeValue(XContentBuilder b, long value) throws IOException {
  1281. b.value(value);
  1282. }
  1283. @Override
  1284. BlockLoader blockLoaderFromDocValues(String fieldName) {
  1285. return new BlockDocValuesReader.LongsBlockLoader(fieldName);
  1286. }
  1287. @Override
  1288. BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
  1289. return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
  1290. }
  1291. @Override
  1292. BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
  1293. var reader = new NumberFallbackSyntheticSourceReader(this, nullValue, coerce) {
  1294. @Override
  1295. public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
  1296. var builder = (BlockLoader.LongBuilder) blockBuilder;
  1297. for (var value : values) {
  1298. builder.appendLong(value.longValue());
  1299. }
  1300. }
  1301. };
  1302. return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
  1303. @Override
  1304. public Builder builder(BlockFactory factory, int expectedCount) {
  1305. return factory.longs(expectedCount);
  1306. }
  1307. };
  1308. }
  1309. private boolean isOutOfRange(Object value) {
  1310. if (value instanceof Long) {
  1311. return false;
  1312. }
  1313. String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
  1314. BigDecimal bigDecimalValue = new BigDecimal(stringValue);
  1315. return bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0
  1316. || bigDecimalValue.compareTo(BigDecimal.valueOf(Long.MIN_VALUE)) < 0;
  1317. }
  1318. };
  1319. private final String name;
  1320. private final NumericType numericType;
  1321. private final TypeParser parser;
  1322. NumberType(String name, NumericType numericType) {
  1323. this.name = name;
  1324. this.numericType = numericType;
  1325. this.parser = createTypeParserWithLegacySupport(
  1326. (n, c) -> new Builder(
  1327. n,
  1328. this,
  1329. c.scriptCompiler(),
  1330. c.getSettings(),
  1331. c.indexVersionCreated(),
  1332. c.getIndexSettings().getMode(),
  1333. c.getIndexSettings().sourceKeepMode()
  1334. )
  1335. );
  1336. }
  1337. /** Get the associated type name. */
  1338. public final String typeName() {
  1339. return name;
  1340. }
  1341. /** Get the associated numeric type */
  1342. public final NumericType numericType() {
  1343. return numericType;
  1344. }
  1345. public final TypeParser parser() {
  1346. return parser;
  1347. }
  1348. public abstract Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues);
  1349. public abstract Query termsQuery(String field, Collection<?> values);
  1350. public abstract Query rangeQuery(
  1351. String field,
  1352. Object lowerTerm,
  1353. Object upperTerm,
  1354. boolean includeLower,
  1355. boolean includeUpper,
  1356. boolean hasDocValues,
  1357. SearchExecutionContext context,
  1358. boolean isIndexed
  1359. );
  1360. public abstract Number parse(XContentParser parser, boolean coerce) throws IOException;
  1361. public abstract Number parse(Object value, boolean coerce);
  1362. public abstract Number parsePoint(byte[] value);
  1363. /**
  1364. * Maps the given {@code value} to one or more Lucene field values ands them to the given {@code document} under the given
  1365. * {@code name}.
  1366. *
  1367. * @param document document to add fields to
  1368. * @param name field name
  1369. * @param value value to map
  1370. * @param indexed whether or not the field is indexed
  1371. * @param docValued whether or not doc values should be added
  1372. * @param stored whether or not the field is stored
  1373. */
  1374. public abstract void addFields(
  1375. LuceneDocument document,
  1376. String name,
  1377. Number value,
  1378. boolean indexed,
  1379. boolean docValued,
  1380. boolean stored
  1381. );
  1382. /**
  1383. * For a given {@code Number}, returns the sortable long representation that will be stored in the doc values.
  1384. * @param value number to convert
  1385. * @return sortable long representation
  1386. */
  1387. public abstract long toSortableLong(Number value);
  1388. public FieldValues<Number> compile(String fieldName, Script script, ScriptCompiler compiler) {
  1389. // only implemented for long and double fields
  1390. throw new IllegalArgumentException("Unknown parameter [script] for mapper [" + fieldName + "]");
  1391. }
  1392. Number valueForSearch(Number value) {
  1393. return value;
  1394. }
  1395. /**
  1396. * Returns true if the object is a number and has a decimal part
  1397. */
  1398. public static boolean hasDecimalPart(Object number) {
  1399. if (number instanceof Byte || number instanceof Short || number instanceof Integer || number instanceof Long) {
  1400. return false;
  1401. }
  1402. if (number instanceof Number) {
  1403. double doubleValue = ((Number) number).doubleValue();
  1404. return doubleValue % 1 != 0;
  1405. }
  1406. if (number instanceof BytesRef) {
  1407. number = ((BytesRef) number).utf8ToString();
  1408. }
  1409. if (number instanceof String) {
  1410. return Double.parseDouble((String) number) % 1 != 0;
  1411. }
  1412. return false;
  1413. }
  1414. /**
  1415. * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0
  1416. */
  1417. static double signum(Object value) {
  1418. if (value instanceof Number) {
  1419. double doubleValue = ((Number) value).doubleValue();
  1420. return Math.signum(doubleValue);
  1421. }
  1422. if (value instanceof BytesRef) {
  1423. value = ((BytesRef) value).utf8ToString();
  1424. }
  1425. return Math.signum(Double.parseDouble(value.toString()));
  1426. }
  1427. /**
  1428. * Converts an Object to a double by checking it against known types first
  1429. */
  1430. public static double objectToDouble(Object value) {
  1431. double doubleValue;
  1432. if (value instanceof Number) {
  1433. doubleValue = ((Number) value).doubleValue();
  1434. } else if (value instanceof BytesRef) {
  1435. doubleValue = Double.parseDouble(((BytesRef) value).utf8ToString());
  1436. } else {
  1437. doubleValue = Double.parseDouble(value.toString());
  1438. }
  1439. return doubleValue;
  1440. }
  1441. /**
  1442. * Converts an Object to a {@code long} by checking it against known
  1443. * types and checking its range.
  1444. */
  1445. public static long objectToLong(Object value, boolean coerce) {
  1446. if (value instanceof Long) {
  1447. return (Long) value;
  1448. }
  1449. double doubleValue = objectToDouble(value);
  1450. // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
  1451. // be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
  1452. if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
  1453. throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
  1454. }
  1455. if (coerce == false && doubleValue % 1 != 0) {
  1456. throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
  1457. }
  1458. // longs need special handling so we don't lose precision while parsing
  1459. String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
  1460. return Numbers.toLong(stringValue, coerce);
  1461. }
  1462. public static Query doubleRangeQuery(
  1463. Object lowerTerm,
  1464. Object upperTerm,
  1465. boolean includeLower,
  1466. boolean includeUpper,
  1467. BiFunction<Double, Double, Query> builder
  1468. ) {
  1469. double l = Double.NEGATIVE_INFINITY;
  1470. double u = Double.POSITIVE_INFINITY;
  1471. if (lowerTerm != null) {
  1472. l = objectToDouble(lowerTerm);
  1473. if (includeLower == false) {
  1474. l = DoublePoint.nextUp(l);
  1475. }
  1476. }
  1477. if (upperTerm != null) {
  1478. u = objectToDouble(upperTerm);
  1479. if (includeUpper == false) {
  1480. u = DoublePoint.nextDown(u);
  1481. }
  1482. }
  1483. return builder.apply(l, u);
  1484. }
  1485. /**
  1486. * Processes query bounds into {@code long}s and delegates the
  1487. * provided {@code builder} to build a range query.
  1488. */
  1489. public static Query longRangeQuery(
  1490. Object lowerTerm,
  1491. Object upperTerm,
  1492. boolean includeLower,
  1493. boolean includeUpper,
  1494. BiFunction<Long, Long, Query> builder
  1495. ) {
  1496. long l = Long.MIN_VALUE;
  1497. long u = Long.MAX_VALUE;
  1498. if (lowerTerm != null) {
  1499. l = objectToLong(lowerTerm, true);
  1500. // if the lower bound is decimal:
  1501. // - if the bound is positive then we increment it:
  1502. // if lowerTerm=1.5 then the (inclusive) bound becomes 2
  1503. // - if the bound is negative then we leave it as is:
  1504. // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
  1505. boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
  1506. if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
  1507. if (l == Long.MAX_VALUE) {
  1508. return new MatchNoDocsQuery();
  1509. }
  1510. ++l;
  1511. }
  1512. }
  1513. if (upperTerm != null) {
  1514. u = objectToLong(upperTerm, true);
  1515. boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
  1516. if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
  1517. if (u == Long.MIN_VALUE) {
  1518. return new MatchNoDocsQuery();
  1519. }
  1520. --u;
  1521. }
  1522. }
  1523. return builder.apply(l, u);
  1524. }
  1525. public abstract IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType);
  1526. public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
  1527. String name,
  1528. ValuesSourceType valuesSourceType,
  1529. SourceProvider sourceProvider,
  1530. ValueFetcher valueFetcher
  1531. ) {
  1532. throw new UnsupportedOperationException("not supported for source fallback");
  1533. }
  1534. /**
  1535. * Adjusts a value to the value it would have been had it been parsed by that mapper
  1536. * and then cast up to a double. This is meant to be an entry point to manipulate values
  1537. * before the actual value is parsed.
  1538. *
  1539. * @param value the value to reduce to the field stored value
  1540. * @return the double value
  1541. */
  1542. public double reduceToStoredPrecision(double value) {
  1543. return ((Number) value).doubleValue();
  1544. }
  1545. abstract void writeValue(XContentBuilder builder, long longValue) throws IOException;
  1546. SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
  1547. return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
  1548. @Override
  1549. public void writeValue(XContentBuilder b, long value) throws IOException {
  1550. NumberType.this.writeValue(b, value);
  1551. }
  1552. };
  1553. }
  1554. abstract BlockLoader blockLoaderFromDocValues(String fieldName);
  1555. abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
  1556. abstract BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce);
  1557. // All values that fit into integer are returned as integers
  1558. private static BlockLoader integerBlockLoaderFromFallbackSyntheticSource(
  1559. NumberType type,
  1560. String fieldName,
  1561. Number nullValue,
  1562. boolean coerce
  1563. ) {
  1564. var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
  1565. @Override
  1566. public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
  1567. var builder = (BlockLoader.IntBuilder) blockBuilder;
  1568. for (var value : values) {
  1569. builder.appendInt(value.intValue());
  1570. }
  1571. }
  1572. };
  1573. return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
  1574. @Override
  1575. public Builder builder(BlockFactory factory, int expectedCount) {
  1576. return factory.ints(expectedCount);
  1577. }
  1578. };
  1579. }
  1580. // All floating point values are returned as doubles
  1581. private static BlockLoader floatingPointBlockLoaderFromFallbackSyntheticSource(
  1582. NumberType type,
  1583. String fieldName,
  1584. Number nullValue,
  1585. boolean coerce
  1586. ) {
  1587. var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
  1588. @Override
  1589. public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
  1590. var builder = (BlockLoader.DoubleBuilder) blockBuilder;
  1591. for (var value : values) {
  1592. builder.appendDouble(value.doubleValue());
  1593. }
  1594. }
  1595. };
  1596. return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
  1597. @Override
  1598. public Builder builder(BlockFactory factory, int expectedCount) {
  1599. return factory.doubles(expectedCount);
  1600. }
  1601. };
  1602. }
  1603. abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader<Number> {
  1604. private final NumberType type;
  1605. private final Number nullValue;
  1606. private final boolean coerce;
  1607. NumberFallbackSyntheticSourceReader(NumberType type, Number nullValue, boolean coerce) {
  1608. super(nullValue);
  1609. this.type = type;
  1610. this.nullValue = nullValue;
  1611. this.coerce = coerce;
  1612. }
  1613. @Override
  1614. public void convertValue(Object value, List<Number> accumulator) {
  1615. if (coerce && value.equals("")) {
  1616. if (nullValue != null) {
  1617. accumulator.add(nullValue);
  1618. }
  1619. }
  1620. try {
  1621. var converted = type.parse(value, coerce);
  1622. accumulator.add(converted);
  1623. } catch (Exception e) {
  1624. // Malformed value, skip it
  1625. }
  1626. }
  1627. @Override
  1628. public void parseNonNullValue(XContentParser parser, List<Number> accumulator) throws IOException {
  1629. // Aligned with implementation of `value(XContentParser)`
  1630. if (coerce && parser.currentToken() == Token.VALUE_STRING && parser.textLength() == 0) {
  1631. if (nullValue != null) {
  1632. accumulator.add(nullValue);
  1633. }
  1634. }
  1635. try {
  1636. Number rawValue = type.parse(parser, coerce);
  1637. // Transform number to correct type (e.g. reduce precision)
  1638. accumulator.add(type.parse(rawValue, coerce));
  1639. } catch (Exception e) {
  1640. // Malformed value, skip it
  1641. }
  1642. }
  1643. };
  1644. }
  1645. public static class NumberFieldType extends SimpleMappedFieldType {
  1646. private final NumberType type;
  1647. private final boolean coerce;
  1648. private final Number nullValue;
  1649. private final FieldValues<Number> scriptValues;
  1650. private final boolean isDimension;
  1651. private final MetricType metricType;
  1652. private final IndexMode indexMode;
  1653. private final boolean isSyntheticSource;
  1654. public NumberFieldType(
  1655. String name,
  1656. NumberType type,
  1657. boolean isIndexed,
  1658. boolean isStored,
  1659. boolean hasDocValues,
  1660. boolean coerce,
  1661. Number nullValue,
  1662. Map<String, String> meta,
  1663. FieldValues<Number> script,
  1664. boolean isDimension,
  1665. MetricType metricType,
  1666. IndexMode indexMode,
  1667. boolean isSyntheticSource
  1668. ) {
  1669. super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
  1670. this.type = Objects.requireNonNull(type);
  1671. this.coerce = coerce;
  1672. this.nullValue = nullValue;
  1673. this.scriptValues = script;
  1674. this.isDimension = isDimension;
  1675. this.metricType = metricType;
  1676. this.indexMode = indexMode;
  1677. this.isSyntheticSource = isSyntheticSource;
  1678. }
  1679. NumberFieldType(String name, Builder builder, boolean isSyntheticSource) {
  1680. this(
  1681. name,
  1682. builder.type,
  1683. builder.indexed.getValue() && builder.indexCreatedVersion.isLegacyIndexVersion() == false,
  1684. builder.stored.getValue(),
  1685. builder.hasDocValues.getValue(),
  1686. builder.coerce.getValue().value(),
  1687. builder.nullValue.getValue(),
  1688. builder.meta.getValue(),
  1689. builder.scriptValues(),
  1690. builder.dimension.getValue(),
  1691. builder.metric.getValue(),
  1692. builder.indexMode,
  1693. isSyntheticSource
  1694. );
  1695. }
  1696. public NumberFieldType(String name, NumberType type) {
  1697. this(name, type, true, true);
  1698. }
  1699. public NumberFieldType(String name, NumberType type, boolean isIndexed, boolean hasDocValues) {
  1700. this(name, type, isIndexed, false, hasDocValues, true, null, Collections.emptyMap(), null, false, null, null, false);
  1701. }
  1702. @Override
  1703. public String typeName() {
  1704. return type.name;
  1705. }
  1706. /**
  1707. * This method reinterprets a double precision value based on the maximum precision of the stored number field. Mostly this
  1708. * corrects for unrepresentable values which have different approximations when cast from floats than when parsed as doubles.
  1709. * It may seem strange to convert a double to a double, and it is. This function's goal is to reduce the precision
  1710. * on the double in the case that the backing number type would have parsed the value differently. This is to address
  1711. * the problem where (e.g.) 0.04F &lt; 0.04D, which causes problems for range aggregations.
  1712. */
  1713. public double reduceToStoredPrecision(double value) {
  1714. if (Double.isInfinite(value)) {
  1715. // Trying to parse infinite values into ints/longs throws. Understandably.
  1716. return value;
  1717. }
  1718. return type.reduceToStoredPrecision(value);
  1719. }
  1720. public NumericType numericType() {
  1721. return type.numericType();
  1722. }
  1723. @Override
  1724. public boolean mayExistInIndex(SearchExecutionContext context) {
  1725. return context.fieldExistsInIndex(this.name());
  1726. }
  1727. public boolean isSearchable() {
  1728. return isIndexed() || hasDocValues();
  1729. }
  1730. @Override
  1731. public Query termQuery(Object value, SearchExecutionContext context) {
  1732. failIfNotIndexedNorDocValuesFallback(context);
  1733. return type.termQuery(name(), value, isIndexed(), hasDocValues());
  1734. }
  1735. @Override
  1736. public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
  1737. failIfNotIndexedNorDocValuesFallback(context);
  1738. if (isIndexed()) {
  1739. return type.termsQuery(name(), values);
  1740. } else {
  1741. return super.termsQuery(values, context);
  1742. }
  1743. }
  1744. @Override
  1745. public Query rangeQuery(
  1746. Object lowerTerm,
  1747. Object upperTerm,
  1748. boolean includeLower,
  1749. boolean includeUpper,
  1750. SearchExecutionContext context
  1751. ) {
  1752. failIfNotIndexedNorDocValuesFallback(context);
  1753. return type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues(), context, isIndexed());
  1754. }
  1755. @Override
  1756. public Function<byte[], Number> pointReaderIfPossible() {
  1757. if (isIndexed()) {
  1758. return this::parsePoint;
  1759. }
  1760. return null;
  1761. }
  1762. @Override
  1763. public BlockLoader blockLoader(BlockLoaderContext blContext) {
  1764. if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) {
  1765. return type.blockLoaderFromDocValues(name());
  1766. }
  1767. // Multi fields don't have fallback synthetic source.
  1768. if (isSyntheticSource && blContext.parentField(name()) == null) {
  1769. return type.blockLoaderFromFallbackSyntheticSource(name(), nullValue, coerce);
  1770. }
  1771. BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && (isStored() || isIndexed())
  1772. // We only write the field names field if there aren't doc values or norms
  1773. ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
  1774. : BlockSourceReader.lookupMatchingAll();
  1775. return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
  1776. }
  1777. @Override
  1778. public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
  1779. FielddataOperation operation = fieldDataContext.fielddataOperation();
  1780. if (fieldDataContext.fielddataOperation() == FielddataOperation.SEARCH) {
  1781. failIfNoDocValues();
  1782. }
  1783. ValuesSourceType valuesSourceType = indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER
  1784. ? TimeSeriesValuesSourceType.COUNTER
  1785. : type.numericType.getValuesSourceType();
  1786. if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) {
  1787. return type.getFieldDataBuilder(this, valuesSourceType);
  1788. }
  1789. if (operation == FielddataOperation.SCRIPT) {
  1790. SearchLookup searchLookup = fieldDataContext.lookupSupplier().get();
  1791. Set<String> sourcePaths = fieldDataContext.sourcePathsLookup().apply(name());
  1792. return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths));
  1793. }
  1794. throw new IllegalStateException("unknown field data type [" + operation.name() + "]");
  1795. }
  1796. @Override
  1797. public Object valueForDisplay(Object value) {
  1798. if (value == null) {
  1799. return null;
  1800. }
  1801. return type.valueForSearch((Number) value);
  1802. }
  1803. @Override
  1804. public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
  1805. if (format != null) {
  1806. throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
  1807. }
  1808. if (this.scriptValues != null) {
  1809. return FieldValues.valueFetcher(this.scriptValues, context);
  1810. }
  1811. return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
  1812. }
  1813. private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
  1814. return new SourceValueFetcher(sourcePaths, nullValue) {
  1815. @Override
  1816. protected Object parseSourceValue(Object value) {
  1817. if (value.equals("")) {
  1818. return nullValue;
  1819. }
  1820. return type.parse(value, coerce);
  1821. }
  1822. };
  1823. }
  1824. @Override
  1825. public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
  1826. checkNoTimeZone(timeZone);
  1827. if (format == null) {
  1828. return DocValueFormat.RAW;
  1829. }
  1830. return new DocValueFormat.Decimal(format);
  1831. }
  1832. public Number parsePoint(byte[] value) {
  1833. return type.parsePoint(value);
  1834. }
  1835. @Override
  1836. public CollapseType collapseType() {
  1837. return CollapseType.NUMERIC;
  1838. }
  1839. @Override
  1840. public boolean isDimension() {
  1841. return isDimension;
  1842. }
  1843. @Override
  1844. public boolean hasScriptValues() {
  1845. return scriptValues != null;
  1846. }
  1847. /**
  1848. * If field is a time series metric field, returns its metric type
  1849. * @return the metric type or null
  1850. */
  1851. public MetricType getMetricType() {
  1852. return metricType;
  1853. }
  1854. }
  1855. private final NumberType type;
  1856. private final boolean indexed;
  1857. private final boolean hasDocValues;
  1858. private final boolean stored;
  1859. private final Explicit<Boolean> ignoreMalformed;
  1860. private final Explicit<Boolean> coerce;
  1861. private final Number nullValue;
  1862. private final FieldValues<Number> scriptValues;
  1863. private final boolean ignoreMalformedByDefault;
  1864. private final boolean coerceByDefault;
  1865. private final boolean dimension;
  1866. private final ScriptCompiler scriptCompiler;
  1867. private final Script script;
  1868. private final MetricType metricType;
  1869. private boolean allowMultipleValues;
  1870. private final IndexVersion indexCreatedVersion;
  1871. private final boolean isSyntheticSource;
  1872. private final String offsetsFieldName;
  1873. private final IndexMode indexMode;
  1874. private final SourceKeepMode indexSourceKeepMode;
  1875. private NumberFieldMapper(
  1876. String simpleName,
  1877. MappedFieldType mappedFieldType,
  1878. BuilderParams builderParams,
  1879. boolean isSyntheticSource,
  1880. Builder builder,
  1881. String offsetsFieldName
  1882. ) {
  1883. super(simpleName, mappedFieldType, builderParams);
  1884. this.type = builder.type;
  1885. this.indexed = builder.indexed.getValue();
  1886. this.hasDocValues = builder.hasDocValues.getValue();
  1887. this.stored = builder.stored.getValue();
  1888. this.ignoreMalformed = builder.ignoreMalformed.getValue();
  1889. this.coerce = builder.coerce.getValue();
  1890. this.nullValue = builder.nullValue.getValue();
  1891. this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
  1892. this.coerceByDefault = builder.coerce.getDefaultValue().value();
  1893. this.scriptValues = builder.scriptValues();
  1894. this.dimension = builder.dimension.getValue();
  1895. this.scriptCompiler = builder.scriptCompiler;
  1896. this.script = builder.script.getValue();
  1897. this.metricType = builder.metric.getValue();
  1898. this.allowMultipleValues = builder.allowMultipleValues;
  1899. this.indexCreatedVersion = builder.indexCreatedVersion;
  1900. this.isSyntheticSource = isSyntheticSource;
  1901. this.indexMode = builder.indexMode;
  1902. this.offsetsFieldName = offsetsFieldName;
  1903. this.indexSourceKeepMode = builder.indexSourceKeepMode;
  1904. }
  1905. boolean coerce() {
  1906. return coerce.value();
  1907. }
  1908. @Override
  1909. public boolean ignoreMalformed() {
  1910. return ignoreMalformed.value();
  1911. }
  1912. @Override
  1913. public NumberFieldType fieldType() {
  1914. return (NumberFieldType) super.fieldType();
  1915. }
  1916. @Override
  1917. public String getOffsetFieldName() {
  1918. return offsetsFieldName;
  1919. }
  1920. public NumberType type() {
  1921. return type;
  1922. }
  1923. @Override
  1924. protected String contentType() {
  1925. return fieldType().type.typeName();
  1926. }
  1927. @Override
  1928. protected void parseCreateField(DocumentParserContext context) throws IOException {
  1929. Number value;
  1930. try {
  1931. value = value(context.parser());
  1932. } catch (IllegalArgumentException e) {
  1933. if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
  1934. context.addIgnoredField(mappedFieldType.name());
  1935. if (isSyntheticSource) {
  1936. // Save a copy of the field so synthetic source can load it
  1937. context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
  1938. }
  1939. return;
  1940. } else {
  1941. throw e;
  1942. }
  1943. }
  1944. if (value != null) {
  1945. indexValue(context, value);
  1946. } else {
  1947. value = fieldType().nullValue;
  1948. }
  1949. if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
  1950. if (value != null) {
  1951. // We cannot simply cast value to Comparable<> because we need to also capture the potential loss of precision that occurs
  1952. // when the value is stored into the doc values.
  1953. long sortableLongValue = type.toSortableLong(value);
  1954. context.getOffSetContext().recordOffset(offsetsFieldName, sortableLongValue);
  1955. } else {
  1956. context.getOffSetContext().recordNull(offsetsFieldName);
  1957. }
  1958. }
  1959. }
  1960. /**
  1961. * Read the value at the current position of the parser. For numeric fields
  1962. * this is called by {@link #parseCreateField} but it is public so it can
  1963. * be used by other fields that want to share the behavior of numeric fields.
  1964. * @throws IllegalArgumentException if there was an error parsing the value from the json
  1965. * @throws IOException if there was any other IO error
  1966. */
  1967. public Number value(XContentParser parser) throws IllegalArgumentException, IOException {
  1968. final Token currentToken = parser.currentToken();
  1969. if (currentToken == Token.VALUE_NULL) {
  1970. return nullValue;
  1971. }
  1972. if (coerce() && currentToken == Token.VALUE_STRING && parser.textLength() == 0) {
  1973. return nullValue;
  1974. }
  1975. if (currentToken == Token.START_OBJECT) {
  1976. throw new IllegalArgumentException("Cannot parse object as number");
  1977. }
  1978. return type.parse(parser, coerce());
  1979. }
  1980. /**
  1981. * Index a value for this field. For numeric fields this is called by
  1982. * {@link #parseCreateField} but it is public so it can be used by other
  1983. * fields that want to share the behavior of numeric fields.
  1984. */
  1985. public void indexValue(DocumentParserContext context, Number numericValue) {
  1986. if (dimension && numericValue != null) {
  1987. context.getRoutingFields().addLong(fieldType().name(), numericValue.longValue());
  1988. }
  1989. fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored);
  1990. if (false == allowMultipleValues && (indexed || hasDocValues || stored)) {
  1991. // the last field is the current field, Add to the key map, so that we can validate if it has been added
  1992. List<IndexableField> fields = context.doc().getFields();
  1993. IndexableField last = fields.get(fields.size() - 1);
  1994. assert last.name().equals(fieldType().name())
  1995. : "last field name [" + last.name() + "] mis match field name [" + fieldType().name() + "]";
  1996. context.doc().onlyAddKey(fieldType().name(), fields.get(fields.size() - 1));
  1997. }
  1998. if (hasDocValues == false && (stored || indexed)) {
  1999. context.addToFieldNames(fieldType().name());
  2000. }
  2001. }
  2002. @Override
  2003. protected void indexScriptValues(
  2004. SearchLookup searchLookup,
  2005. LeafReaderContext readerContext,
  2006. int doc,
  2007. DocumentParserContext documentParserContext
  2008. ) {
  2009. this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, value -> indexValue(documentParserContext, value));
  2010. }
  2011. @Override
  2012. public FieldMapper.Builder getMergeBuilder() {
  2013. return new Builder(
  2014. leafName(),
  2015. type,
  2016. scriptCompiler,
  2017. ignoreMalformedByDefault,
  2018. coerceByDefault,
  2019. indexCreatedVersion,
  2020. indexMode,
  2021. indexSourceKeepMode
  2022. ).dimension(dimension).metric(metricType).allowMultipleValues(allowMultipleValues).init(this);
  2023. }
  2024. @Override
  2025. public void doValidate(MappingLookup lookup) {
  2026. if (dimension && null != lookup.nestedLookup().getNestedParent(fullPath())) {
  2027. throw new IllegalArgumentException(
  2028. TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + fullPath() + "]"
  2029. );
  2030. }
  2031. }
  2032. private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
  2033. if (offsetsFieldName != null) {
  2034. var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
  2035. layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue));
  2036. if (ignoreMalformed.value()) {
  2037. layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
  2038. }
  2039. return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
  2040. } else {
  2041. return type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value());
  2042. }
  2043. }
  2044. @Override
  2045. protected SyntheticSourceSupport syntheticSourceSupport() {
  2046. if (hasDocValues) {
  2047. return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
  2048. }
  2049. return super.syntheticSourceSupport();
  2050. }
  2051. // For testing only:
  2052. void setAllowMultipleValues(boolean allowMultipleValues) {
  2053. this.allowMultipleValues = allowMultipleValues;
  2054. }
  2055. }