|
|
@@ -28,7 +28,6 @@ import org.elasticsearch.common.Priority;
|
|
|
import org.elasticsearch.common.Strings;
|
|
|
import org.elasticsearch.common.UUIDs;
|
|
|
import org.elasticsearch.common.ValidationException;
|
|
|
-import org.elasticsearch.common.bytes.BytesReference;
|
|
|
import org.elasticsearch.common.compress.CompressedXContent;
|
|
|
import org.elasticsearch.common.inject.Inject;
|
|
|
import org.elasticsearch.common.logging.DeprecationLogger;
|
|
|
@@ -41,19 +40,17 @@ import org.elasticsearch.core.Nullable;
|
|
|
import org.elasticsearch.core.TimeValue;
|
|
|
import org.elasticsearch.index.Index;
|
|
|
import org.elasticsearch.index.IndexService;
|
|
|
+import org.elasticsearch.index.mapper.DateFieldMapper;
|
|
|
import org.elasticsearch.index.mapper.MapperParsingException;
|
|
|
import org.elasticsearch.index.mapper.MapperService;
|
|
|
import org.elasticsearch.index.mapper.MapperService.MergeReason;
|
|
|
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
|
|
|
import org.elasticsearch.indices.IndexTemplateMissingException;
|
|
|
import org.elasticsearch.indices.IndicesService;
|
|
|
import org.elasticsearch.indices.InvalidIndexTemplateException;
|
|
|
import org.elasticsearch.xcontent.NamedXContentRegistry;
|
|
|
-import org.elasticsearch.xcontent.XContentBuilder;
|
|
|
-import org.elasticsearch.xcontent.XContentFactory;
|
|
|
-import org.elasticsearch.xcontent.XContentType;
|
|
|
|
|
|
import java.io.IOException;
|
|
|
-import java.io.UncheckedIOException;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Arrays;
|
|
|
import java.util.Collections;
|
|
|
@@ -84,23 +81,35 @@ import static org.elasticsearch.indices.cluster.IndicesClusterStateService.Alloc
|
|
|
public class MetadataIndexTemplateService {
|
|
|
|
|
|
public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp";
|
|
|
- public static final String DEFAULT_TIMESTAMP_MAPPING = "{\n"
|
|
|
- + " \"properties\": {\n"
|
|
|
- + " \"@timestamp\": {\n"
|
|
|
- + " \"type\": \"date\"\n"
|
|
|
- + " }\n"
|
|
|
- + " }\n"
|
|
|
- + " }";
|
|
|
- public static final String DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING = "{\n"
|
|
|
- + "\"_routing\" : {\n"
|
|
|
- + " \"required\" : true\n"
|
|
|
- + " },"
|
|
|
- + " \"properties\": {\n"
|
|
|
- + " \"@timestamp\": {\n"
|
|
|
- + " \"type\": \"date\"\n"
|
|
|
- + " }\n"
|
|
|
- + " }\n"
|
|
|
- + " }";
|
|
|
+ private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING;
|
|
|
+
|
|
|
+ private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING;
|
|
|
+
|
|
|
+ static {
|
|
|
+ final Map<String, Map<String, String>> defaultTimestampField = Map.of(
|
|
|
+ DEFAULT_TIMESTAMP_FIELD,
|
|
|
+ Map.of("type", DateFieldMapper.CONTENT_TYPE)
|
|
|
+ );
|
|
|
+ try {
|
|
|
+ DEFAULT_TIMESTAMP_MAPPING = new CompressedXContent(
|
|
|
+ (builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
|
|
|
+ .field("properties", defaultTimestampField)
|
|
|
+ .endObject()
|
|
|
+ );
|
|
|
+ DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING = new CompressedXContent(
|
|
|
+ (builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
|
|
|
+ .startObject(RoutingFieldMapper.NAME)
|
|
|
+ .field("required", true)
|
|
|
+ .endObject()
|
|
|
+ .field("properties")
|
|
|
+ .map(defaultTimestampField)
|
|
|
+ .endObject()
|
|
|
+ );
|
|
|
+ } catch (IOException e) {
|
|
|
+ throw new AssertionError(e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
private static final Logger logger = LogManager.getLogger(MetadataIndexTemplateService.class);
|
|
|
private final ClusterService clusterService;
|
|
|
private final AliasValidator aliasValidator;
|
|
|
@@ -216,7 +225,7 @@ public class MetadataIndexTemplateService {
|
|
|
}
|
|
|
|
|
|
CompressedXContent mappings = template.template().mappings();
|
|
|
- String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
|
|
|
+ CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry);
|
|
|
|
|
|
// We may need to normalize index settings, so do that also
|
|
|
Settings finalSettings = template.template().settings();
|
|
|
@@ -261,18 +270,14 @@ public class MetadataIndexTemplateService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- final Template finalTemplate = new Template(
|
|
|
- finalSettings,
|
|
|
- stringMappings == null ? null : new CompressedXContent(stringMappings),
|
|
|
- template.template().aliases()
|
|
|
- );
|
|
|
+ final Template finalTemplate = new Template(finalSettings, wrappedMappings, template.template().aliases());
|
|
|
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
|
|
|
|
|
|
if (finalComponentTemplate.equals(existing)) {
|
|
|
return currentState;
|
|
|
}
|
|
|
|
|
|
- validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
|
|
|
+ validateTemplate(finalSettings, wrappedMappings, indicesService, xContentRegistry);
|
|
|
validate(name, finalComponentTemplate);
|
|
|
|
|
|
// Validate all composable index templates that use this component template
|
|
|
@@ -319,30 +324,29 @@ public class MetadataIndexTemplateService {
|
|
|
}
|
|
|
|
|
|
@Nullable
|
|
|
- private static String wrapMappingsIfNecessary(@Nullable String mappings, NamedXContentRegistry xContentRegistry) throws Exception {
|
|
|
+ private static CompressedXContent wrapMappingsIfNecessary(@Nullable CompressedXContent mappings, NamedXContentRegistry xContentRegistry)
|
|
|
+ throws IOException {
|
|
|
// Mappings in templates don't have to include _doc, so update
|
|
|
// the mappings to include this single type if necessary
|
|
|
|
|
|
- String stringMappings = mappings;
|
|
|
- if (stringMappings != null) {
|
|
|
- Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
|
|
|
+ CompressedXContent wrapped = mappings;
|
|
|
+ if (wrapped != null) {
|
|
|
+ Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, mappings);
|
|
|
if (parsedMappings.size() > 0) {
|
|
|
if (parsedMappings.size() == 1) {
|
|
|
final String keyName = parsedMappings.keySet().iterator().next();
|
|
|
// Check if it's already wrapped in `_doc`, only rewrap if needed
|
|
|
if (MapperService.SINGLE_MAPPING_NAME.equals(keyName) == false) {
|
|
|
- stringMappings = Strings.toString(
|
|
|
- XContentFactory.jsonBuilder().startObject().field(MapperService.SINGLE_MAPPING_NAME, parsedMappings).endObject()
|
|
|
+ wrapped = new CompressedXContent(
|
|
|
+ (builder, params) -> builder.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
|
|
|
);
|
|
|
}
|
|
|
} else {
|
|
|
- stringMappings = Strings.toString(
|
|
|
- XContentFactory.jsonBuilder().startObject().field(MapperService.SINGLE_MAPPING_NAME, parsedMappings).endObject()
|
|
|
- );
|
|
|
+ wrapped = new CompressedXContent((builder, params) -> builder.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings));
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- return stringMappings;
|
|
|
+ return wrapped;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -430,7 +434,7 @@ public class MetadataIndexTemplateService {
|
|
|
static void validateNotInUse(Metadata metadata, String... templateNameOrWildcard) {
|
|
|
final Predicate<String> predicate;
|
|
|
if (templateNameOrWildcard.length > 1) {
|
|
|
- predicate = name -> Arrays.stream(templateNameOrWildcard).anyMatch(s -> Objects.equals(s, name));
|
|
|
+ predicate = name -> Arrays.asList(templateNameOrWildcard).contains(name);
|
|
|
} else {
|
|
|
predicate = name -> Regex.simpleMatch(templateNameOrWildcard[0], name);
|
|
|
}
|
|
|
@@ -580,13 +584,9 @@ public class MetadataIndexTemplateService {
|
|
|
// If an inner template was specified, its mappings may need to be
|
|
|
// adjusted (to add _doc) and it should be validated
|
|
|
CompressedXContent mappings = innerTemplate.mappings();
|
|
|
- String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
|
|
|
+ CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry);
|
|
|
|
|
|
- final Template finalTemplate = new Template(
|
|
|
- finalSettings,
|
|
|
- stringMappings == null ? null : new CompressedXContent(stringMappings),
|
|
|
- innerTemplate.aliases()
|
|
|
- );
|
|
|
+ final Template finalTemplate = new Template(finalSettings, wrappedMappings, innerTemplate.aliases());
|
|
|
finalIndexTemplate = new ComposableIndexTemplate(
|
|
|
template.indexPatterns(),
|
|
|
finalTemplate,
|
|
|
@@ -1136,12 +1136,8 @@ public class MetadataIndexTemplateService {
|
|
|
/**
|
|
|
* Collect the given v2 template into an ordered list of mappings.
|
|
|
*/
|
|
|
- public static List<CompressedXContent> collectMappings(
|
|
|
- final ClusterState state,
|
|
|
- final String templateName,
|
|
|
- final String indexName,
|
|
|
- final NamedXContentRegistry xContentRegistry
|
|
|
- ) throws Exception {
|
|
|
+ public static List<CompressedXContent> collectMappings(final ClusterState state, final String templateName, final String indexName)
|
|
|
+ throws Exception {
|
|
|
final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
|
|
|
assert template != null
|
|
|
: "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state";
|
|
|
@@ -1150,7 +1146,7 @@ public class MetadataIndexTemplateService {
|
|
|
}
|
|
|
|
|
|
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
|
|
|
- return collectMappings(template, componentTemplates, indexName, xContentRegistry);
|
|
|
+ return collectMappings(template, componentTemplates, indexName);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -1159,8 +1155,7 @@ public class MetadataIndexTemplateService {
|
|
|
public static List<CompressedXContent> collectMappings(
|
|
|
final ComposableIndexTemplate template,
|
|
|
final Map<String, ComponentTemplate> componentTemplates,
|
|
|
- final String indexName,
|
|
|
- final NamedXContentRegistry xContentRegistry
|
|
|
+ final String indexName
|
|
|
) throws Exception {
|
|
|
Objects.requireNonNull(template, "Composable index template must be provided");
|
|
|
List<CompressedXContent> mappings = template.composedOf()
|
|
|
@@ -1177,9 +1172,9 @@ public class MetadataIndexTemplateService {
|
|
|
// add a default mapping for the `@timestamp` field, at the lowest precedence, to make bootstrapping data streams more
|
|
|
// straightforward as all backing indices are required to have a timestamp field
|
|
|
if (template.getDataStreamTemplate().isAllowCustomRouting()) {
|
|
|
- mappings.add(0, new CompressedXContent(wrapMappingsIfNecessary(DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING, xContentRegistry)));
|
|
|
+ mappings.add(0, DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING);
|
|
|
} else {
|
|
|
- mappings.add(0, new CompressedXContent(wrapMappingsIfNecessary(DEFAULT_TIMESTAMP_MAPPING, xContentRegistry)));
|
|
|
+ mappings.add(0, DEFAULT_TIMESTAMP_MAPPING);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -1187,17 +1182,9 @@ public class MetadataIndexTemplateService {
|
|
|
if (indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
|
|
|
// Only if template has data stream definition this should be added and
|
|
|
// adding this template last, since _timestamp field should have highest precedence:
|
|
|
- Optional.ofNullable(template.getDataStreamTemplate())
|
|
|
- .map(ComposableIndexTemplate.DataStreamTemplate::getDataStreamMappingSnippet)
|
|
|
- .map(mapping -> {
|
|
|
- try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
|
|
|
- builder.value(mapping);
|
|
|
- return new CompressedXContent(BytesReference.bytes(builder));
|
|
|
- } catch (IOException e) {
|
|
|
- throw new UncheckedIOException(e);
|
|
|
- }
|
|
|
- })
|
|
|
- .ifPresent(mappings::add);
|
|
|
+ if (template.getDataStreamTemplate() != null) {
|
|
|
+ mappings.add(ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET);
|
|
|
+ }
|
|
|
}
|
|
|
return Collections.unmodifiableList(mappings);
|
|
|
}
|
|
|
@@ -1380,7 +1367,7 @@ public class MetadataIndexTemplateService {
|
|
|
String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName;
|
|
|
// Parse mappings to ensure they are valid after being composed
|
|
|
|
|
|
- List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName, xContentRegistry);
|
|
|
+ List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName);
|
|
|
try {
|
|
|
MapperService mapperService = tempIndexService.mapperService();
|
|
|
for (CompressedXContent mapping : mappings) {
|
|
|
@@ -1399,19 +1386,10 @@ public class MetadataIndexTemplateService {
|
|
|
|
|
|
private static void validateTemplate(
|
|
|
Settings validateSettings,
|
|
|
- String mappings,
|
|
|
+ CompressedXContent mappings,
|
|
|
IndicesService indicesService,
|
|
|
NamedXContentRegistry xContentRegistry
|
|
|
) throws Exception {
|
|
|
- // First check to see if mappings are valid XContent
|
|
|
- if (mappings != null) {
|
|
|
- try {
|
|
|
- new CompressedXContent(mappings);
|
|
|
- } catch (Exception e) {
|
|
|
- throw new MapperParsingException("Failed to parse mapping: {}", e, mappings);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
// Hard to validate settings if they're non-existent, so used empty ones if none were provided
|
|
|
Settings settings = validateSettings;
|
|
|
if (settings == null) {
|
|
|
@@ -1592,7 +1570,7 @@ public class MetadataIndexTemplateService {
|
|
|
Integer version;
|
|
|
List<String> indexPatterns;
|
|
|
Settings settings = Settings.EMPTY;
|
|
|
- String mappings = null;
|
|
|
+ CompressedXContent mappings = null;
|
|
|
List<Alias> aliases = new ArrayList<>();
|
|
|
|
|
|
TimeValue masterTimeout = MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
|
|
|
@@ -1622,7 +1600,7 @@ public class MetadataIndexTemplateService {
|
|
|
return this;
|
|
|
}
|
|
|
|
|
|
- public PutRequest mappings(String mappings) {
|
|
|
+ public PutRequest mappings(CompressedXContent mappings) {
|
|
|
this.mappings = mappings;
|
|
|
return this;
|
|
|
}
|