|  | @@ -6,7 +6,6 @@
 | 
	
		
			
				|  |  |   */
 | 
	
		
			
				|  |  |  package org.elasticsearch.xpack.core.ml.job.persistence;
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -import org.elasticsearch.Version;
 | 
	
		
			
				|  |  |  import org.elasticsearch.action.ActionListener;
 | 
	
		
			
				|  |  |  import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
 | 
	
		
			
				|  |  |  import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 | 
	
	
		
			
				|  | @@ -19,6 +18,7 @@ import org.elasticsearch.cluster.ClusterState;
 | 
	
		
			
				|  |  |  import org.elasticsearch.cluster.metadata.IndexMetadata;
 | 
	
		
			
				|  |  |  import org.elasticsearch.cluster.metadata.MappingMetadata;
 | 
	
		
			
				|  |  |  import org.elasticsearch.cluster.metadata.Metadata;
 | 
	
		
			
				|  |  | +import org.elasticsearch.common.Strings;
 | 
	
		
			
				|  |  |  import org.elasticsearch.common.settings.Settings;
 | 
	
		
			
				|  |  |  import org.elasticsearch.common.util.concurrent.ThreadContext;
 | 
	
		
			
				|  |  |  import org.elasticsearch.index.IndexVersion;
 | 
	
	
		
			
				|  | @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 | 
	
		
			
				|  |  |  import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 | 
	
		
			
				|  |  |  import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames;
 | 
	
		
			
				|  |  |  import org.elasticsearch.xpack.core.ml.job.results.Result;
 | 
	
		
			
				|  |  | +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
 | 
	
		
			
				|  |  |  import org.junit.Assert;
 | 
	
		
			
				|  |  |  import org.mockito.ArgumentCaptor;
 | 
	
		
			
				|  |  |  
 | 
	
	
		
			
				|  | @@ -126,8 +127,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
 | 
	
		
			
				|  |  |      }
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      public void testMappingRequiresUpdateNoMapping() {
 | 
	
		
			
				|  |  | -        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
 | 
	
		
			
				|  |  | -        ClusterState cs = csBuilder.build();
 | 
	
		
			
				|  |  | +        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
 | 
	
		
			
				|  |  |          String[] indices = new String[] { "no_index" };
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
	
		
			
				|  | @@ -145,86 +145,139 @@ public class ElasticsearchMappingsTests extends ESTestCase {
 | 
	
		
			
				|  |  |          assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  |      }
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    public void testMappingRequiresUpdateRecentMappingVersion() {
 | 
	
		
			
				|  |  | +    /**
 | 
	
		
			
				|  |  | +     * In 8.10 we switched from using the product version to using the per-index mappings
 | 
	
		
			
				|  |  | +     * version to determine whether mappings need updating. So any case of the per-index
 | 
	
		
			
				|  |  | +     * mappings version not being present should result in the mappings being updated.
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    public void testMappingRequiresUpdateOnlyProductVersion() {
 | 
	
		
			
				|  |  | +        int newVersion = randomIntBetween(1, 100);
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_current", Version.CURRENT.toString())
 | 
	
		
			
				|  |  | +                Collections.singletonMap("product_version_only", MlIndexAndAlias.BWC_MAPPINGS_VERSION)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_current" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "product_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        {
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("product_version_only", "8.10.2"));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "product_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        {
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("product_version_only", "7.17.13"));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "product_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        {
 | 
	
		
			
				|  |  | +            // Serverless versions may be build hashes
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("product_version_only", "a1b2c3d4"));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "product_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    /**
 | 
	
		
			
				|  |  | +     * In 8.10 we switched from using the product version to using the per-index mappings
 | 
	
		
			
				|  |  | +     * version to determine whether mappings need updating. The per-index mappings version
 | 
	
		
			
				|  |  | +     * should determine the need for update regardless of the value of the product version
 | 
	
		
			
				|  |  | +     * field.
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    public void testMappingRequiresUpdateGivenProductVersionAndMappingsVersion() {
 | 
	
		
			
				|  |  | +        int currentVersion = randomIntBetween(1, 100);
 | 
	
		
			
				|  |  | +        int newVersion = currentVersion + randomIntBetween(1, 100);
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_current", Version.CURRENT.toString()),
 | 
	
		
			
				|  |  | -                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1)
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", MlIndexAndAlias.BWC_MAPPINGS_VERSION),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_current" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(Strings.EMPTY_ARRAY, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, currentVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_current", Version.CURRENT.toString()),
 | 
	
		
			
				|  |  | -                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1)
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", "8.10.2"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_current" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(Strings.EMPTY_ARRAY, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, currentVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  | -    }
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    public void testMappingRequiresUpdateMaliciousMappingVersion() {
 | 
	
		
			
				|  |  | -        ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -            Collections.singletonMap("version_current", Collections.singletonMap("nested", "1.0"))
 | 
	
		
			
				|  |  | -        );
 | 
	
		
			
				|  |  | -        String[] indices = new String[] { "version_nested" };
 | 
	
		
			
				|  |  | -        assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | -    }
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    public void testMappingRequiresUpdateBogusMappingVersion() {
 | 
	
		
			
				|  |  | -        ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0"));
 | 
	
		
			
				|  |  | -        String[] indices = new String[] { "version_bogus" };
 | 
	
		
			
				|  |  | -        assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | -    }
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    public void testMappingRequiresUpdateNewerMappingVersion() {
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  | -            ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", 2));
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_newer" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            // Serverless versions may be build hashes
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", "a1b2c3d4"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  | +            );
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(Strings.EMPTY_ARRAY, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, currentVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_newer", Version.CURRENT),
 | 
	
		
			
				|  |  | -                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 2)
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", MlIndexAndAlias.BWC_MAPPINGS_VERSION),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_newer" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_newer", Version.CURRENT),
 | 
	
		
			
				|  |  | -                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1)
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", "8.10.2"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_newer" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        {
 | 
	
		
			
				|  |  | +            // Serverless versions may be build hashes
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | +                Collections.singletonMap("both", "a1b2c3d4"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  | +            );
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "both" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |      }
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    public void testMappingRequiresUpdateNewerMappingVersionMinor() {
 | 
	
		
			
				|  |  | +    /**
 | 
	
		
			
				|  |  | +     * In 8.10 we switched from using the product version to using the per-index mappings
 | 
	
		
			
				|  |  | +     * version to determine whether mappings need updating. The per-index mappings version
 | 
	
		
			
				|  |  | +     * should determine the need for update even if the product version field is not present.
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    public void testMappingRequiresUpdateGivenOnlyMappingsVersion() {
 | 
	
		
			
				|  |  | +        int currentVersion = randomIntBetween(1, 100);
 | 
	
		
			
				|  |  | +        int newVersion = currentVersion + randomIntBetween(1, 100);
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  | -            ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", 1));
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_newer_minor" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | +                Collections.singletonMap("mappings_version_only", "NO_VERSION_FIELD"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  | +            );
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "mappings_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(Strings.EMPTY_ARRAY, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, currentVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |          {
 | 
	
		
			
				|  |  |              ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | -                Collections.singletonMap("version_newer_minor", Version.CURRENT),
 | 
	
		
			
				|  |  | -                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1)
 | 
	
		
			
				|  |  | +                Collections.singletonMap("mappings_version_only", "NO_VERSION_FIELD"),
 | 
	
		
			
				|  |  | +                Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, currentVersion)
 | 
	
		
			
				|  |  |              );
 | 
	
		
			
				|  |  | -            String[] indices = new String[] { "version_newer_minor" };
 | 
	
		
			
				|  |  | -            assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +            String[] indices = new String[] { "mappings_version_only" };
 | 
	
		
			
				|  |  | +            assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, newVersion));
 | 
	
		
			
				|  |  |          }
 | 
	
		
			
				|  |  |      }
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +    public void testMappingRequiresUpdateMaliciousMappingVersion() {
 | 
	
		
			
				|  |  | +        ClusterState cs = getClusterStateWithMappingsWithMetadata(
 | 
	
		
			
				|  |  | +            Collections.singletonMap("version_nested", Collections.singletonMap("nested", "1.0"))
 | 
	
		
			
				|  |  | +        );
 | 
	
		
			
				|  |  | +        String[] indices = new String[] { "version_nested" };
 | 
	
		
			
				|  |  | +        assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    public void testMappingRequiresUpdateBogusMappingVersion() {
 | 
	
		
			
				|  |  | +        ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0"));
 | 
	
		
			
				|  |  | +        String[] indices = new String[] { "version_bogus" };
 | 
	
		
			
				|  |  | +        assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1));
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |      @SuppressWarnings({ "unchecked" })
 | 
	
		
			
				|  |  |      public void testAddDocMappingIfMissing() {
 | 
	
		
			
				|  |  |          ThreadPool threadPool = mock(ThreadPool.class);
 | 
	
	
		
			
				|  | @@ -287,7 +340,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
 | 
	
		
			
				|  |  |                  meta.put("version", version);
 | 
	
		
			
				|  |  |              }
 | 
	
		
			
				|  |  |              if (metaData != null) {
 | 
	
		
			
				|  |  | -                metaData.forEach((k, v) -> meta.putIfAbsent(k, v));
 | 
	
		
			
				|  |  | +                metaData.forEach(meta::putIfAbsent);
 | 
	
		
			
				|  |  |              }
 | 
	
		
			
				|  |  |              mapping.put("_meta", meta);
 | 
	
		
			
				|  |  |  
 |