Fail node containing ancient closed index (#44264)

Today we fail the node at startup if it contains an index that is too old to be
compatible with the current version, unless that index is closed. If the index
is closed then the node will start up and this puts us into a bad state: the
index cannot be opened and must be reindexed using an earlier version, but we
offer no way to get that index into a node running an earlier version so that
it can be reindexed. Downgrading the node in-place is decidedly unsupported and
cannot be expected to work since the node already started up and upgraded the
rest of its metadata. Since #41731 we actively reject downgrades to versions ≥
v7.2.0 too.

This commit prevents the node from starting in the presence of any too-old
indices (closed or not). In particular, it does not write any upgraded metadata
in this situation, increasing the chances an in-place downgrade might be
successful. We still actively reject the downgrade using #41731, because we
wrote the node metadata file before checking the index metadata, but at least
there is a way to override this check.

Relates #21830, #44230

docs/reference/migration/migrate_8_0/node.asciidoc

@@ -35,4 +35,12 @@ each of these folders to an appropriate location and then configure the
 corresponding node to use this location for its data path. If your nodes each
 have more than one data path in their `path.data` settings then you should move
 all the corresponding subfolders in parallel. Each node uses the same subfolder
-(e.g. `nodes/2`) across all its data paths.
+(e.g. `nodes/2`) across all its data paths.
+
+[float]
+==== Rejection of ancient closed indices
+
+In earlier versions a node would start up even if it had data from indices
+created in a version before the previous major version, as long as those
+indices were closed. {es} now ensures that it is compatible with every index,
+open or closed, at startup time.

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -115,18 +115,15 @@ public class MetaDataIndexUpgradeService {
     }
 
     /**
-     * Elasticsearch v6.0 no longer supports indices created pre v5.0. All indices
-     * that were created before Elasticsearch v5.0 should be re-indexed in Elasticsearch 5.x
-     * before they can be opened by this version of elasticsearch.
+     * Elasticsearch does not support indices created before the previous major version. They must be reindexed using an earlier version
+     * before they can be opened here.
      */
     private void checkSupportedVersion(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
-        if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData,
-            minimumIndexCompatibilityVersion) == false) {
-            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created with version ["
+        if (isSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion) == false) {
+            throw new IllegalStateException("The index " + indexMetaData.getIndex() + " was created with version ["
                 + indexMetaData.getCreationVersion() + "] but the minimum compatible version is ["
-
-                + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch " + minimumIndexCompatibilityVersion.major
-                + ".x before upgrading to " + Version.CURRENT + ".");
+                + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch "
+                + minimumIndexCompatibilityVersion.major + ".x before upgrading to " + Version.CURRENT + ".");
         }
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -106,9 +107,9 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
             .build());
         String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(metaData,
             Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
-        assertEquals(message, "The index [[foo/BOOM]] was created with version [" + indexCreated + "] " +
+        assertThat(message, equalTo("The index [foo/" + metaData.getIndexUUID() + "] was created with version [" + indexCreated + "] " +
              "but the minimum compatible version is [" + minCompat + "]." +
-            " It should be re-indexed in Elasticsearch " + minCompat.major + ".x before upgrading to " + Version.CURRENT.toString() + ".");
+            " It should be re-indexed in Elasticsearch " + minCompat.major + ".x before upgrading to " + Version.CURRENT.toString() + "."));
 
         indexCreated = VersionUtils.randomVersionBetween(random(), minCompat, Version.CURRENT);
         indexUpgraded = VersionUtils.randomVersionBetween(random(), indexCreated, Version.CURRENT);
@@ -160,15 +161,25 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
     }
 
     public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
-        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_CREATION_DATE, 1)
-            .put(IndexMetaData.SETTING_INDEX_UUID, "BOOM")
-            .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT.minimumIndexCompatibilityVersion())
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, randomEarlierCompatibleVersion())
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, between(0, 5))
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 5))
+            .put(IndexMetaData.SETTING_CREATION_DATE, randomNonNegativeLong())
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, randomEarlierCompatibleVersion())
             .put(indexSettings)
             .build();
-        return IndexMetaData.builder(name).settings(build).build();
+        final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(name).settings(settings);
+        if (randomBoolean()) {
+            indexMetaDataBuilder.state(IndexMetaData.State.CLOSE);
+        }
+        return indexMetaDataBuilder.build();
+    }
+
+    private static Version randomEarlierCompatibleVersion() {
+        return randomValueOtherThan(Version.CURRENT, () -> VersionUtils.randomVersionBetween(random(),
+            Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT));
     }
 
 }