Upgrade folders after settings validation (#87319)

When upgrading from 7.x to 8.x, we remove the `nodes/0` subfolders from the
data path. This is an irreversible change and once done, downgrade is not
possible. Now do settings validation prior to moving folders to ensure that
a bad setting does not prevent a downgrade.

docs/changelog/87319.yaml

@@ -0,0 +1,5 @@
+pr: 87319
+summary: Upgrade folders after settings validation
+area: Infra/Node Lifecycle
+type: bug
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java

@@ -224,6 +224,13 @@ public class NodeEnvironmentIT extends ESIntegTestCase {
             Files.createFile(sharedCache);
         }
 
+        // check that settings are validated prior to moving folders
+        dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes"))));
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> internalCluster().startNode(Settings.builder().put(dataPathSettings).put("bad", "setting"))
+        );
+
         // check that upgrade works
         dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes"))));
         internalCluster().startNode(dataPathSettings);

server/src/main/java/org/elasticsearch/node/Node.java

@@ -394,19 +394,6 @@ public class Node implements Closeable {
              */
             this.environment = new Environment(settings, initialEnvironment.configFile());
             Environment.assertEquivalent(initialEnvironment, this.environment);
-            nodeEnvironment = new NodeEnvironment(tmpSettings, environment);
-            logger.info(
-                "node name [{}], node ID [{}], cluster name [{}], roles {}",
-                NODE_NAME_SETTING.get(tmpSettings),
-                nodeEnvironment.nodeId(),
-                ClusterName.CLUSTER_NAME_SETTING.get(tmpSettings).value(),
-                DiscoveryNode.getRolesFromSettings(settings)
-                    .stream()
-                    .map(DiscoveryNodeRole::roleName)
-                    .collect(Collectors.toCollection(LinkedHashSet::new))
-            );
-            resourcesToClose.add(nodeEnvironment);
-            localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
 
             final List<ExecutorBuilder<?>> executorBuilders = pluginsService.flatMap(p -> p.getExecutorBuilders(settings)).toList();
 
@@ -445,6 +432,23 @@ public class Node implements Closeable {
                 pluginsService.flatMap(Plugin::getSettingsFilter).toList(),
                 settingsUpgraders
             );
+
+            // creating `NodeEnvironment` breaks the ability to rollback to 7.x on an 8.0 upgrade (`upgradeLegacyNodeFolders`) so do this
+            // after settings validation.
+            nodeEnvironment = new NodeEnvironment(tmpSettings, environment);
+            logger.info(
+                "node name [{}], node ID [{}], cluster name [{}], roles {}",
+                NODE_NAME_SETTING.get(tmpSettings),
+                nodeEnvironment.nodeId(),
+                ClusterName.CLUSTER_NAME_SETTING.get(tmpSettings).value(),
+                DiscoveryNode.getRolesFromSettings(settings)
+                    .stream()
+                    .map(DiscoveryNodeRole::roleName)
+                    .collect(Collectors.toCollection(LinkedHashSet::new))
+            );
+            resourcesToClose.add(nodeEnvironment);
+            localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
+
             ScriptModule.registerClusterSettingsListeners(scriptService, settingsModule.getClusterSettings());
             final NetworkService networkService = new NetworkService(
                 getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class))