Add protection in windows for slow file lock releasing (#50884)

This commit adds retries for windows cleanup after tests, which may fail
due to file locks not being immediately released after a windows process
exits.

closes #50825

qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java

@@ -20,10 +20,12 @@
 package org.elasticsearch.packaging.util;
 
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.lsGlob;
@@ -79,13 +81,13 @@ public class Cleanup {
 
         // delete files that may still exist
         lsGlob(getTempDir(), "elasticsearch*").forEach(FileUtils::rm);
-        final List<String> filesToDelete = Platforms.WINDOWS
-            ? ELASTICSEARCH_FILES_WINDOWS
-            : ELASTICSEARCH_FILES_LINUX;
+        final List<String> filesToDelete = Platforms.WINDOWS ? ELASTICSEARCH_FILES_WINDOWS : ELASTICSEARCH_FILES_LINUX;
+        // windows needs leniency due to asinine releasing of file locking async from a process exiting
+        Consumer<? super Path> rm = Platforms.WINDOWS ? FileUtils::rmWithRetries : FileUtils::rm;
         filesToDelete.stream()
             .map(Paths::get)
             .filter(Files::exists)
-            .forEach(FileUtils::rm);
+            .forEach(rm);
 
         // disable elasticsearch service
         // todo add this for windows when adding tests for service intallation

qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java

@@ -84,6 +84,30 @@ public class FileUtils {
         }
     }
 
+    public static void rmWithRetries(Path... paths) {
+        int tries = 10;
+        Exception exception = null;
+        while (tries-- > 0) {
+            try {
+                IOUtils.rm(paths);
+                return;
+            } catch (IOException e) {
+                if (exception == null) {
+                    exception = e;
+                } else {
+                    exception.addSuppressed(e);
+                }
+            }
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException interrupted) {
+                Thread.currentThread().interrupt();
+                return;
+            }
+        }
+        throw new RuntimeException(exception);
+    }
+
     public static Path mktempDir(Path path) {
         try {
             return Files.createTempDirectory(path,"tmp");