Move preallocate functionality to native access (#110678)

This commit moves the file preallocation functionality into
NativeAccess. The code is basically the same. One small tweak is that
instead of breaking Java access boundaries in order to get an open file
handle, the new code uses posix open directly.

relates #104876

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
         "org.elasticsearch.pluginclassloader",
-        "org.elasticsearch.preallocate",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

distribution/build.gradle

@@ -280,8 +280,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   dependencies {
     libs project(':server')
-    // this is a special circumstance of a jar that is not a dependency of server, but needs to be in the module path
-    libs project(':libs:elasticsearch-preallocate')
 
     libsVersionChecker project(':distribution:tools:java-version-checker')
     libsCliLauncher project(':distribution:tools:cli-launcher')

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -69,11 +69,6 @@ final class SystemJvmOptions {
                  * parsing will break in an incompatible way for some date patterns and locales.
                  */
                 "-Djava.locale.providers=SPI,COMPAT",
-                /*
-                 * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary
-                 * while we explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
-                 */
-                "--add-opens=java.base/java.io=org.elasticsearch.preallocate",
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),

libs/native/jna/src/main/java/module-info.java

@@ -14,6 +14,7 @@ module org.elasticsearch.nativeaccess.jna {
     requires org.elasticsearch.nativeaccess;
     requires org.elasticsearch.logging;
     requires com.sun.jna;
+    requires java.desktop;
 
     exports org.elasticsearch.nativeaccess.jna to com.sun.jna;
 

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java

@@ -15,12 +15,14 @@ import com.sun.jna.Pointer;
 import com.sun.jna.Structure;
 import com.sun.jna.Structure.ByReference;
 import com.sun.jna.WString;
+import com.sun.jna.ptr.IntByReference;
 import com.sun.jna.win32.StdCallLibrary;
 
 import org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler;
 import org.elasticsearch.nativeaccess.lib.Kernel32Library;
 
 import java.util.List;
+import java.util.function.IntConsumer;
 
 class JnaKernel32Library implements Kernel32Library {
     private static class JnaHandle implements Handle {
@@ -158,6 +160,8 @@ class JnaKernel32Library implements Kernel32Library {
 
         boolean SetProcessWorkingSetSize(Pointer handle, SizeT minSize, SizeT maxSize);
 
+        int GetCompressedFileSizeW(WString lpFileName, IntByReference lpFileSizeHigh);
+
         int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer);
 
         boolean SetConsoleCtrlHandler(StdCallLibrary.StdCallCallback handler, boolean add);
@@ -232,6 +236,15 @@ class JnaKernel32Library implements Kernel32Library {
         return functions.SetProcessWorkingSetSize(jnaHandle.pointer, new SizeT(minSize), new SizeT(maxSize));
     }
 
+    @Override
+    public int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh) {
+        var wideFileName = new WString(lpFileName);
+        var fileSizeHigh = new IntByReference();
+        int ret = functions.GetCompressedFileSizeW(wideFileName, fileSizeHigh);
+        lpFileSizeHigh.accept(fileSizeHigh.getValue());
+        return ret;
+    }
+
     @Override
     public int GetShortPathNameW(String lpszLongPath, char[] lpszShortPath, int cchBuffer) {
         var wideFileName = new WString(lpszLongPath);

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java

@@ -60,6 +60,8 @@ class JnaLinuxCLibrary implements LinuxCLibrary {
          * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing
          */
         NativeLong syscall(NativeLong number, Object... args);
+
+        int fallocate(int fd, int mode, long offset, long length);
     }
 
     private final NativeFunctions functions;
@@ -91,4 +93,9 @@ class JnaLinuxCLibrary implements LinuxCLibrary {
     public long syscall(long number, int operation, int flags, long address) {
         return functions.syscall(new NativeLong(number), operation, flags, address).longValue();
     }
+
+    @Override
+    public int fallocate(int fd, int mode, long offset, long length) {
+        return functions.fallocate(fd, mode, offset, length);
+    }
 }

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java

@@ -9,8 +9,10 @@
 package org.elasticsearch.nativeaccess.jna;
 
 import com.sun.jna.Library;
+import com.sun.jna.Memory;
 import com.sun.jna.Native;
 import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
 import com.sun.jna.Structure;
 
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
@@ -51,37 +53,58 @@ class JnaPosixCLibrary implements PosixCLibrary {
         }
     }
 
-    public static class JnaFStore extends Structure implements Structure.ByReference, FStore {
+    public static final class JnaStat64 implements Stat64 {
+        final Memory memory;
+        private final int stSizeOffset;
+        private final int stBlocksOffset;
 
-        public int fst_flags = 0;
-        public int fst_posmode = 0;
-        public NativeLong fst_offset = new NativeLong(0);
-        public NativeLong fst_length = new NativeLong(0);
-        public NativeLong fst_bytesalloc = new NativeLong(0);
+        JnaStat64(int sizeof, int stSizeOffset, int stBlocksOffset) {
+            this.memory = new Memory(sizeof);
+            this.stSizeOffset = stSizeOffset;
+            this.stBlocksOffset = stBlocksOffset;
+        }
+
+        @Override
+        public long st_size() {
+            return memory.getLong(stSizeOffset);
+        }
+
+        @Override
+        public long st_blocks() {
+            return memory.getLong(stBlocksOffset);
+        }
+    }
+
+    public static class JnaFStore implements FStore {
+        final Memory memory;
+
+        JnaFStore() {
+            this.memory = new Memory(32);
+        }
 
         @Override
         public void set_flags(int flags) {
-            this.fst_flags = flags;
+            memory.setInt(0, flags);
         }
 
         @Override
         public void set_posmode(int posmode) {
-            this.fst_posmode = posmode;
+            memory.setInt(4, posmode);
         }
 
         @Override
         public void set_offset(long offset) {
-            fst_offset.setValue(offset);
+            memory.setLong(8, offset);
         }
 
         @Override
         public void set_length(long length) {
-            fst_length.setValue(length);
+            memory.setLong(16, length);
         }
 
         @Override
         public long bytesalloc() {
-            return fst_bytesalloc.longValue();
+            return memory.getLong(24);
         }
     }
 
@@ -94,15 +117,40 @@ class JnaPosixCLibrary implements PosixCLibrary {
 
         int mlockall(int flags);
 
-        int fcntl(int fd, int cmd, JnaFStore fst);
+        int fcntl(int fd, int cmd, Pointer fst);
+
+        int ftruncate(int fd, NativeLong length);
+
+        int open(String filename, int flags, Object... mode);
+
+        int close(int fd);
 
         String strerror(int errno);
     }
 
+    private interface FStat64Function extends Library {
+        int fstat64(int fd, Pointer stat);
+    }
+
+    private interface FXStatFunction extends Library {
+        int __fxstat(int version, int fd, Pointer stat);
+    }
+
     private final NativeFunctions functions;
+    private final FStat64Function fstat64;
 
     JnaPosixCLibrary() {
         this.functions = Native.load("c", NativeFunctions.class);
+        FStat64Function fstat64;
+        try {
+            fstat64 = Native.load("c", FStat64Function.class);
+        } catch (UnsatisfiedLinkError e) {
+            // TODO: explain
+            var fxstat = Native.load("c", FXStatFunction.class);
+            int version = System.getProperty("os.arch").equals("aarch64") ? 0 : 1;
+            fstat64 = (fd, stat) -> fxstat.__fxstat(version, fd, stat);
+        }
+        this.fstat64 = fstat64;
     }
 
     @Override
@@ -115,6 +163,11 @@ class JnaPosixCLibrary implements PosixCLibrary {
         return new JnaRLimit();
     }
 
+    @Override
+    public Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset) {
+        return new JnaStat64(sizeof, stSizeOffset, stBlocksOffset);
+    }
+
     @Override
     public int getrlimit(int resource, RLimit rlimit) {
         assert rlimit instanceof JnaRLimit;
@@ -143,7 +196,34 @@ class JnaPosixCLibrary implements PosixCLibrary {
     public int fcntl(int fd, int cmd, FStore fst) {
         assert fst instanceof JnaFStore;
         var jnaFst = (JnaFStore) fst;
-        return functions.fcntl(fd, cmd, jnaFst);
+        return functions.fcntl(fd, cmd, jnaFst.memory);
+    }
+
+    @Override
+    public int ftruncate(int fd, long length) {
+        return functions.ftruncate(fd, new NativeLong(length));
+    }
+
+    @Override
+    public int open(String pathname, int flags) {
+        return functions.open(pathname, flags);
+    }
+
+    @Override
+    public int open(String pathname, int flags, int mode) {
+        return functions.open(pathname, flags, mode);
+    }
+
+    @Override
+    public int close(int fd) {
+        return functions.close(fd);
+    }
+
+    @Override
+    public int fstat64(int fd, Stat64 stats) {
+        assert stats instanceof JnaStat64;
+        var jnaStats = (JnaStat64) stats;
+        return fstat64.fstat64(fd, jnaStats.memory);
     }
 
     @Override

libs/native/src/main/java/module-info.java

@@ -19,6 +19,7 @@ module org.elasticsearch.nativeaccess {
         to
             org.elasticsearch.nativeaccess.jna,
             org.elasticsearch.server,
+            org.elasticsearch.blobcache,
             org.elasticsearch.simdvec,
             org.elasticsearch.systemd;
     // allows jna to implement a library provider, and ProviderLocator to load it

libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java

@@ -18,6 +18,8 @@ import java.util.Map;
 
 class LinuxNativeAccess extends PosixNativeAccess {
 
+    private static final int STATX_BLOCKS = 0x400; /* Want/got stx_blocks */
+
     /** the preferred method is seccomp(2), since we can apply to all threads of the process */
     static final int SECCOMP_SET_MODE_FILTER = 1;   // since Linux 3.17
     static final int SECCOMP_FILTER_FLAG_TSYNC = 1;   // since Linux 3.17
@@ -88,7 +90,7 @@ class LinuxNativeAccess extends PosixNativeAccess {
     private final Systemd systemd;
 
     LinuxNativeAccess(NativeLibraryProvider libraryProvider) {
-        super("Linux", libraryProvider, new PosixConstants(-1L, 9, 1, 8));
+        super("Linux", libraryProvider, new PosixConstants(-1L, 9, 1, 8, 64, 144, 48, 64));
         this.linuxLibc = libraryProvider.getLibrary(LinuxCLibrary.class);
         this.systemd = new Systemd(libraryProvider.getLibrary(SystemdLibrary.class));
     }
@@ -120,6 +122,16 @@ class LinuxNativeAccess extends PosixNativeAccess {
         logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect.");
     }
 
+    @Override
+    protected boolean nativePreallocate(int fd, long currentSize, long newSize) {
+        final int rc = linuxLibc.fallocate(fd, 0, currentSize, newSize - currentSize);
+        if (rc != 0) {
+            logger.warn("fallocate failed: " + libc.strerror(libc.errno()));
+            return false;
+        }
+        return true;
+    }
+
     /**
      * Installs exec system call filtering for Linux.
      * <p>

libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java

@@ -22,6 +22,11 @@ import java.util.Collections;
 
 class MacNativeAccess extends PosixNativeAccess {
 
+    private static final int F_PREALLOCATE = 42;
+    private static final int F_ALLOCATECONTIG = 0x2; // allocate contiguous space
+    private static final int F_ALLOCATEALL = 0x4; // allocate all the requested space or no space at all
+    private static final int F_PEOFPOSMODE = 3; // allocate from the physical end of the file
+
     /** The only supported flag... */
     static final int SANDBOX_NAMED = 1;
     /** Allow everything except process fork and execution */
@@ -30,7 +35,7 @@ class MacNativeAccess extends PosixNativeAccess {
     private final MacCLibrary macLibc;
 
     MacNativeAccess(NativeLibraryProvider libraryProvider) {
-        super("MacOS", libraryProvider, new PosixConstants(9223372036854775807L, 5, 1, 6));
+        super("MacOS", libraryProvider, new PosixConstants(9223372036854775807L, 5, 1, 6, 512, 144, 96, 104));
         this.macLibc = libraryProvider.getLibrary(MacCLibrary.class);
     }
 
@@ -44,6 +49,31 @@ class MacNativeAccess extends PosixNativeAccess {
         // we don't have instructions for macos
     }
 
+    @Override
+    protected boolean nativePreallocate(int fd, long currentSize, long newSize) {
+        var fst = libc.newFStore();
+        fst.set_flags(F_ALLOCATECONTIG);
+        fst.set_posmode(F_PEOFPOSMODE);
+        fst.set_offset(0);
+        fst.set_length(newSize);
+        // first, try allocating contiguously
+        if (libc.fcntl(fd, F_PREALLOCATE, fst) != 0) {
+            // TODO: log warning?
+            // that failed, so let us try allocating non-contiguously
+            fst.set_flags(F_ALLOCATEALL);
+            if (libc.fcntl(fd, F_PREALLOCATE, fst) != 0) {
+                // i'm afraid captain dale had to bail
+                logger.warn("Could not allocate non-contiguous size: " + libc.strerror(libc.errno()));
+                return false;
+            }
+        }
+        if (libc.ftruncate(fd, newSize) != 0) {
+            logger.warn("Could not truncate file: " + libc.strerror(libc.errno()));
+            return false;
+        }
+        return true;
+    }
+
     /**
      * Installs exec system call filtering on MacOS.
      * <p>

libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java

@@ -8,7 +8,9 @@
 
 package org.elasticsearch.nativeaccess;
 
+import java.nio.file.Path;
 import java.util.Optional;
+import java.util.OptionalLong;
 
 /**
  * Provides access to native functionality needed by Elastisearch.
@@ -62,6 +64,16 @@ public interface NativeAccess {
      */
     Zstd getZstd();
 
+    /**
+     * Retrieves the actual number of bytes of disk storage used to store a specified file.
+     *
+     * @param path the path to the file
+     * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid
+     */
+    OptionalLong allocatedSizeInBytes(Path path);
+
+    void tryPreallocate(Path file, long size);
+
     /**
      * Returns an accessor for native functions only available on Windows, or {@code null} if not on Windows.
      */

libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java

@@ -11,7 +11,9 @@ package org.elasticsearch.nativeaccess;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
+import java.nio.file.Path;
 import java.util.Optional;
+import java.util.OptionalLong;
 
 class NoopNativeAccess implements NativeAccess {
 
@@ -51,6 +53,17 @@ class NoopNativeAccess implements NativeAccess {
         return ExecSandboxState.NONE;
     }
 
+    @Override
+    public OptionalLong allocatedSizeInBytes(Path path) {
+        logger.warn("Cannot get allocated size of file [" + path + "] because native access is not available");
+        return OptionalLong.empty();
+    }
+
+    @Override
+    public void tryPreallocate(Path file, long size) {
+        logger.warn("Cannot preallocate file size because native access is not available");
+    }
+
     @Override
     public Systemd systemd() {
         logger.warn("Cannot get systemd access because native access is not available");

libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java

@@ -11,4 +11,13 @@ package org.elasticsearch.nativeaccess;
 /**
  * Code constants on POSIX systems.
  */
-record PosixConstants(long RLIMIT_INFINITY, int RLIMIT_AS, int RLIMIT_FSIZE, int RLIMIT_MEMLOCK) {}
+record PosixConstants(
+    long RLIMIT_INFINITY,
+    int RLIMIT_AS,
+    int RLIMIT_FSIZE,
+    int RLIMIT_MEMLOCK,
+    int O_CREAT,
+    int statStructSize,
+    int statStructSizeOffset,
+    int statStructBlocksOffset
+) {}

libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java

@@ -12,12 +12,17 @@ import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 import org.elasticsearch.nativeaccess.lib.VectorLibrary;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Optional;
+import java.util.OptionalLong;
 
 abstract class PosixNativeAccess extends AbstractNativeAccess {
 
     public static final int MCL_CURRENT = 1;
     public static final int ENOMEM = 12;
+    public static final int O_RDONLY = 0;
+    public static final int O_WRONLY = 1;
 
     protected final PosixCLibrary libc;
     protected final VectorSimilarityFunctions vectorDistance;
@@ -121,6 +126,52 @@ abstract class PosixNativeAccess extends AbstractNativeAccess {
 
     protected abstract void logMemoryLimitInstructions();
 
+    @Override
+    public OptionalLong allocatedSizeInBytes(Path path) {
+        assert Files.isRegularFile(path) : path;
+        var stats = libc.newStat64(constants.statStructSize(), constants.statStructSizeOffset(), constants.statStructBlocksOffset());
+
+        int fd = libc.open(path.toAbsolutePath().toString(), O_RDONLY);
+        if (fd == -1) {
+            logger.warn("Could not open file [" + path + "] to get allocated size: " + libc.strerror(libc.errno()));
+            return OptionalLong.empty();
+        }
+
+        if (libc.fstat64(fd, stats) != 0) {
+            logger.warn("Could not get stats for file [" + path + "] to get allocated size: " + libc.strerror(libc.errno()));
+            return OptionalLong.empty();
+        }
+        if (libc.close(fd) != 0) {
+            logger.warn("Failed to close file [" + path + "] after getting stats: " + libc.strerror(libc.errno()));
+        }
+        return OptionalLong.of(stats.st_blocks() * 512);
+    }
+
+    @Override
+    public void tryPreallocate(Path file, long newSize) {
+        // get fd and current size, then pass to OS variant
+        int fd = libc.open(file.toAbsolutePath().toString(), O_WRONLY, constants.O_CREAT());
+        if (fd == -1) {
+            logger.warn("Could not open file [" + file + "] to preallocate size: " + libc.strerror(libc.errno()));
+            return;
+        }
+
+        var stats = libc.newStat64(constants.statStructSize(), constants.statStructSizeOffset(), constants.statStructBlocksOffset());
+        if (libc.fstat64(fd, stats) != 0) {
+            logger.warn("Could not get stats for file [" + file + "] to preallocate size: " + libc.strerror(libc.errno()));
+        } else {
+            if (nativePreallocate(fd, stats.st_size(), newSize)) {
+                logger.debug("pre-allocated file [{}] to {} bytes", file, newSize);
+            } // OS specific preallocate logs its own errors
+        }
+
+        if (libc.close(fd) != 0) {
+            logger.warn("Could not close file [" + file + "] after trying to preallocate size: " + libc.strerror(libc.errno()));
+        }
+    }
+
+    protected abstract boolean nativePreallocate(int fd, long currentSize, long newSize);
+
     @Override
     public Optional<VectorSimilarityFunctions> getVectorSimilarityFunctions() {
         return Optional.ofNullable(vectorDistance);

libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java

@@ -12,7 +12,11 @@ import org.elasticsearch.nativeaccess.lib.Kernel32Library;
 import org.elasticsearch.nativeaccess.lib.Kernel32Library.Handle;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.lang.management.ManagementFactory.getMemoryMXBean;
 
@@ -27,6 +31,8 @@ class WindowsNativeAccess extends AbstractNativeAccess {
     public static final int PAGE_GUARD = 0x0100;
     public static final int MEM_COMMIT = 0x1000;
 
+    private static final int INVALID_FILE_SIZE = -1;
+
     /**
      * Constant for JOBOBJECT_BASIC_LIMIT_INFORMATION in Query/Set InformationJobObject
      */
@@ -119,6 +125,37 @@ class WindowsNativeAccess extends AbstractNativeAccess {
         logger.debug("Windows ActiveProcessLimit initialization successful");
     }
 
+    @Override
+    public OptionalLong allocatedSizeInBytes(Path path) {
+        assert Files.isRegularFile(path) : path;
+        String fileName = "\\\\?\\" + path;
+        AtomicInteger lpFileSizeHigh = new AtomicInteger();
+
+        final int lpFileSizeLow = kernel.GetCompressedFileSizeW(fileName, lpFileSizeHigh::set);
+        if (lpFileSizeLow == INVALID_FILE_SIZE) {
+            logger.warn("Unable to get allocated size of file [{}]. Error code {}", path, kernel.GetLastError());
+            return OptionalLong.empty();
+        }
+
+        // convert lpFileSizeLow to unsigned long and combine with signed/shifted lpFileSizeHigh
+        final long allocatedSize = (((long) lpFileSizeHigh.get()) << Integer.SIZE) | Integer.toUnsignedLong(lpFileSizeLow);
+        if (logger.isTraceEnabled()) {
+            logger.trace(
+                "executing native method GetCompressedFileSizeW returned [high={}, low={}, allocated={}] for file [{}]",
+                lpFileSizeHigh.get(),
+                lpFileSizeLow,
+                allocatedSize,
+                path
+            );
+        }
+        return OptionalLong.of(allocatedSize);
+    }
+
+    @Override
+    public void tryPreallocate(Path file, long size) {
+        logger.warn("Cannot preallocate file size because operation is not available on Windows");
+    }
+
     @Override
     public ProcessLimits getProcessLimits() {
         return new ProcessLimits(ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN);

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java

@@ -10,6 +10,8 @@ package org.elasticsearch.nativeaccess.lib;
 
 import org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler;
 
+import java.util.function.IntConsumer;
+
 public non-sealed interface Kernel32Library extends NativeLibrary {
     interface Handle {}
 
@@ -81,6 +83,17 @@ public non-sealed interface Kernel32Library extends NativeLibrary {
      */
     boolean SetProcessWorkingSetSize(Handle handle, long minSize, long maxSize);
 
+    /**
+     * Retrieves the actual number of bytes of disk storage used to store a specified file.
+     *
+     * https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getcompressedfilesizew
+     *
+     * @param lpFileName the path string
+     * @param lpFileSizeHigh pointer to high-order DWORD for compressed file size (or null if not needed)
+     * @return the low-order DWORD for compressed file size
+     */
+    int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh);
+
     /**
      * Retrieves the short path form of the specified path.
      *

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java

@@ -35,4 +35,6 @@ public non-sealed interface LinuxCLibrary extends NativeLibrary {
      * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing
      */
     long syscall(long number, int operation, int flags, long address);
+
+    int fallocate(int fd, int mode, long offset, long length);
 }

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java

@@ -55,6 +55,25 @@ public non-sealed interface PosixCLibrary extends NativeLibrary {
      */
     int mlockall(int flags);
 
+    /** corresponds to struct stat64 */
+    interface Stat64 {
+        long st_size();
+
+        long st_blocks();
+    }
+
+    Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset);
+
+    int open(String pathname, int flags, int mode);
+
+    int open(String pathname, int flags);
+
+    int close(int fd);
+
+    int fstat64(int fd, Stat64 stats);
+
+    int ftruncate(int fd, long length);
+
     interface FStore {
         void set_flags(int flags); /* IN: flags word */
 

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java

@@ -20,6 +20,7 @@ import java.lang.foreign.StructLayout;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.VarHandle;
 import java.nio.charset.StandardCharsets;
+import java.util.function.IntConsumer;
 
 import static java.lang.foreign.MemoryLayout.PathElement.groupElement;
 import static java.lang.foreign.MemoryLayout.paddingLayout;
@@ -57,6 +58,10 @@ class JdkKernel32Library implements Kernel32Library {
         "SetProcessWorkingSetSize",
         FunctionDescriptor.of(ADDRESS, JAVA_LONG, JAVA_LONG)
     );
+    private static final MethodHandle GetCompressedFileSizeW$mh = downcallHandleWithError(
+        "GetCompressedFileSizeW",
+        FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS)
+    );
     private static final MethodHandle GetShortPathNameW$mh = downcallHandleWithError(
         "GetShortPathNameW",
         FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT)
@@ -276,6 +281,20 @@ class JdkKernel32Library implements Kernel32Library {
         }
     }
 
+    @Override
+    public int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh) {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment wideFileName = ArenaUtil.allocateFrom(arena, lpFileName + "\0", StandardCharsets.UTF_16LE);
+            MemorySegment fileSizeHigh = arena.allocate(JAVA_INT);
+
+            int ret = (int) GetCompressedFileSizeW$mh.invokeExact(lastErrorState, wideFileName, fileSizeHigh);
+            lpFileSizeHigh.accept(fileSizeHigh.get(JAVA_INT, 0));
+            return ret;
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
     @Override
     public int GetShortPathNameW(String lpszLongPath, char[] lpszShortPath, int cchBuffer) {
         try (Arena arena = Arena.ofConfined()) {

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java

@@ -49,6 +49,10 @@ class JdkLinuxCLibrary implements LinuxCLibrary {
         CAPTURE_ERRNO_OPTION,
         Linker.Option.firstVariadicArg(1)
     );
+    private static final MethodHandle fallocate$mh = downcallHandleWithErrno(
+        "fallocate",
+        FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, JAVA_LONG, JAVA_LONG)
+    );
 
     private static class JdkSockFProg implements SockFProg {
         private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_SHORT, paddingLayout(6), ADDRESS);
@@ -100,4 +104,13 @@ class JdkLinuxCLibrary implements LinuxCLibrary {
             throw new AssertionError(t);
         }
     }
+
+    @Override
+    public int fallocate(int fd, int mode, long offset, long length) {
+        try {
+            return (int) fallocate$mh.invokeExact(errnoState, fd, mode, offset, length);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
 }

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java

@@ -19,6 +19,7 @@ import java.lang.foreign.MemoryLayout;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.StructLayout;
 import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
 
 import static java.lang.foreign.MemoryLayout.PathElement.groupElement;
@@ -49,6 +50,37 @@ class JdkPosixCLibrary implements PosixCLibrary {
     );
     private static final MethodHandle mlockall$mh = downcallHandleWithErrno("mlockall", FunctionDescriptor.of(JAVA_INT, JAVA_INT));
     private static final MethodHandle fcntl$mh = downcallHandle("fcntl", FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, ADDRESS));
+    private static final MethodHandle ftruncate$mh = downcallHandle("ftruncate", FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_LONG));
+    private static final MethodHandle open$mh = downcallHandle(
+        "open",
+        FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT),
+        CAPTURE_ERRNO_OPTION,
+        Linker.Option.firstVariadicArg(2)
+    );
+    private static final MethodHandle openWithMode$mh = downcallHandle(
+        "open",
+        FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT, JAVA_INT),
+        CAPTURE_ERRNO_OPTION,
+        Linker.Option.firstVariadicArg(2)
+    );
+    private static final MethodHandle close$mh = downcallHandleWithErrno("close", FunctionDescriptor.of(JAVA_INT, JAVA_INT));
+    private static final MethodHandle fstat$mh;
+    static {
+        MethodHandle fstat;
+        try {
+            fstat = downcallHandleWithErrno("fstat64", FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS));
+        } catch (LinkageError e) {
+            // Due to different sizes of the stat structure for 32 vs 64 bit machines, on some systems fstat actually points to
+            // an internal symbol. So we fall back to looking for that symbol.
+            int version = System.getProperty("os.arch").equals("aarch64") ? 0 : 1;
+            fstat = MethodHandles.insertArguments(
+                downcallHandleWithErrno("__fxstat", FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, ADDRESS)),
+                1,
+                version
+            );
+        }
+        fstat$mh = fstat;
+    }
 
     static final MemorySegment errnoState = Arena.ofAuto().allocate(CAPTURE_ERRNO_LAYOUT);
 
@@ -85,6 +117,16 @@ class JdkPosixCLibrary implements PosixCLibrary {
         return new JdkRLimit();
     }
 
+    @Override
+    public Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset) {
+        return new JdkStat64(sizeof, stSizeOffset, stBlocksOffset);
+    }
+
+    @Override
+    public FStore newFStore() {
+        return new JdkFStore();
+    }
+
     @Override
     public int getrlimit(int resource, RLimit rlimit) {
         assert rlimit instanceof JdkRLimit;
@@ -116,11 +158,6 @@ class JdkPosixCLibrary implements PosixCLibrary {
         }
     }
 
-    @Override
-    public FStore newFStore() {
-        return new JdkFStore();
-    }
-
     @Override
     public int fcntl(int fd, int cmd, FStore fst) {
         assert fst instanceof JdkFStore;
@@ -132,6 +169,55 @@ class JdkPosixCLibrary implements PosixCLibrary {
         }
     }
 
+    @Override
+    public int ftruncate(int fd, long length) {
+        try {
+            return (int) ftruncate$mh.invokeExact(errnoState, fd, length);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public int open(String pathname, int flags) {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment nativePathname = MemorySegmentUtil.allocateString(arena, pathname);
+            return (int) open$mh.invokeExact(errnoState, nativePathname, flags);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public int open(String pathname, int flags, int mode) {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment nativePathname = MemorySegmentUtil.allocateString(arena, pathname);
+            return (int) openWithMode$mh.invokeExact(errnoState, nativePathname, flags, mode);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public int close(int fd) {
+        try {
+            return (int) close$mh.invokeExact(errnoState, fd);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public int fstat64(int fd, Stat64 stat64) {
+        assert stat64 instanceof JdkStat64;
+        var jdkStat = (JdkStat64) stat64;
+        try {
+            return (int) fstat$mh.invokeExact(errnoState, fd, jdkStat.segment);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
     static class JdkRLimit implements RLimit {
         private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_LONG, JAVA_LONG);
         private static final VarHandle rlim_cur$vh = varHandleWithoutOffset(layout, groupElement(0));
@@ -170,6 +256,29 @@ class JdkPosixCLibrary implements PosixCLibrary {
         }
     }
 
+    private static class JdkStat64 implements Stat64 {
+
+        private final MemorySegment segment;
+        private final int stSizeOffset;
+        private final int stBlocksOffset;
+
+        JdkStat64(int sizeof, int stSizeOffset, int stBlocksOffset) {
+            this.segment = Arena.ofAuto().allocate(sizeof, 8);
+            this.stSizeOffset = stSizeOffset;
+            this.stBlocksOffset = stBlocksOffset;
+        }
+
+        @Override
+        public long st_size() {
+            return segment.get(JAVA_LONG, stSizeOffset);
+        }
+
+        @Override
+        public long st_blocks() {
+            return segment.get(JAVA_LONG, stBlocksOffset);
+        }
+    }
+
     private static class JdkFStore implements FStore {
         private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_INT, JAVA_INT, JAVA_LONG, JAVA_LONG, JAVA_LONG);
         private static final VarHandle st_flags$vh = layout.varHandle(groupElement(0));

libs/preallocate/build.gradle

@@ -1,18 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-apply plugin: 'elasticsearch.build'
-
-dependencies {
-  implementation project(':libs:elasticsearch-core')
-  implementation project(':libs:elasticsearch-logging')
-  implementation "net.java.dev.jna:jna:${versions.jna}"
-}
-
-tasks.named('forbiddenApisMain').configure {
-  replaceSignatureFiles 'jdk-signatures'
-}

libs/preallocate/licenses/jna-LICENSE.txt

@@ -1,177 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS

libs/preallocate/licenses/jna-NOTICE.txt

@@ -1 +0,0 @@
- 

libs/preallocate/src/main/java/module-info.java

@@ -1,17 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-module org.elasticsearch.preallocate {
-    requires org.elasticsearch.base;
-    requires org.elasticsearch.logging;
-    requires com.sun.jna;
-
-    exports org.elasticsearch.preallocate to org.elasticsearch.blobcache, com.sun.jna;
-
-    provides org.elasticsearch.jdk.ModuleQualifiedExportsService with org.elasticsearch.preallocate.PreallocateModuleExportsService;
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/AbstractPosixPreallocator.java

@@ -1,148 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.preallocate;
-
-import com.sun.jna.FunctionMapper;
-import com.sun.jna.Library;
-import com.sun.jna.Native;
-import com.sun.jna.NativeLong;
-import com.sun.jna.Platform;
-import com.sun.jna.Structure;
-
-import java.io.IOException;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Locale;
-import java.util.Map;
-
-abstract class AbstractPosixPreallocator implements Preallocator {
-
-    /**
-     * Constants relating to posix libc.
-     *
-     * @param SIZEOF_STAT The size of the stat64 structure, ie sizeof(stat64_t), found by importing sys/stat.h
-     * @param STAT_ST_SIZE_OFFSET The offsite into stat64 at which st_size exists, ie offsetof(stat64_t, st_size),
-     *                           found by importing sys/stat.h
-     * @param O_CREAT The file mode for creating a file upon opening, found by importing fcntl.h
-     */
-    protected record PosixConstants(int SIZEOF_STAT, int STAT_ST_SIZE_OFFSET, int O_CREAT) {}
-
-    private static final int O_WRONLY = 1;
-
-    static final class Stat64 extends Structure implements Structure.ByReference {
-        public byte[] _ignore1;
-        public NativeLong st_size = new NativeLong(0);
-        public byte[] _ignore2;
-
-        Stat64(int sizeof, int stSizeOffset) {
-            this._ignore1 = new byte[stSizeOffset];
-            this._ignore2 = new byte[sizeof - stSizeOffset - 8];
-        }
-    }
-
-    private interface NativeFunctions extends Library {
-        String strerror(int errno);
-
-        int open(String filename, int flags, Object... mode);
-
-        int close(int fd);
-    }
-
-    private interface FStat64Function extends Library {
-        int fstat64(int fd, Stat64 stat);
-    }
-
-    public static final boolean NATIVES_AVAILABLE;
-    private static final NativeFunctions functions;
-    private static final FStat64Function fstat64;
-
-    static {
-        functions = AccessController.doPrivileged((PrivilegedAction<NativeFunctions>) () -> {
-            try {
-                return Native.load(Platform.C_LIBRARY_NAME, NativeFunctions.class);
-            } catch (final UnsatisfiedLinkError e) {
-                return null;
-            }
-        });
-        fstat64 = AccessController.doPrivileged((PrivilegedAction<FStat64Function>) () -> {
-            try {
-                return Native.load(Platform.C_LIBRARY_NAME, FStat64Function.class);
-            } catch (final UnsatisfiedLinkError e) {
-                try {
-                    // on Linux fstat64 isn't available as a symbol, but instead uses a special __ name
-                    var options = Map.of(Library.OPTION_FUNCTION_MAPPER, (FunctionMapper) (lib, method) -> "__fxstat64");
-                    return Native.load(Platform.C_LIBRARY_NAME, FStat64Function.class, options);
-                } catch (UnsatisfiedLinkError e2) {
-                    return null;
-                }
-            }
-        });
-        NATIVES_AVAILABLE = functions != null && fstat64 != null;
-    }
-
-    private class PosixNativeFileHandle implements NativeFileHandle {
-
-        private final int fd;
-
-        PosixNativeFileHandle(int fd) {
-            this.fd = fd;
-        }
-
-        @Override
-        public int fd() {
-            return fd;
-        }
-
-        @Override
-        public long getSize() throws IOException {
-            var stat = new Stat64(constants.SIZEOF_STAT, constants.STAT_ST_SIZE_OFFSET);
-            if (fstat64.fstat64(fd, stat) == -1) {
-                throw newIOException("Could not get size of file");
-            }
-            return stat.st_size.longValue();
-        }
-
-        @Override
-        public void close() throws IOException {
-            if (functions.close(fd) != 0) {
-                throw newIOException("Could not close file");
-            }
-        }
-    }
-
-    protected final PosixConstants constants;
-
-    AbstractPosixPreallocator(PosixConstants constants) {
-        this.constants = constants;
-    }
-
-    @Override
-    public boolean useNative() {
-        return false;
-    }
-
-    @Override
-    public NativeFileHandle open(String path) throws IOException {
-        int fd = functions.open(path, O_WRONLY, constants.O_CREAT);
-        if (fd < 0) {
-            throw newIOException(String.format(Locale.ROOT, "Could not open file [%s] for preallocation", path));
-        }
-        return new PosixNativeFileHandle(fd);
-    }
-
-    @Override
-    public String error(int errno) {
-        return functions.strerror(errno);
-    }
-
-    private static IOException newIOException(String prefix) {
-        int errno = Native.getLastError();
-        return new IOException(String.format(Locale.ROOT, "%s(errno=%d): %s", prefix, errno, functions.strerror(errno)));
-    }
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/LinuxPreallocator.java

@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import com.sun.jna.Native;
-import com.sun.jna.Platform;
-
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-
-final class LinuxPreallocator extends AbstractPosixPreallocator {
-
-    LinuxPreallocator() {
-        super(new PosixConstants(144, 48, 64));
-    }
-
-    @Override
-    public boolean useNative() {
-        return Natives.NATIVES_AVAILABLE && super.useNative();
-    }
-
-    @Override
-    public int preallocate(final int fd, final long currentSize, final long fileSize) {
-        final int rc = Natives.fallocate(fd, 0, currentSize, fileSize - currentSize);
-        return rc == 0 ? 0 : Native.getLastError();
-    }
-
-    private static class Natives {
-
-        public static final boolean NATIVES_AVAILABLE;
-
-        static {
-            NATIVES_AVAILABLE = AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> {
-                try {
-                    Native.register(Natives.class, Platform.C_LIBRARY_NAME);
-                } catch (final UnsatisfiedLinkError e) {
-                    return false;
-                }
-                return true;
-            });
-        }
-
-        static native int fallocate(int fd, int mode, long offset, long length);
-    }
-
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/MacOsPreallocator.java

@@ -1,103 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import com.sun.jna.Native;
-import com.sun.jna.NativeLong;
-import com.sun.jna.Platform;
-import com.sun.jna.Structure;
-
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Arrays;
-import java.util.List;
-
-final class MacOsPreallocator extends AbstractPosixPreallocator {
-
-    MacOsPreallocator() {
-        super(new PosixConstants(144, 96, 512));
-    }
-
-    @Override
-    public boolean useNative() {
-        return Natives.NATIVES_AVAILABLE && super.useNative();
-    }
-
-    @Override
-    public int preallocate(final int fd, final long currentSize /* unused */ , final long fileSize) {
-        // the Structure.ByReference constructor requires access to declared members
-        final Natives.Fcntl.FStore fst = AccessController.doPrivileged((PrivilegedAction<Natives.Fcntl.FStore>) Natives.Fcntl.FStore::new);
-        fst.fst_flags = Natives.Fcntl.F_ALLOCATECONTIG;
-        fst.fst_posmode = Natives.Fcntl.F_PEOFPOSMODE;
-        fst.fst_offset = new NativeLong(0);
-        fst.fst_length = new NativeLong(fileSize);
-        // first, try allocating contiguously
-        if (Natives.fcntl(fd, Natives.Fcntl.F_PREALLOCATE, fst) != 0) {
-            // that failed, so let us try allocating non-contiguously
-            fst.fst_flags = Natives.Fcntl.F_ALLOCATEALL;
-            if (Natives.fcntl(fd, Natives.Fcntl.F_PREALLOCATE, fst) != 0) {
-                // i'm afraid captain dale had to bail
-                return Native.getLastError();
-            }
-        }
-        if (Natives.ftruncate(fd, new NativeLong(fileSize)) != 0) {
-            return Native.getLastError();
-        }
-        return 0;
-    }
-
-    private static class Natives {
-
-        static boolean NATIVES_AVAILABLE;
-
-        static {
-            NATIVES_AVAILABLE = AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> {
-                try {
-                    Native.register(Natives.class, Platform.C_LIBRARY_NAME);
-                } catch (final UnsatisfiedLinkError e) {
-                    return false;
-                }
-                return true;
-            });
-        }
-
-        static class Fcntl {
-            private static final int F_PREALLOCATE = 42;
-
-            // allocate flags; these might be unused, but are here for reference
-            @SuppressWarnings("unused")
-            private static final int F_ALLOCATECONTIG = 0x00000002; // allocate contiguous space
-            private static final int F_ALLOCATEALL = 0x00000004; // allocate all the requested space or no space at all
-
-            // position modes; these might be unused, but are here for reference
-            private static final int F_PEOFPOSMODE = 3; // allocate from the physical end of the file
-            @SuppressWarnings("unused")
-            private static final int F_VOLPOSMODE = 4; // allocate from the volume offset
-
-            public static final class FStore extends Structure implements Structure.ByReference {
-                public int fst_flags = 0;
-                public int fst_posmode = 0;
-                public NativeLong fst_offset = new NativeLong(0);
-                public NativeLong fst_length = new NativeLong(0);
-                @SuppressWarnings("unused")
-                public NativeLong fst_bytesalloc = new NativeLong(0);
-
-                @Override
-                protected List<String> getFieldOrder() {
-                    return Arrays.asList("fst_flags", "fst_posmode", "fst_offset", "fst_length", "fst_bytesalloc");
-                }
-
-            }
-        }
-
-        static native int fcntl(int fd, int cmd, Fcntl.FStore fst);
-
-        static native int ftruncate(int fd, NativeLong length);
-    }
-
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/NoNativePreallocator.java

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import java.io.IOException;
-
-final class NoNativePreallocator implements Preallocator {
-
-    @Override
-    public boolean useNative() {
-        return false;
-    }
-
-    @Override
-    public NativeFileHandle open(String path) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int preallocate(final int fd, final long currentSize, final long fileSize) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public String error(final int errno) {
-        throw new UnsupportedOperationException();
-    }
-
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocate.java

@@ -1,110 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.logging.LogManager;
-import org.elasticsearch.logging.Logger;
-import org.elasticsearch.preallocate.Preallocator.NativeFileHandle;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.lang.reflect.Field;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.security.PrivilegedExceptionAction;
-
-public class Preallocate {
-
-    private static final Logger logger = LogManager.getLogger(Preallocate.class);
-
-    private static final boolean IS_LINUX;
-    private static final boolean IS_MACOS;
-    static {
-        String osName = System.getProperty("os.name");
-        IS_LINUX = osName.startsWith("Linux");
-        IS_MACOS = osName.startsWith("Mac OS X");
-    }
-
-    public static void preallocate(final Path cacheFile, final long fileSize) throws IOException {
-        if (IS_LINUX) {
-            preallocate(cacheFile, fileSize, new LinuxPreallocator());
-        } else if (IS_MACOS) {
-            preallocate(cacheFile, fileSize, new MacOsPreallocator());
-        } else {
-            preallocate(cacheFile, fileSize, new NoNativePreallocator());
-        }
-    }
-
-    @SuppressForbidden(reason = "need access to toFile for RandomAccessFile")
-    private static void preallocate(final Path cacheFile, final long fileSize, final Preallocator prealloactor) throws IOException {
-        boolean success = false;
-        try {
-            if (prealloactor.useNative()) {
-                try (NativeFileHandle openFile = prealloactor.open(cacheFile.toAbsolutePath().toString())) {
-                    long currentSize = openFile.getSize();
-                    if (currentSize < fileSize) {
-                        logger.info("pre-allocating cache file [{}] ({} bytes) using native methods", cacheFile, fileSize);
-                        final int errno = prealloactor.preallocate(openFile.fd(), currentSize, fileSize - currentSize);
-                        if (errno == 0) {
-                            success = true;
-                            logger.debug("pre-allocated cache file [{}] using native methods", cacheFile);
-                        } else {
-                            logger.warn(
-                                "failed to pre-allocate cache file [{}] using native methods, errno: [{}], error: [{}]",
-                                cacheFile,
-                                errno,
-                                prealloactor.error(errno)
-                            );
-                        }
-                    }
-                } catch (final Exception e) {
-                    logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using native methods", e);
-                }
-            }
-            // even if allocation was successful above, verify again here
-            try (RandomAccessFile raf = new RandomAccessFile(cacheFile.toFile(), "rw")) {
-                if (raf.length() != fileSize) {
-                    logger.info("pre-allocating cache file [{}] ({} bytes) using setLength method", cacheFile, fileSize);
-                    raf.setLength(fileSize);
-                    logger.debug("pre-allocated cache file [{}] using setLength method", cacheFile);
-                }
-                success = raf.length() == fileSize;
-            } catch (final Exception e) {
-                logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using setLength method", e);
-                throw e;
-            }
-        } finally {
-            if (success == false) {
-                // if anything goes wrong, delete the potentially created file to not waste disk space
-                Files.deleteIfExists(cacheFile);
-            }
-        }
-    }
-
-    @SuppressForbidden(reason = "need access to fd on FileOutputStream")
-    private static class FileDescriptorFieldAction implements PrivilegedExceptionAction<Field> {
-
-        private final FileOutputStream fileOutputStream;
-
-        private FileDescriptorFieldAction(FileOutputStream fileOutputStream) {
-            this.fileOutputStream = fileOutputStream;
-        }
-
-        @Override
-        public Field run() throws IOException, NoSuchFieldException {
-            // accessDeclaredMembers
-            final Field f = fileOutputStream.getFD().getClass().getDeclaredField("fd");
-            // suppressAccessChecks
-            f.setAccessible(true);
-            return f;
-        }
-    }
-
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/PreallocateModuleExportsService.java

@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import org.elasticsearch.jdk.ModuleQualifiedExportsService;
-
-public class PreallocateModuleExportsService extends ModuleQualifiedExportsService {
-
-    @Override
-    protected void addExports(String pkg, Module target) {
-        module.addExports(pkg, target);
-    }
-
-    @Override
-    protected void addOpens(String pkg, Module target) {
-        module.addOpens(pkg, target);
-    }
-}

libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocator.java

@@ -1,59 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.preallocate;
-
-import java.io.IOException;
-
-/**
- * Represents platform native methods for pre-allocating files.
- */
-interface Preallocator {
-
-    /** A handle for an open file */
-    interface NativeFileHandle extends AutoCloseable {
-        /** A valid native file descriptor */
-        int fd();
-
-        /** Retrieves the current size of the file */
-        long getSize() throws IOException;
-    }
-
-    /**
-     * Returns if native methods for pre-allocating files are available.
-     *
-     * @return true if native methods are available, otherwise false
-     */
-    boolean useNative();
-
-    /**
-     * Open a file for preallocation.
-     *
-     * @param path The absolute path to the file to be opened
-     * @return a handle to the open file that may be used for preallocate
-     */
-    NativeFileHandle open(String path) throws IOException;
-
-    /**
-     * Pre-allocate a file of given current size to the specified size using the given file descriptor.
-     *
-     * @param fd the file descriptor
-     * @param currentSize the current size of the file
-     * @param fileSize the size to pre-allocate
-     * @return 0 upon success
-     */
-    int preallocate(int fd, long currentSize, long fileSize);
-
-    /**
-     * Provide a string representation of the given error number.
-     *
-     * @param errno the error number
-     * @return the error message
-     */
-    String error(int errno);
-
-}

server/build.gradle

@@ -40,8 +40,6 @@ dependencies {
   implementation project(":libs:elasticsearch-simdvec")
 
   implementation project(':libs:elasticsearch-plugin-classloader')
-  // no compile dependency by server, but server defines security policy for this codebase so it i>
-  runtimeOnly project(":libs:elasticsearch-preallocate")
 
   // lucene
   api "org.apache.lucene:lucene-core:${versions.lucene}"

server/src/main/java/module-info.java

@@ -189,7 +189,6 @@ module org.elasticsearch.server {
     exports org.elasticsearch.common.compress;
     exports org.elasticsearch.common.document;
     exports org.elasticsearch.common.file;
-    exports org.elasticsearch.common.filesystem;
     exports org.elasticsearch.common.geo;
     exports org.elasticsearch.common.hash;
     exports org.elasticsearch.common.inject;

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -19,7 +19,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ReleaseVersions;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.ReferenceDocs;
-import org.elasticsearch.common.filesystem.FileSystemNatives;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.network.IfConfig;
@@ -318,9 +317,6 @@ class Elasticsearch {
 
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
-
-        // init filesystem natives
-        FileSystemNatives.init();
     }
 
     static void initializeProbes() {

server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java

@@ -294,8 +294,8 @@ public class PolicyUtil {
                             + " in policy file ["
                             + policyFile
                             + "]"
-                            + "\nAvailable codebases: "
-                            + codebaseProperties.keySet()
+                            + "\nAvailable codebases: \n  "
+                            + String.join("\n  ", codebaseProperties.keySet().stream().sorted().toList())
                     );
                 }
                 return policy;

server/src/main/java/org/elasticsearch/common/filesystem/FileSystemNatives.java

@@ -1,68 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.common.filesystem;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.Constants;
-
-import java.nio.file.Path;
-import java.util.OptionalLong;
-
-/**
- * This class provides utility methods for calling some native methods related to filesystems.
- */
-public final class FileSystemNatives {
-
-    private static final Logger logger = LogManager.getLogger(FileSystemNatives.class);
-
-    @FunctionalInterface
-    interface Provider {
-        OptionalLong allocatedSizeInBytes(Path path);
-    }
-
-    private static final Provider NOOP_FILE_SYSTEM_NATIVES_PROVIDER = path -> OptionalLong.empty();
-    private static final Provider JNA_PROVIDER = loadJnaProvider();
-
-    private static Provider loadJnaProvider() {
-        try {
-            // load one of the main JNA classes to see if the classes are available. this does not ensure that all native
-            // libraries are available, only the ones necessary by JNA to function
-            Class.forName("com.sun.jna.Native");
-            if (Constants.WINDOWS) {
-                return WindowsFileSystemNatives.getInstance();
-            } else if (Constants.LINUX && Constants.JRE_IS_64BIT) {
-                return LinuxFileSystemNatives.getInstance();
-            }
-        } catch (ClassNotFoundException e) {
-            logger.warn("JNA not found. FileSystemNatives methods will be disabled.", e);
-        } catch (LinkageError e) {
-            logger.warn("unable to load JNA native support library, FileSystemNatives methods will be disabled.", e);
-        }
-        return NOOP_FILE_SYSTEM_NATIVES_PROVIDER;
-    }
-
-    private FileSystemNatives() {}
-
-    public static void init() {
-        assert JNA_PROVIDER != null;
-    }
-
-    /**
-     * Returns the number of allocated bytes on disk for a given file.
-     *
-     * @param path the path to the file
-     * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file. The optional is empty is the
-     * allocated size of the file failed be retrieved using native methods
-     */
-    public static OptionalLong allocatedSizeInBytes(Path path) {
-        return JNA_PROVIDER.allocatedSizeInBytes(path);
-    }
-
-}

server/src/main/java/org/elasticsearch/common/filesystem/LinuxFileSystemNatives.java

@@ -1,197 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.common.filesystem;
-
-import com.sun.jna.LastErrorException;
-import com.sun.jna.Native;
-import com.sun.jna.Platform;
-import com.sun.jna.Structure;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.Constants;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.util.OptionalLong;
-
-import static org.elasticsearch.core.Strings.format;
-
-/**
- * {@link FileSystemNatives.Provider} implementation for Linux x86-64bits
- */
-final class LinuxFileSystemNatives implements FileSystemNatives.Provider {
-
-    private static final Logger logger = LogManager.getLogger(LinuxFileSystemNatives.class);
-
-    private static final LinuxFileSystemNatives INSTANCE = new LinuxFileSystemNatives();
-
-    /** st_blocks field indicates the number of blocks allocated to the file, 512-byte units **/
-    private static final long ST_BLOCKS_UNIT = 512L;
-
-    /**
-     * Version of the `struct stat' data structure.
-     *
-     * To allow the `struct stat' structure bits to vary without changing shared library major version number, the `stat' function is often
-     * an inline wrapper around `xstat' which takes a leading version-number argument designating the data structure and bits used.
-     *
-     * In glibc this version is defined in bits/stat.h (or bits/struct_stat.h in glibc 2.33, or bits/xstatver.h in more recent versions).
-     *
-     * For x86-64 the _STAT_VER used is:
-     *  # define _STAT_VER_LINUX    1
-     *  # define _STAT_VER _STAT_VER_LINUX
-     *
-     * For other architectures the _STAT_VER used is:
-     *  # define _STAT_VER_LINUX    0
-     *  # define _STAT_VER _STAT_VER_LINUX
-     **/
-    private static int loadStatVersion() {
-        return "aarch64".equalsIgnoreCase(Constants.OS_ARCH) ? 0 : 1;
-    }
-
-    private static final int STAT_VER = loadStatVersion();
-
-    private LinuxFileSystemNatives() {
-        assert Constants.LINUX : Constants.OS_NAME;
-        assert Constants.JRE_IS_64BIT : Constants.OS_ARCH;
-        try {
-            Native.register(XStatLibrary.class, Platform.C_LIBRARY_NAME);
-            logger.debug("C library loaded");
-        } catch (LinkageError e) {
-            logger.warn("unable to link C library. native methods and handlers will be disabled.", e);
-            throw e;
-        }
-    }
-
-    static LinuxFileSystemNatives getInstance() {
-        return INSTANCE;
-    }
-
-    public static class XStatLibrary {
-        public static native int __xstat(int version, String path, Stat stats) throws LastErrorException;
-    }
-
-    /**
-     * Retrieves the actual number of bytes of disk storage used to store a specified file.
-     *
-     * @param path the path to the file
-     * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid
-     */
-    @Override
-    public OptionalLong allocatedSizeInBytes(Path path) {
-        assert Files.isRegularFile(path) : path;
-        try {
-            final Stat stats = new Stat();
-            final int rc = XStatLibrary.__xstat(STAT_VER, path.toString(), stats);
-            if (logger.isTraceEnabled()) {
-                logger.trace("executing native method __xstat() returned {} with error code [{}] for file [{}]", stats, rc, path);
-            }
-            return OptionalLong.of(stats.st_blocks * ST_BLOCKS_UNIT);
-        } catch (LastErrorException e) {
-            logger.warn(
-                () -> format(
-                    "error when executing native method __xstat(int vers, const char *name, struct stat *buf) for file [%s]",
-                    path
-                ),
-                e
-            );
-        }
-        return OptionalLong.empty();
-    }
-
-    @Structure.FieldOrder(
-        {
-            "st_dev",
-            "st_ino",
-            "st_nlink",
-            "st_mode",
-            "st_uid",
-            "st_gid",
-            "__pad0",
-            "st_rdev",
-            "st_size",
-            "st_blksize",
-            "st_blocks",
-            "st_atim",
-            "st_mtim",
-            "st_ctim",
-            "__glibc_reserved0",
-            "__glibc_reserved1",
-            "__glibc_reserved2" }
-    )
-    public static class Stat extends Structure {
-
-        /**
-         * The stat structure varies across architectures in the glibc and kernel source codes. For example some fields might be ordered
-         * differently and/or some padding bytes might be present between some fields.
-         *
-         * The struct implemented here refers to the Linux x86 architecture in the glibc source files:
-         * - glibc version 2.23: sysdeps/unix/sysv/linux/x86/bits/stat.h
-         * - glibc version 2.33: sysdeps/unix/sysv/linux/x86/bits/struct_stat.h
-         *
-         * The following command is useful to compile the stat struct on a given system:
-         *     echo "#include <sys/stat.h>" | gcc -xc - -E -dD | grep -ve '^$' | grep -A23 '^struct stat'
-         */
-        public long st_dev;         // __dev_t st_dev; /* Device. */
-        public long st_ino;         // __ino_t st_ino; /* File serial number. */
-        public long st_nlink;       // __nlink_t st_nlink; /* Link count. */
-        public int st_mode;         // __mode_t st_mode; /* File mode. */
-        public int st_uid;          // __uid_t st_uid; /* User ID of the file's owner. */
-        public int st_gid;          // __gid_t st_gid; /* Group ID of the file's group. */
-        public int __pad0;
-        public long st_rdev;        // __dev_t st_rdev; /* Device number, if device. */
-        public long st_size;        // __off_t st_size; /* Size of file, in bytes. */
-        public long st_blksize;     // __blksize_t st_blksize; /* Optimal block size for I/O. */
-        public long st_blocks;      // __blkcnt_t st_blocks; /* Number 512-byte blocks allocated. */
-        public Time st_atim;        // struct timespec st_atim; /* Time of last access. */
-        public Time st_mtim;        // struct timespec st_mtim; /* Time of last modification. */
-        public Time st_ctim;        // struct timespec st_ctim; /* Time of last status change. */
-        public long __glibc_reserved0;      // __syscall_slong_t
-        public long __glibc_reserved1;      // __syscall_slong_t
-        public long __glibc_reserved2;      // __syscall_slong_t
-
-        @Override
-        public String toString() {
-            return "[st_dev="
-                + st_dev
-                + ", st_ino="
-                + st_ino
-                + ", st_nlink="
-                + st_nlink
-                + ", st_mode="
-                + st_mode
-                + ", st_uid="
-                + st_uid
-                + ", st_gid="
-                + st_gid
-                + ", st_rdev="
-                + st_rdev
-                + ", st_size="
-                + st_size
-                + ", st_blksize="
-                + st_blksize
-                + ", st_blocks="
-                + st_blocks
-                + ", st_atim="
-                + Instant.ofEpochSecond(st_atim.tv_sec, st_atim.tv_nsec)
-                + ", st_mtim="
-                + Instant.ofEpochSecond(st_mtim.tv_sec, st_mtim.tv_nsec)
-                + ", st_ctim="
-                + Instant.ofEpochSecond(st_ctim.tv_sec, st_ctim.tv_nsec)
-                + ']';
-        }
-    }
-
-    @Structure.FieldOrder({ "tv_sec", "tv_nsec" })
-    public static class Time extends Structure {
-        public long tv_sec;
-        public long tv_nsec;
-    }
-}

server/src/main/java/org/elasticsearch/common/filesystem/WindowsFileSystemNatives.java

@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.common.filesystem;
-
-import com.sun.jna.Native;
-import com.sun.jna.WString;
-import com.sun.jna.ptr.IntByReference;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.Constants;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.OptionalLong;
-
-/**
- * {@link FileSystemNatives.Provider} implementation for Windows/Kernel32
- */
-final class WindowsFileSystemNatives implements FileSystemNatives.Provider {
-
-    private static final Logger logger = LogManager.getLogger(WindowsFileSystemNatives.class);
-
-    private static final WindowsFileSystemNatives INSTANCE = new WindowsFileSystemNatives();
-
-    private static final int INVALID_FILE_SIZE = -1;
-    private static final int NO_ERROR = 0;
-
-    private WindowsFileSystemNatives() {
-        assert Constants.WINDOWS : Constants.OS_NAME;
-        try {
-            Native.register("kernel32");
-            logger.debug("windows/Kernel32 library loaded");
-        } catch (LinkageError e) {
-            logger.warn("unable to link Windows/Kernel32 library. native methods and handlers will be disabled.", e);
-            throw e;
-        }
-    }
-
-    static WindowsFileSystemNatives getInstance() {
-        return INSTANCE;
-    }
-
-    /**
-     * Retrieves the actual number of bytes of disk storage used to store a specified file.
-     *
-     * https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getcompressedfilesizew
-     *
-     * @param lpFileName the path string
-     * @param lpFileSizeHigh pointer to high-order DWORD for compressed file size (or null if not needed)
-     * @return the low-order DWORD for compressed file siz
-     */
-    private native int GetCompressedFileSizeW(WString lpFileName, IntByReference lpFileSizeHigh);
-
-    /**
-     * Retrieves the actual number of bytes of disk storage used to store a specified file. If the file is located on a volume that supports
-     * compression and the file is compressed, the value obtained is the compressed size of the specified file. If the file is located on a
-     * volume that supports sparse files and the file is a sparse file, the value obtained is the sparse size of the specified file.
-     *
-     * This method uses Win32 DLL native method {@link #GetCompressedFileSizeW(WString, IntByReference)}.
-     *
-     * @param path the path to the file
-     * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid
-     */
-    public OptionalLong allocatedSizeInBytes(Path path) {
-        assert Files.isRegularFile(path) : path;
-        final WString fileName = new WString("\\\\?\\" + path);
-        final IntByReference lpFileSizeHigh = new IntByReference();
-
-        final int lpFileSizeLow = GetCompressedFileSizeW(fileName, lpFileSizeHigh);
-        if (lpFileSizeLow == INVALID_FILE_SIZE) {
-            final int err = Native.getLastError();
-            if (err != NO_ERROR) {
-                logger.warn("error [{}] when executing native method GetCompressedFileSizeW for file [{}]", err, path);
-                return OptionalLong.empty();
-            }
-        }
-
-        // convert lpFileSizeLow to unsigned long and combine with signed/shifted lpFileSizeHigh
-        final long allocatedSize = (((long) lpFileSizeHigh.getValue()) << Integer.SIZE) | Integer.toUnsignedLong(lpFileSizeLow);
-        if (logger.isTraceEnabled()) {
-            logger.trace(
-                "executing native method GetCompressedFileSizeW returned [high={}, low={}, allocated={}] for file [{}]",
-                lpFileSizeHigh,
-                lpFileSizeLow,
-                allocatedSize,
-                path
-            );
-        }
-        return OptionalLong.of(allocatedSize);
-    }
-}

server/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -70,19 +70,12 @@ grant codeBase "${codebase.elasticsearch-cli}" {
 grant codeBase "${codebase.jna}" {
   // for registering native methods
   permission java.lang.RuntimePermission "accessDeclaredMembers";
-  permission java.lang.reflect.ReflectPermission "newProxyInPackage.org.elasticsearch.preallocate";
 };
 
 grant codeBase "${codebase.log4j-api}" {
   permission java.lang.RuntimePermission "getClassLoader";
 };
 
-grant codeBase "${codebase.elasticsearch-preallocate}" {
-  // for registering native methods
-  permission java.lang.RuntimePermission "accessDeclaredMembers";
-  permission java.lang.reflect.ReflectPermission "newProxyInPackage.org.elasticsearch.preallocate";
-};
-
 grant codeBase "${codebase.elasticsearch-simdvec}" {
   // for access MemorySegmentIndexInput internals
   permission java.lang.RuntimePermission "accessDeclaredMembers";

test/framework/build.gradle

@@ -16,7 +16,6 @@ dependencies {
   api project(':libs:elasticsearch-ssl-config')
   api project(":server")
   api project(":libs:elasticsearch-cli")
-  api project(':libs:elasticsearch-preallocate')
   api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   api "junit:junit:${versions.junit}"
   api "org.hamcrest:hamcrest:${versions.hamcrest}"

test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -14,7 +14,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.filesystem.FileSystemNatives;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.network.IfConfig;
 import org.elasticsearch.common.settings.Settings;
@@ -91,9 +90,6 @@ public class BootstrapForTesting {
         final boolean systemCallFilter = Booleans.parseBoolean(System.getProperty("tests.system_call_filter", "true"));
         Elasticsearch.initializeNatives(javaTmpDir, memoryLock, systemCallFilter, true);
 
-        // init filesystem natives
-        FileSystemNatives.init();
-
         // initialize probes
         Elasticsearch.initializeProbes();
 
@@ -222,7 +218,6 @@ public class BootstrapForTesting {
         addClassCodebase(codebases, "elasticsearch-rest-client", "org.elasticsearch.client.RestClient");
         addClassCodebase(codebases, "elasticsearch-core", "org.elasticsearch.core.Booleans");
         addClassCodebase(codebases, "elasticsearch-cli", "org.elasticsearch.cli.Command");
-        addClassCodebase(codebases, "elasticsearch-preallocate", "org.elasticsearch.preallocate.Preallocate");
         addClassCodebase(codebases, "elasticsearch-simdvec", "org.elasticsearch.simdvec.VectorScorerFactory");
         addClassCodebase(codebases, "framework", "org.elasticsearch.test.ESTestCase");
         return codebases;

x-pack/plugin/blob-cache/build.gradle

@@ -16,5 +16,5 @@ esplugin {
 }
 
 dependencies {
-    compileOnly project(path: ':libs:elasticsearch-preallocate')
+    compileOnly project(path: ':libs:elasticsearch-native')
 }

x-pack/plugin/blob-cache/src/main/java/module-info.java

@@ -8,7 +8,7 @@
 module org.elasticsearch.blobcache {
     requires org.elasticsearch.base;
     requires org.elasticsearch.server;
-    requires org.elasticsearch.preallocate;
+    requires org.elasticsearch.nativeaccess;
 
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java

@@ -18,10 +18,11 @@ import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.preallocate.Preallocate;
+import org.elasticsearch.nativeaccess.NativeAccess;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
@@ -78,7 +79,7 @@ public class SharedBytes extends AbstractRefCounted {
         Path cacheFile = null;
         if (fileSize > 0) {
             cacheFile = findCacheSnapshotCacheFilePath(environment, fileSize);
-            Preallocate.preallocate(cacheFile, fileSize);
+            preallocate(cacheFile, fileSize);
             this.fileChannel = FileChannel.open(cacheFile, OPEN_OPTIONS);
             assert this.fileChannel.size() == fileSize : "expected file size " + fileSize + " but was " + fileChannel.size();
         } else {
@@ -141,6 +142,24 @@ public class SharedBytes extends AbstractRefCounted {
         }
     }
 
+    @SuppressForbidden(reason = "random access file needed to set file size")
+    static void preallocate(Path cacheFile, long fileSize) throws IOException {
+        // first try using native methods to preallocate space in the file
+        NativeAccess.instance().tryPreallocate(cacheFile, fileSize);
+        // even if allocation was successful above, verify again here
+        try (RandomAccessFile raf = new RandomAccessFile(cacheFile.toFile(), "rw")) {
+            if (raf.length() != fileSize) {
+                logger.info("pre-allocating cache file [{}] ({} bytes) using setLength method", cacheFile, fileSize);
+                raf.setLength(fileSize);
+                logger.debug("pre-allocated cache file [{}] using setLength method", cacheFile);
+            }
+        } catch (final Exception e) {
+            logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using setLength method", e);
+            // if anything goes wrong, delete the potentially created file to not waste disk space
+            Files.deleteIfExists(cacheFile);
+        }
+    }
+
     /**
      * Copy {@code length} bytes from {@code input} to {@code fc}, only doing writes aligned along {@link #PAGE_SIZE}.
      *

x-pack/plugin/searchable-snapshots/build.gradle

@@ -15,6 +15,7 @@ base {
 dependencies {
   compileOnly project(path: xpackModule('core'))
   compileOnly project(path: xpackModule('blob-cache'))
+  compileOnly project(path: ':libs:elasticsearch-native')
   testImplementation(testArtifact(project(xpackModule('blob-cache'))))
   internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
   internalClusterTestImplementation(project(path: xpackModule('shutdown')))

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java

@@ -14,11 +14,11 @@ import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.blobcache.BlobCacheTestUtils;
 import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.filesystem.FileSystemNatives;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.nativeaccess.NativeAccess;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile.EvictionListener;
@@ -393,15 +393,7 @@ public class CacheFileTests extends ESTestCase {
         }
     }
 
-    private static void assumeLinux64bitsOrWindows() {
-        assumeTrue(
-            "This test uses native methods implemented only for Windows & Linux 64bits",
-            Constants.WINDOWS || Constants.LINUX && Constants.JRE_IS_64BIT
-        );
-    }
-
     public void testCacheFileCreatedAsSparseFile() throws Exception {
-        assumeLinux64bitsOrWindows();
         final long fourKb = 4096L;
         final long oneMb = 1 << 20;
 
@@ -420,7 +412,7 @@ public class CacheFileTests extends ESTestCase {
             final FileChannel fileChannel = cacheFile.getChannel();
             assertTrue(Files.exists(file));
 
-            OptionalLong sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file);
+            OptionalLong sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file);
             assertTrue(sizeOnDisk.isPresent());
             assertThat(sizeOnDisk.getAsLong(), equalTo(0L));
 
@@ -430,7 +422,7 @@ public class CacheFileTests extends ESTestCase {
             fill(fileChannel, Math.toIntExact(cacheFile.getLength() - 1L), Math.toIntExact(cacheFile.getLength()));
             fileChannel.force(false);
 
-            sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file);
+            sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file);
             assertTrue(sizeOnDisk.isPresent());
             assertThat(
                 "Cache file should be sparse and not fully allocated on disk",
@@ -445,7 +437,7 @@ public class CacheFileTests extends ESTestCase {
             fill(fileChannel, 0, Math.toIntExact(cacheFile.getLength()));
             fileChannel.force(false);
 
-            sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file);
+            sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file);
             assertTrue(sizeOnDisk.isPresent());
             assertThat(
                 "Cache file should be fully allocated on disk (maybe more given cluster/block size)",