Merge pull request #10887 from gmarz/feature/virtuallock

mlockall for Windows (VirtualLock)

src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.PidFile;
@@ -27,7 +28,6 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.inject.spi.Message;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.jna.Kernel32Library;
 import org.elasticsearch.common.jna.Natives;
 import org.elasticsearch.common.lease.Releasables;
@@ -88,7 +88,11 @@ public class Bootstrap {
     public static void initializeNatives(boolean mlockAll, boolean ctrlHandler) {
         // mlockall if requested
         if (mlockAll) {
-            Natives.tryMlockall();
+            if (Constants.WINDOWS) {
+               Natives.tryVirtualLock();
+            } else {
+               Natives.tryMlockall();
+            }
         }
         
         // check if the user is running as root, and bail

src/main/java/org/elasticsearch/common/jna/Kernel32Library.java

@@ -20,7 +20,7 @@
 package org.elasticsearch.common.jna;
 
 import com.google.common.collect.ImmutableList;
-import com.sun.jna.Native;
+import com.sun.jna.*;
 import com.sun.jna.win32.StdCallLibrary;
 
 import org.apache.lucene.util.Constants;
@@ -28,6 +28,7 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 
@@ -134,4 +135,91 @@ public class Kernel32Library {
          */
         boolean handle(int code);
     }
+
+
+    /**
+     * Memory protection constraints
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366786%28v=vs.85%29.aspx
+     */
+    public static final int PAGE_NOACCESS = 0x0001;
+    public static final int PAGE_GUARD = 0x0100;
+    public static final int MEM_COMMIT = 0x1000;
+
+    /**
+     * Contains information about a range of pages in the virtual address space of a process.
+     * The VirtualQuery and VirtualQueryEx functions use this structure.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366775%28v=vs.85%29.aspx
+     */
+    public static class MemoryBasicInformation extends Structure {
+        public Pointer BaseAddress;
+        public Pointer AllocationBase;
+        public NativeLong AllocationProtect;
+        public SizeT RegionSize;
+        public NativeLong State;
+        public NativeLong Protect;
+        public NativeLong Type;
+
+        @Override
+        protected List getFieldOrder() {
+            return Arrays.asList(new String[]{"BaseAddress", "AllocationBase", "AllocationProtect", "RegionSize", "State", "Protect", "Type"});
+        }
+    }
+
+    /**
+     * Locks the specified region of the process's virtual address space into physical
+     * memory, ensuring that subsequent access to the region will not incur a page fault.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx
+     *
+     * @param address A pointer to the base address of the region of pages to be locked.
+     * @param size The size of the region to be locked, in bytes.
+     * @return true if the function succeeds
+     */
+    public native boolean VirtualLock(Pointer address, SizeT size);
+
+    /**
+     * Retrieves information about a range of pages within the virtual address space of a specified process.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366907%28v=vs.85%29.aspx
+     *
+     * @param handle A handle to the process whose memory information is queried.
+     * @param address A pointer to the base address of the region of pages to be queried.
+     * @param memoryInfo A pointer to a structure in which information about the specified page range is returned.
+     * @param length The size of the buffer pointed to by the memoryInfo parameter, in bytes.
+     * @return the actual number of bytes returned in the information buffer.
+     */
+    public native int VirtualQueryEx(Pointer handle, Pointer address, MemoryBasicInformation memoryInfo, int length);
+
+    /**
+     * Sets the minimum and maximum working set sizes for the specified process.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234%28v=vs.85%29.aspx
+     *
+     * @param handle A handle to the process whose working set sizes is to be set.
+     * @param minSize The minimum working set size for the process, in bytes.
+     * @param maxSize The maximum working set size for the process, in bytes.
+     * @return true if the function succeeds.
+     */
+    public native boolean SetProcessWorkingSetSize(Pointer handle, SizeT minSize, SizeT maxSize);
+
+    /**
+     * Retrieves a pseudo handle for the current process.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/ms683179%28v=vs.85%29.aspx
+     *
+     * @return a pseudo handle to the current process.
+     */
+    public native Pointer GetCurrentProcess();
+
+    /**
+     * Closes an open object handle.
+     *
+     * https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx
+     *
+     * @param handle A valid handle to an open object.
+     * @return true if the function succeeds.
+     */
+    public native boolean CloseHandle(Pointer handle);
 }

src/main/java/org/elasticsearch/common/jna/Natives.java

@@ -20,10 +20,13 @@
 package org.elasticsearch.common.jna;
 
 import com.sun.jna.Native;
+import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.jna.Kernel32Library.ConsoleCtrlHandler;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.util.Locale;
 
@@ -75,6 +78,42 @@ public class Natives {
         }
     }
 
+    public static void tryVirtualLock()
+    {
+        Kernel32Library kernel = Kernel32Library.getInstance();
+        Pointer process = null;
+        try {
+            process = kernel.GetCurrentProcess();
+            // By default, Windows limits the number of pages that can be locked.
+            // Thus, we need to first increase the working set size of the JVM by
+            // the amount of memory we wish to lock, plus a small overhead (1MB).
+            SizeT size = new SizeT(JvmInfo.jvmInfo().getMem().getHeapInit().getBytes() + (1024 * 1024));
+            if (!kernel.SetProcessWorkingSetSize(process, size, size)) {
+                logger.warn("Unable to lock JVM memory. Failed to set working set size. Error code " + Native.getLastError());
+            } else {
+                Kernel32Library.MemoryBasicInformation memInfo = new Kernel32Library.MemoryBasicInformation();
+                long address = 0;
+                while (kernel.VirtualQueryEx(process, new Pointer(address), memInfo, memInfo.size()) != 0) {
+                    boolean lockable = memInfo.State.longValue() == Kernel32Library.MEM_COMMIT
+                            && (memInfo.Protect.longValue() & Kernel32Library.PAGE_NOACCESS) != Kernel32Library.PAGE_NOACCESS
+                            && (memInfo.Protect.longValue() & Kernel32Library.PAGE_GUARD) != Kernel32Library.PAGE_GUARD;
+                    if (lockable) {
+                        kernel.VirtualLock(memInfo.BaseAddress, new SizeT(memInfo.RegionSize.longValue()));
+                    }
+                    // Move to the next region
+                    address += memInfo.RegionSize.longValue();
+                }
+                LOCAL_MLOCKALL = true;
+            }
+        } catch (UnsatisfiedLinkError e) {
+            // this will have already been logged by Kernel32Library, no need to repeat it
+        } finally {
+            if (process != null) {
+                kernel.CloseHandle(process);
+            }
+        }
+    }
+
     public static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
         // The console Ctrl handler is necessary on Windows platforms only.
         if (Constants.WINDOWS) {

src/main/java/org/elasticsearch/common/jna/SizeT.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.jna;
+
+import com.sun.jna.IntegerType;
+import com.sun.jna.Native;
+
+public class SizeT extends IntegerType {
+
+    public SizeT() {
+       this(0);
+    }
+
+    public SizeT(long value) {
+        super(Native.SIZE_T_SIZE, value);
+    }
+
+}