Add LZ4 compressor with Java9 perf improvements (#77153)

Java9 added a number of features that are useful to improve compression
and decompression. These include the Arrays#mismatch method and
VarHandles. This commit adds compression tools forked from the java-lz4
library which include these improvements. We hope to contribute these
changes back to the original project, however the project currently
supports Java7 so this is not possible at the moment.

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -110,7 +110,7 @@ public abstract class LicenseHeadersTask extends DefaultTask {
      * Allowed license families for this project.
      */
     @Input
-    private List<String> approvedLicenses = new ArrayList<String>(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored"));
+    private List<String> approvedLicenses = new ArrayList<String>(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored", "Apache LZ4-Java"));
     /**
      * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
      * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
@@ -154,6 +154,8 @@ public abstract class LicenseHeadersTask extends DefaultTask {
         matchers.add(subStringMatcher("BSD4 ", "Original BSD License (with advertising clause)", "All advertising materials"));
         // Apache
         matchers.add(subStringMatcher("AL   ", "Apache", "Licensed to Elasticsearch B.V. under one or more contributor"));
+        // Apache lz4-java
+        matchers.add(subStringMatcher("ALLZ4", "Apache LZ4-Java", "Copyright 2020 Adrien Grand and the lz4-java contributors"));
         // Generated resources
         matchers.add(subStringMatcher("GEN  ", "Generated", "ANTLR GENERATED CODE"));
         // Vendored Code

build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle

@@ -54,6 +54,7 @@ def projectPathsToExclude = [
   ':libs:elasticsearch-dissect',
   ':libs:elasticsearch-geo',
   ':libs:elasticsearch-grok',
+  ':libs:elasticsearch-lz4',
   ':libs:elasticsearch-nio',
   ':libs:elasticsearch-plugin-classloader',
   ':libs:elasticsearch-secure-sm',

libs/lz4/build.gradle

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.publish'
+
+dependencies {
+  api 'org.lz4:lz4-java:1.8.0'
+  api project(':libs:elasticsearch-core')
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.elasticsearch', module: 'elasticsearch-lz4'
+  }
+}
+
+tasks.named("forbiddenPatterns").configure {
+  exclude '**/*.binary'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // lz4 does not depend on core, so only jdk signatures should be checked
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreViolations(
+    // from java-lz4
+    'net.jpountz.util.UnsafeUtils'
+  )
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Compressor.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Exception;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4JavaSafeCompressor.
+ *
+ * It modifies the original implementation to use custom LZ4SafeUtils and SafeUtils implementations which
+ * include performance improvements. Additionally, instead of allocating a new hashtable for each compress
+ * call, it reuses thread-local hashtables. Comments are included to mark the changes.
+ */
+public class ESLZ4Compressor extends LZ4Compressor {
+
+    // Modified to add thread-local hash tables
+    private static final ThreadLocal<short[]> sixtyFourKBHashTable = ThreadLocal.withInitial(() -> new short[8192]);
+    private static final ThreadLocal<int[]> biggerHashTable = ThreadLocal.withInitial(() -> new int[4096]);
+
+    public static final LZ4Compressor INSTANCE = new ESLZ4Compressor();
+
+    ESLZ4Compressor() {
+    }
+
+    static int compress64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int destEnd) {
+        int srcEnd = srcOff + srcLen;
+        int srcLimit = srcEnd - 5;
+        int mflimit = srcEnd - 12;
+        int dOff = destOff;
+        int anchor = srcOff;
+        if (srcLen >= 13) {
+            // Modified to use thread-local hash table
+            short[] hashTable = sixtyFourKBHashTable.get();
+            Arrays.fill(hashTable, (short) 0);
+            int sOff = srcOff + 1;
+
+            label53:
+            while(true) {
+                int forwardOff = sOff;
+                int step = 1;
+                int var16 = 1 << LZ4Constants.SKIP_STRENGTH;
+
+                int ref;
+                int excess;
+                do {
+                    sOff = forwardOff;
+                    forwardOff += step;
+                    step = var16++ >>> LZ4Constants.SKIP_STRENGTH;
+                    if (forwardOff > mflimit) {
+                        break label53;
+                    }
+
+                    excess = LZ4Utils.hash64k(SafeUtils.readInt(src, sOff));
+                    ref = srcOff + SafeUtils.readShort(hashTable, excess);
+                    SafeUtils.writeShort(hashTable, excess, sOff - srcOff);
+                    // Modified to use explicit == false
+                } while(LZ4SafeUtils.readIntEquals(src, ref, sOff) == false);
+
+                excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
+                sOff -= excess;
+                ref -= excess;
+                int runLen = sOff - anchor;
+                int tokenOff = dOff++;
+                if (dOff + runLen + 8 + (runLen >>> 8) > destEnd) {
+                    throw new LZ4Exception("maxDestLen is too small");
+                }
+
+                if (runLen >= 15) {
+                    SafeUtils.writeByte(dest, tokenOff, 240);
+                    dOff = LZ4SafeUtils.writeLen(runLen - 15, dest, dOff);
+                } else {
+                    SafeUtils.writeByte(dest, tokenOff, runLen << 4);
+                }
+
+                LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
+                dOff += runLen;
+
+                while(true) {
+                    SafeUtils.writeShortLE(dest, dOff, (short)(sOff - ref));
+                    dOff += 2;
+                    sOff += 4;
+                    ref += 4;
+                    int matchLen = LZ4SafeUtils.commonBytes(src, ref, sOff, srcLimit);
+                    if (dOff + 6 + (matchLen >>> 8) > destEnd) {
+                        throw new LZ4Exception("maxDestLen is too small");
+                    }
+
+                    sOff += matchLen;
+                    if (matchLen >= 15) {
+                        SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | 15);
+                        dOff = LZ4SafeUtils.writeLen(matchLen - 15, dest, dOff);
+                    } else {
+                        SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen);
+                    }
+
+                    if (sOff > mflimit) {
+                        anchor = sOff;
+                        break label53;
+                    }
+
+                    SafeUtils.writeShort(hashTable, LZ4Utils.hash64k(SafeUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff);
+                    int h = LZ4Utils.hash64k(SafeUtils.readInt(src, sOff));
+                    ref = srcOff + SafeUtils.readShort(hashTable, h);
+                    SafeUtils.writeShort(hashTable, h, sOff - srcOff);
+                    // Modified to use explicit == false
+                    if (LZ4SafeUtils.readIntEquals(src, sOff, ref) == false) {
+                        anchor = sOff++;
+                        break;
+                    }
+
+                    tokenOff = dOff++;
+                    SafeUtils.writeByte(dest, tokenOff, 0);
+                }
+            }
+        }
+
+        dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
+        return dOff - destOff;
+    }
+
+    public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
+        SafeUtils.checkRange(src, srcOff, srcLen);
+        SafeUtils.checkRange(dest, destOff, maxDestLen);
+        int destEnd = destOff + maxDestLen;
+        if (srcLen < 65547) {
+            return compress64k(src, srcOff, srcLen, dest, destOff, destEnd);
+        } else {
+            int srcEnd = srcOff + srcLen;
+            int srcLimit = srcEnd - 5;
+            int mflimit = srcEnd - 12;
+            int dOff = destOff;
+            int sOff = srcOff + 1;
+            int anchor = srcOff;
+            // Modified to use thread-local hash table
+            int[] hashTable = biggerHashTable.get();
+            Arrays.fill(hashTable, srcOff);
+
+            label63:
+            while(true) {
+                int forwardOff = sOff;
+                int step = 1;
+                int var18 = 1 << LZ4Constants.SKIP_STRENGTH;
+
+                while(true) {
+                    sOff = forwardOff;
+                    forwardOff += step;
+                    step = var18++ >>> LZ4Constants.SKIP_STRENGTH;
+                    if (forwardOff <= mflimit) {
+                        int excess = LZ4Utils.hash(SafeUtils.readInt(src, sOff));
+                        int ref = SafeUtils.readInt(hashTable, excess);
+                        int back = sOff - ref;
+                        SafeUtils.writeInt(hashTable, excess, sOff);
+                        // Modified to use explicit == false
+                        if (back >= 65536 || LZ4SafeUtils.readIntEquals(src, ref, sOff) == false) {
+                            continue;
+                        }
+
+                        excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
+                        sOff -= excess;
+                        ref -= excess;
+                        int runLen = sOff - anchor;
+                        int tokenOff = dOff++;
+                        if (dOff + runLen + 8 + (runLen >>> 8) > destEnd) {
+                            throw new LZ4Exception("maxDestLen is too small");
+                        }
+
+                        if (runLen >= 15) {
+                            SafeUtils.writeByte(dest, tokenOff, 240);
+                            dOff = LZ4SafeUtils.writeLen(runLen - 15, dest, dOff);
+                        } else {
+                            SafeUtils.writeByte(dest, tokenOff, runLen << 4);
+                        }
+
+                        LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
+                        dOff += runLen;
+
+                        while(true) {
+                            SafeUtils.writeShortLE(dest, dOff, back);
+                            dOff += 2;
+                            sOff += 4;
+                            int matchLen = LZ4SafeUtils.commonBytes(src, ref + 4, sOff, srcLimit);
+                            if (dOff + 6 + (matchLen >>> 8) > destEnd) {
+                                throw new LZ4Exception("maxDestLen is too small");
+                            }
+
+                            sOff += matchLen;
+                            if (matchLen >= 15) {
+                                SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | 15);
+                                dOff = LZ4SafeUtils.writeLen(matchLen - 15, dest, dOff);
+                            } else {
+                                SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen);
+                            }
+
+                            if (sOff > mflimit) {
+                                anchor = sOff;
+                                break;
+                            }
+
+                            SafeUtils.writeInt(hashTable, LZ4Utils.hash(SafeUtils.readInt(src, sOff - 2)), sOff - 2);
+                            int h = LZ4Utils.hash(SafeUtils.readInt(src, sOff));
+                            ref = SafeUtils.readInt(hashTable, h);
+                            SafeUtils.writeInt(hashTable, h, sOff);
+                            back = sOff - ref;
+                            // Modified to use explicit == false
+                            if (back >= 65536 || LZ4SafeUtils.readIntEquals(src, ref, sOff) == false) {
+                                anchor = sOff++;
+                                continue label63;
+                            }
+
+                            tokenOff = dOff++;
+                            SafeUtils.writeByte(dest, tokenOff, 0);
+                        }
+                    }
+
+                    dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
+                    return dOff - destOff;
+                }
+            }
+        }
+    }
+
+    @Override
+    public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
+        if (src.hasArray() && dest.hasArray()) {
+            return this.compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen);
+        } else {
+            throw new AssertionError("Do not support compression on direct buffers");
+        }
+    }
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4JavaSafeFastDecompressor.
+ *
+ * It modifies the original implementation to use custom LZ4SafeUtils and SafeUtils implementations which
+ * include performance improvements.
+ */
+public class ESLZ4Decompressor extends LZ4FastDecompressor {
+    public static final LZ4FastDecompressor INSTANCE = new ESLZ4Decompressor();
+
+    ESLZ4Decompressor() {
+    }
+
+    public int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
+        SafeUtils.checkRange(src, srcOff);
+        SafeUtils.checkRange(dest, destOff, destLen);
+        if (destLen == 0) {
+            if (SafeUtils.readByte(src, srcOff) != 0) {
+                throw new LZ4Exception("Malformed input at " + srcOff);
+            } else {
+                return 1;
+            }
+        } else {
+            int destEnd = destOff + destLen;
+            int sOff = srcOff;
+            int dOff = destOff;
+
+            while(true) {
+                int token = SafeUtils.readByte(src, sOff) & 255;
+                ++sOff;
+                int literalLen = token >>> 4;
+                if (literalLen == 15) {
+                    byte len;
+                    for(boolean var11 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; literalLen += 255) {
+                    }
+
+                    literalLen += len & 255;
+                }
+
+                int literalCopyEnd = dOff + literalLen;
+                if (literalCopyEnd > destEnd - 8) {
+                    if (literalCopyEnd != destEnd) {
+                        throw new LZ4Exception("Malformed input at " + sOff);
+                    } else {
+                        LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
+                        sOff += literalLen;
+                        return sOff - srcOff;
+                    }
+                }
+
+                LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
+                sOff += literalLen;
+                int matchDec = SafeUtils.readShortLE(src, sOff);
+                sOff += 2;
+                int matchOff = literalCopyEnd - matchDec;
+                if (matchOff < destOff) {
+                    throw new LZ4Exception("Malformed input at " + sOff);
+                }
+
+                int matchLen = token & 15;
+                if (matchLen == 15) {
+                    byte len;
+                    for(boolean var15 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; matchLen += 255) {
+                    }
+
+                    matchLen += len & 255;
+                }
+
+                matchLen += 4;
+                int matchCopyEnd = literalCopyEnd + matchLen;
+                if (matchCopyEnd > destEnd - 8) {
+                    if (matchCopyEnd > destEnd) {
+                        throw new LZ4Exception("Malformed input at " + sOff);
+                    }
+
+                    LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, literalCopyEnd, matchLen);
+                } else {
+                    LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, literalCopyEnd, matchCopyEnd);
+                }
+
+                dOff = matchCopyEnd;
+            }
+        }
+    }
+
+    public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen) {
+        if (src.hasArray() && dest.hasArray()) {
+            return this.decompress(src.array(), srcOff + src.arrayOffset(), dest.array(), destOff + dest.arrayOffset(), destLen);
+        } else {
+            throw new AssertionError("Do not support decompression on direct buffers");
+        }
+    }
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Constants.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.elasticsearch.lz4;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4Constants.
+ *
+ * There are no modifications. It is copied to this package for reuse as the original implementation is
+ * package private.
+ */
+enum LZ4Constants {
+    ;
+
+    static final int DEFAULT_COMPRESSION_LEVEL = 8+1;
+    static final int MAX_COMPRESSION_LEVEL = 16+1;
+
+    static final int MEMORY_USAGE = 14;
+    static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6;
+
+    static final int MIN_MATCH = 4;
+
+    static final int HASH_LOG = MEMORY_USAGE - 2;
+    static final int HASH_TABLE_SIZE = 1 << HASH_LOG;
+
+    static final int SKIP_STRENGTH = Math.max(NOT_COMPRESSIBLE_DETECTION_LEVEL, 2);
+    static final int COPY_LENGTH = 8;
+    static final int LAST_LITERALS = 5;
+    static final int MF_LIMIT = COPY_LENGTH + MIN_MATCH;
+    static final int MIN_LENGTH = MF_LIMIT + 1;
+
+    static final int MAX_DISTANCE = 1 << 16;
+
+    static final int ML_BITS = 4;
+    static final int ML_MASK = (1 << ML_BITS) - 1;
+    static final int RUN_BITS = 8 - ML_BITS;
+    static final int RUN_MASK = (1 << RUN_BITS) - 1;
+
+    static final int LZ4_64K_LIMIT = (1 << 16) + (MF_LIMIT - 1);
+    static final int HASH_LOG_64K = HASH_LOG + 1;
+    static final int HASH_TABLE_SIZE_64K = 1 << HASH_LOG_64K;
+
+    static final int HASH_LOG_HC = 15;
+    static final int HASH_TABLE_SIZE_HC = 1 << HASH_LOG_HC;
+    static final int OPTIMAL_ML = ML_MASK - 1 + MIN_MATCH;
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java

@@ -0,0 +1,242 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.util.Utils;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.util.Arrays;
+
+import static org.elasticsearch.lz4.LZ4Constants.LAST_LITERALS;
+import static org.elasticsearch.lz4.LZ4Constants.ML_BITS;
+import static org.elasticsearch.lz4.LZ4Constants.ML_MASK;
+import static org.elasticsearch.lz4.LZ4Constants.RUN_MASK;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4SafeUtils.
+ *
+ * It modifies the original implementation to use Java9 array mismatch method and varhandle performance
+ * improvements. Comments are included to mark the changes.
+ */
+enum LZ4SafeUtils {
+    ;
+
+    // Added VarHandle
+    private static final VarHandle intPlatformNative = MethodHandles.byteArrayViewVarHandle(int[].class, Utils.NATIVE_BYTE_ORDER);
+    private static final VarHandle longPlatformNative = MethodHandles.byteArrayViewVarHandle(long[].class, Utils.NATIVE_BYTE_ORDER);
+
+    static int hash(byte[] buf, int i) {
+        return LZ4Utils.hash(SafeUtils.readInt(buf, i));
+    }
+
+    static int hash64k(byte[] buf, int i) {
+        return LZ4Utils.hash64k(SafeUtils.readInt(buf, i));
+    }
+
+    static boolean readIntEquals(byte[] buf, int i, int j) {
+        return SafeUtils.readInt(buf, i) == SafeUtils.readInt(buf, j);
+    }
+
+    static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) {
+        for (int i = 0; i < matchLen; ++i) {
+            dest[dOff + i] = dest[matchOff + i];
+        }
+    }
+
+    // Modified wildIncrementalCopy to mirror version in LZ4UnsafeUtils
+    static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) {
+        if (dOff - matchOff < 4) {
+            for (int i = 0; i < 4; ++i) {
+                dest[dOff + i] = dest[matchOff + i];
+            }
+            dOff += 4;
+            matchOff += 4;
+            int dec = 0;
+            assert dOff >= matchOff && dOff - matchOff < 8;
+            switch (dOff - matchOff) {
+                case 1:
+                    matchOff -= 3;
+                    break;
+                case 2:
+                    matchOff -= 2;
+                    break;
+                case 3:
+                    matchOff -= 3;
+                    dec = -1;
+                    break;
+                case 5:
+                    dec = 1;
+                    break;
+                case 6:
+                    dec = 2;
+                    break;
+                case 7:
+                    dec = 3;
+                    break;
+                default:
+                    break;
+            }
+
+            copy4Bytes(dest, matchOff, dest, dOff);
+            dOff += 4;
+            matchOff -= dec;
+        } else if (dOff - matchOff < LZ4Constants.COPY_LENGTH) {
+            copy8Bytes(dest, matchOff, dest, dOff);
+            dOff += dOff - matchOff;
+        }
+        while (dOff < matchCopyEnd) {
+            copy8Bytes(dest, matchOff, dest, dOff);
+            dOff += 8;
+            matchOff += 8;
+        }
+    }
+
+    // Modified to use VarHandle
+    static void copy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) {
+        longPlatformNative.set(dest, dOff, (long) longPlatformNative.get(src, sOff));
+    }
+
+    // Added to copy single int
+    static void copy4Bytes(byte[] src, int sOff, byte[] dest, int dOff) {
+        intPlatformNative.set(dest, dOff, (int) intPlatformNative.get(src, sOff));
+    }
+
+    // Modified to use Arrays.mismatch
+    static int commonBytes(byte[] b, int o1, int o2, int limit) {
+        int mismatch = Arrays.mismatch(b, o1, limit, b, o2, limit);
+        return mismatch == -1 ? limit : mismatch;
+    }
+
+    static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) {
+        int count = 0;
+        while (o1 > l1 && o2 > l2 && b[--o1] == b[--o2]) {
+            ++count;
+        }
+        return count;
+    }
+
+    static void safeArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) {
+        System.arraycopy(src, sOff, dest, dOff, len);
+    }
+
+    static void wildArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) {
+        try {
+            for (int i = 0; i < len; i += 8) {
+                copy8Bytes(src, sOff + i, dest, dOff + i);
+            }
+            // Modified to catch IndexOutOfBoundsException instead of ArrayIndexOutOfBoundsException.
+            // VarHandles throw IndexOutOfBoundsException
+        } catch (IndexOutOfBoundsException e) {
+            throw new LZ4Exception("Malformed input at offset " + sOff);
+        }
+    }
+
+    static int encodeSequence(byte[] src, int anchor, int matchOff, int matchRef, int matchLen, byte[] dest, int dOff, int destEnd) {
+        final int runLen = matchOff - anchor;
+        final int tokenOff = dOff++;
+
+        if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
+            throw new LZ4Exception("maxDestLen is too small");
+        }
+
+        int token;
+        if (runLen >= RUN_MASK) {
+            token = (byte) (RUN_MASK << ML_BITS);
+            dOff = writeLen(runLen - RUN_MASK, dest, dOff);
+        } else {
+            token = runLen << ML_BITS;
+        }
+
+        // copy literals
+        wildArraycopy(src, anchor, dest, dOff, runLen);
+        dOff += runLen;
+
+        // encode offset
+        final int matchDec = matchOff - matchRef;
+        dest[dOff++] = (byte) matchDec;
+        dest[dOff++] = (byte) (matchDec >>> 8);
+
+        // encode match len
+        matchLen -= 4;
+        if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
+            throw new LZ4Exception("maxDestLen is too small");
+        }
+        if (matchLen >= ML_MASK) {
+            token |= ML_MASK;
+            dOff = writeLen(matchLen - RUN_MASK, dest, dOff);
+        } else {
+            token |= matchLen;
+        }
+
+        dest[tokenOff] = (byte) token;
+
+        return dOff;
+    }
+
+    static int lastLiterals(byte[] src, int sOff, int srcLen, byte[] dest, int dOff, int destEnd) {
+        final int runLen = srcLen;
+
+        if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) {
+            throw new LZ4Exception();
+        }
+
+        if (runLen >= RUN_MASK) {
+            dest[dOff++] = (byte) (RUN_MASK << ML_BITS);
+            dOff = writeLen(runLen - RUN_MASK, dest, dOff);
+        } else {
+            dest[dOff++] = (byte) (runLen << ML_BITS);
+        }
+        // copy literals
+        System.arraycopy(src, sOff, dest, dOff, runLen);
+        dOff += runLen;
+
+        return dOff;
+    }
+
+    static int writeLen(int len, byte[] dest, int dOff) {
+        while (len >= 0xFF) {
+            dest[dOff++] = (byte) 0xFF;
+            len -= 0xFF;
+        }
+        dest[dOff++] = (byte) len;
+        return dOff;
+    }
+
+    static class Match {
+        int start, ref, len;
+
+        void fix(int correction) {
+            start += correction;
+            ref += correction;
+            len -= correction;
+        }
+
+        int end() {
+            return start + len;
+        }
+    }
+
+    static void copyTo(LZ4SafeUtils.Match m1, LZ4SafeUtils.Match m2) {
+        m2.len = m1.len;
+        m2.start = m1.start;
+        m2.ref = m1.ref;
+    }
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.elasticsearch.lz4;
+
+import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG;
+import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG_64K;
+import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG_HC;
+import static org.elasticsearch.lz4.LZ4Constants.MIN_MATCH;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4Utils.
+ *
+ * There are no modifications. It is copied to this package for reuse as the original implementation is
+ * package private.
+ */
+enum LZ4Utils {
+    ;
+
+    private static final int MAX_INPUT_SIZE = 0x7E000000;
+
+    static int maxCompressedLength(int length) {
+        if (length < 0) {
+            throw new IllegalArgumentException("length must be >= 0, got " + length);
+        } else if (length >= MAX_INPUT_SIZE) {
+            throw new IllegalArgumentException("length must be < " + MAX_INPUT_SIZE);
+        }
+        return length + length / 255 + 16;
+    }
+
+    static int hash(int i) {
+        return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG);
+    }
+
+    static int hash64k(int i) {
+        return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_64K);
+    }
+
+    static int hashHC(int i) {
+        return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_HC);
+    }
+
+    static class Match {
+        int start, ref, len;
+
+        void fix(int correction) {
+            start += correction;
+            ref += correction;
+            len -= correction;
+        }
+
+        int end() {
+            return start + len;
+        }
+    }
+
+    static void copyTo(LZ4Utils.Match m1, LZ4Utils.Match m2) {
+        m2.len = m1.len;
+        m2.start = m1.start;
+        m2.ref = m1.ref;
+    }
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/SafeUtils.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.util.Utils;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteOrder;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.SafeUtils.
+ *
+ * It modifies the original implementation to use Java9 varhandle performance improvements. Comments
+ * are included to mark the changes.
+ */
+public enum SafeUtils {
+    ;
+
+    // Added VarHandle
+    private static final VarHandle intPlatformNative = MethodHandles.byteArrayViewVarHandle(int[].class, Utils.NATIVE_BYTE_ORDER);
+
+    private static final VarHandle shortLittleEndian = MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.LITTLE_ENDIAN);
+
+    public static void checkRange(byte[] buf, int off) {
+        if (off < 0 || off >= buf.length) {
+            throw new ArrayIndexOutOfBoundsException(off);
+        }
+    }
+
+    public static void checkRange(byte[] buf, int off, int len) {
+        checkLength(len);
+        if (len > 0) {
+            checkRange(buf, off);
+            checkRange(buf, off + len - 1);
+        }
+    }
+
+    public static void checkLength(int len) {
+        if (len < 0) {
+            throw new IllegalArgumentException("lengths must be >= 0");
+        }
+    }
+
+    public static byte readByte(byte[] buf, int i) {
+        return buf[i];
+    }
+
+    // Deleted unused dedicated LE/BE readInt methods
+
+    // Modified to use VarHandle
+    public static int readInt(byte[] buf, int i) {
+        return (int) intPlatformNative.get(buf, i);
+    }
+
+    // Unused in forked instance, no need to optimize
+    public static long readLongLE(byte[] buf, int i) {
+        return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24)
+            | ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56);
+    }
+
+    // Modified to use VarHandle
+    public static void writeShortLE(byte[] buf, int off, int v) {
+        shortLittleEndian.set(buf, off, (short) v);
+    }
+
+    public static void writeInt(int[] buf, int off, int v) {
+        buf[off] = v;
+    }
+
+    public static int readInt(int[] buf, int off) {
+        return buf[off];
+    }
+
+    public static void writeByte(byte[] dest, int off, int i) {
+        dest[off] = (byte) i;
+    }
+
+    public static void writeShort(short[] buf, int off, int v) {
+        buf[off] = (short) v;
+    }
+
+    // Modified to use VarHandle
+    public static int readShortLE(byte[] buf, int i) {
+        return Short.toUnsignedInt((short) shortLittleEndian.get(buf, i));
+    }
+
+    public static int readShort(short[] buf, int off) {
+        return buf[off] & 0xFFFF;
+    }
+}

libs/lz4/src/main/java/org/elasticsearch/lz4/package-info.java

@@ -0,0 +1,20 @@
+/* @notice
+ *
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+
+package org.elasticsearch.lz4;

libs/lz4/src/test/java/org/elasticsearch/lz4/AbstractLZ4TestCase.java

@@ -0,0 +1,379 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4CompressorWithLength;
+import net.jpountz.lz4.LZ4DecompressorWithLength;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.AbstractLZ4Test.
+ *
+ * It modifies the abstract test case to only test byte arrays and byte array backed byte buffers. These are
+ * the only bytes we support.
+ */
+abstract class AbstractLZ4TestCase extends ESTestCase {
+
+    public interface TesterBase<T> {
+
+        T allocate(int length);
+        T copyOf(byte[] array);
+        byte[] copyOf(T data, int off, int len);
+        int maxCompressedLength(int len);
+        void fill(T instance, byte b);
+
+        // Modified to remove redundant modifiers
+        class ByteArrayTesterBase implements TesterBase<byte[]> {
+
+            @Override
+            public byte[] allocate(int length) {
+                return new byte[length];
+            }
+
+            @Override
+            public byte[] copyOf(byte[] array) {
+                return Arrays.copyOf(array, array.length);
+            }
+
+            @Override
+            public byte[] copyOf(byte[] data, int off, int len) {
+                return Arrays.copyOfRange(data, off, off + len);
+            }
+
+            @Override
+            public int maxCompressedLength(int len) {
+                return LZ4Utils.maxCompressedLength(len);
+            }
+
+            @Override
+            public void fill(byte[] instance, byte b) {
+                Arrays.fill(instance, b);
+            }
+        }
+
+        // Modified to remove redundant modifiers
+        class ByteBufferTesterBase implements TesterBase<ByteBuffer> {
+
+            @Override
+            public ByteBuffer allocate(int length) {
+                ByteBuffer bb;
+                int slice = randomInt(5);
+                // Modified to only test heap ByteBuffers
+                bb = ByteBuffer.allocate(length + slice);
+                bb.position(slice);
+                bb = bb.slice();
+                if (randomBoolean()) {
+                    bb.order(ByteOrder.LITTLE_ENDIAN);
+                } else {
+                    bb.order(ByteOrder.BIG_ENDIAN);
+                }
+                return bb;
+            }
+
+            @Override
+            public ByteBuffer copyOf(byte[] array) {
+                ByteBuffer bb = allocate(array.length).put(array);
+                // Modified to not test read only buffers as they do not make the array accessible
+                bb.position(0);
+                return bb;
+            }
+
+            @Override
+            public byte[] copyOf(ByteBuffer data, int off, int len) {
+                byte[] copy = new byte[len];
+                data.position(off);
+                data.get(copy);
+                return copy;
+            }
+
+            @Override
+            public int maxCompressedLength(int len) {
+                return LZ4Utils.maxCompressedLength(len);
+            }
+
+            @Override
+            public void fill(ByteBuffer instance, byte b) {
+                for (int i = 0; i < instance.capacity(); ++i) {
+                    instance.put(i, b);
+                }
+            }
+        }
+    }
+
+    public interface Tester<T> extends TesterBase<T> {
+
+        int compress(LZ4Compressor compressor, T src, int srcOff, int srcLen, T dest, int destOff, int maxDestLen);
+        int decompress(LZ4FastDecompressor decompressor, T src, int srcOff, T dest, int destOff, int destLen);
+        int decompress(LZ4SafeDecompressor decompressor, T src, int srcOff, int srcLen, T dest, int destOff, int maxDestLen);
+
+        // Modified to remove redundant modifiers
+        class ByteArrayTester extends ByteArrayTesterBase implements Tester<byte[]> {
+
+            @Override
+            public int compress(LZ4Compressor compressor, byte[] src, int srcOff,
+                                int srcLen, byte[] dest, int destOff, int maxDestLen) {
+                return compressor.compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor,
+                                  byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
+                return decompressor.decompress(src, srcOff, dest, destOff, destLen);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor,
+                                  byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
+                return decompressor.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+        }
+        // Modified to remove redundant modifiers
+        Tester<byte[]> BYTE_ARRAY = new ByteArrayTester();
+        // Modified to remove redundant modifiers
+        Tester<byte[]> BYTE_ARRAY_WITH_LENGTH = new ByteArrayTester() {
+            @Override
+            public int compress(LZ4Compressor compressor, byte[] src, int srcOff,
+                                int srcLen, byte[] dest, int destOff, int maxDestLen) {
+                return new LZ4CompressorWithLength(compressor).compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor,
+                                  byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, dest, destOff);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor,
+                                  byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, srcLen, dest, destOff);
+            }
+        };
+
+        // Modified to remove redundant modifiers
+        class ByteBufferTester extends ByteBufferTesterBase implements Tester<ByteBuffer> {
+
+            @Override
+            public int compress(LZ4Compressor compressor, ByteBuffer src, int srcOff,
+                                int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
+                return compressor.compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src,
+                                  int srcOff, ByteBuffer dest, int destOff, int destLen) {
+                return decompressor.decompress(src, srcOff, dest, destOff, destLen);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src,
+                                  int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
+                return decompressor.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+        }
+        // Modified to remove redundant modifiers
+        Tester<ByteBuffer> BYTE_BUFFER = new ByteBufferTester();
+        // Modified to remove redundant modifiers
+        Tester<ByteBuffer> BYTE_BUFFER_WITH_LENGTH = new ByteBufferTester() {
+            @Override
+            public int compress(LZ4Compressor compressor, ByteBuffer src, int srcOff,
+                                int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
+                return new LZ4CompressorWithLength(compressor).compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src,
+                                  int srcOff, ByteBuffer dest, int destOff, int destLen) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, dest, destOff);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src,
+                                  int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, srcLen, dest, destOff);
+            }
+        };
+    }
+
+    // Tester to test a simple compress/decompress(src, dest) type of APIs
+    public interface SrcDestTester<T> extends TesterBase<T> {
+
+        int compress(LZ4Compressor compressor, T src, T dest);
+        int decompress(LZ4FastDecompressor decompressor, T src, T dest);
+        int decompress(LZ4SafeDecompressor decompressor, T src, T dest);
+
+        // Modified to remove redundant modifiers
+        class ByteArrayTester extends ByteArrayTesterBase implements SrcDestTester<byte[]> {
+
+            @Override
+            public int compress(LZ4Compressor compressor, byte[] src, byte[] dest) {
+                return compressor.compress(src, dest);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, byte[] src, byte[] dest) {
+                return decompressor.decompress(src, dest);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, byte[] src, byte[] dest) {
+                return decompressor.decompress(src, dest);
+            }
+        }
+        // Modified to remove redundant modifiers
+        SrcDestTester<byte[]> BYTE_ARRAY = new ByteArrayTester();
+        // Modified to remove redundant modifiers
+        SrcDestTester<byte[]> BYTE_ARRAY_WITH_LENGTH = new ByteArrayTester() {
+            @Override
+            public int compress(LZ4Compressor compressor, byte[] src, byte[] dest) {
+                return new LZ4CompressorWithLength(compressor).compress(src, dest);
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, byte[] src, byte[] dest) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, dest);
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, byte[] src, byte[] dest) {
+                return new LZ4DecompressorWithLength(decompressor).decompress(src, dest);
+            }
+        };
+
+        // Modified to remove redundant modifiers
+        class ByteBufferTester extends ByteBufferTesterBase implements SrcDestTester<ByteBuffer> {
+
+            @Override
+            public int compress(LZ4Compressor compressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = dest.position();
+                compressor.compress(src, dest);
+                return dest.position() - pos;
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = src.position();
+                decompressor.decompress(src, dest);
+                return src.position() - pos;
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = dest.position();
+                decompressor.decompress(src, dest);
+                return dest.position() - pos;
+            }
+        }
+        // Modified to remove redundant modifiers
+        SrcDestTester<ByteBuffer> BYTE_BUFFER = new ByteBufferTester();
+        // Modified to remove redundant modifiers
+        SrcDestTester<ByteBuffer> BYTE_BUFFER_WITH_LENGTH = new ByteBufferTester() {
+            @Override
+            public int compress(LZ4Compressor compressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = dest.position();
+                new LZ4CompressorWithLength(compressor).compress(src, dest);
+                return dest.position() - pos;
+            }
+
+            @Override
+            public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = src.position();
+                new LZ4DecompressorWithLength(decompressor).decompress(src, dest);
+                return src.position() - pos;
+            }
+
+            @Override
+            public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, ByteBuffer dest) {
+                final int pos = dest.position();
+                new LZ4DecompressorWithLength(decompressor).decompress(src, dest);
+                return dest.position() - pos;
+            }
+        };
+    }
+
+    protected class RandomBytes {
+        private final byte[] bytes;
+        RandomBytes(int n) {
+            assert n > 0 && n <= 256;
+            bytes = new byte[n];
+            for (int i = 0; i < n; ++i) {
+                bytes[i] = (byte) randomInt(255);
+            }
+        }
+        byte next() {
+            final int i = randomInt(bytes.length - 1);
+            return bytes[i];
+        }
+    }
+
+    protected static byte[] readResource(String resource) throws IOException {
+        InputStream is = AbstractLZ4TestCase.class.getResourceAsStream(resource);
+        if (is == null) {
+            throw new IllegalStateException("Cannot find " + resource);
+        }
+        byte[] buf = new byte[4096];
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            while (true) {
+                final int read = is.read(buf);
+                if (read == -1) {
+                    break;
+                }
+                baos.write(buf, 0, read);
+            }
+        } finally {
+            is.close();
+        }
+        return baos.toByteArray();
+    }
+
+    protected byte[] randomArray(int len, int n) {
+        byte[] result = new byte[len];
+        RandomBytes randomBytes = new RandomBytes(n);
+        for (int i = 0; i < result.length; ++i) {
+            result[i] = randomBytes.next();
+        }
+        return result;
+    }
+
+    protected ByteBuffer copyOf(byte[] bytes, int offset, int len) {
+        ByteBuffer buffer;
+        // Modified to only test heap ByteBuffers
+        buffer = ByteBuffer.allocate(bytes.length);
+        buffer.put(bytes);
+        buffer.position(offset);
+        buffer.limit(offset + len);
+        if (randomBoolean()) {
+            buffer = buffer.asReadOnlyBuffer();
+        }
+        return buffer;
+    }
+}

libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Compressor;
+
+import net.jpountz.lz4.LZ4Factory;
+
+import net.jpountz.lz4.LZ4FastDecompressor;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class ESLZ4CompressorTests extends ESTestCase {
+
+    public void testCompressRealisticUnicode() {
+        for (int i = 0; i < 15; ++i) {
+            int stringLengthMultiplier = randomFrom(5, 10, 20, 40, 80, 160, 320);
+
+            final String uncompressedString = randomRealisticUnicodeOfCodepointLength(stringLengthMultiplier * 1024);
+            byte[] uncompressed = uncompressedString.getBytes(StandardCharsets.UTF_8);
+
+            byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            byte[] unForkedCompressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            LZ4Compressor compressor = ESLZ4Compressor.INSTANCE;
+            int forkedCompressedSize = compressor.compress(uncompressed, compressed);
+            LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor();
+            int unForkedCompressedSize = unForkedCompressor.compress(uncompressed, unForkedCompressed);
+            assertEquals(unForkedCompressedSize, forkedCompressedSize);
+            assertArrayEquals(compressed, unForkedCompressed);
+
+            LZ4FastDecompressor decompressor = LZ4Factory.safeInstance().fastDecompressor();
+            byte[] output = new byte[uncompressed.length];
+            decompressor.decompress(compressed, output);
+
+            assertArrayEquals(uncompressed, output);
+        }
+    }
+
+    public void testCompressRandomIntBytes() throws IOException {
+        for (int i = 0; i < 15; ++i) {
+            int uncompressedBytesLength = randomFrom(16, 32, 64, 128, 256, 512, 1024) * 1024;
+
+            BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(uncompressedBytesLength);
+            for (int j = 0; j < uncompressedBytesLength / 4; ++j) {
+                bytesStreamOutput.writeInt(randomFrom(0, 1, randomInt()));
+            }
+            byte[] uncompressed = new byte[uncompressedBytesLength];
+            bytesStreamOutput.bytes().streamInput().read(uncompressed);
+
+            byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            byte[] unForkedCompressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            LZ4Compressor compressor = ESLZ4Compressor.INSTANCE;
+            int forkedCompressedSize = compressor.compress(uncompressed, compressed);
+            LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor();
+            int unForkedCompressedSize = unForkedCompressor.compress(uncompressed, unForkedCompressed);
+            assertEquals(unForkedCompressedSize, forkedCompressedSize);
+            assertArrayEquals(unForkedCompressed, compressed);
+
+            LZ4FastDecompressor decompressor = LZ4Factory.safeInstance().fastDecompressor();
+            byte[] output = new byte[uncompressed.length];
+            decompressor.decompress(compressed, output);
+
+            assertArrayEquals(uncompressed, output);
+        }
+    }
+}

libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class ESLZ4DecompressorTests extends ESTestCase {
+
+    public void testDecompressRealisticUnicode() {
+        for (int i = 0; i < 15; ++i) {
+            int stringLengthMultiplier = randomFrom(5, 10, 20, 40, 80, 160, 320);
+
+            final String uncompressedString = randomRealisticUnicodeOfCodepointLength(stringLengthMultiplier * 1024);
+            byte[] uncompressed = uncompressedString.getBytes(StandardCharsets.UTF_8);
+
+            byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            LZ4Compressor compressor = LZ4Factory.safeInstance().fastCompressor();
+            int unForkedDestinationBytes = compressor.compress(uncompressed, compressed);
+
+            LZ4FastDecompressor decompressor = ESLZ4Decompressor.INSTANCE;
+            byte[] output = new byte[uncompressed.length];
+            int forkedDestinationBytes = decompressor.decompress(compressed, output);
+
+            assertEquals(unForkedDestinationBytes, forkedDestinationBytes);
+            assertArrayEquals(uncompressed, output);
+        }
+    }
+
+    public void testDecompressRandomBytes() throws IOException {
+        for (int i = 0; i < 15; ++i) {
+            int uncompressedBytesLength = randomFrom(16, 32, 64, 128, 256, 512, 1024) * 1024;
+
+            BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(uncompressedBytesLength);
+            for (int j = 0; j < uncompressedBytesLength / 4; ++j) {
+                bytesStreamOutput.writeInt(randomFrom(0, 1, randomInt()));
+            }
+            byte[] uncompressed = new byte[uncompressedBytesLength];
+            bytesStreamOutput.bytes().streamInput().read(uncompressed);
+
+            byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16];
+            LZ4Compressor compressor = LZ4Factory.safeInstance().fastCompressor();
+            int unForkedDestinationBytes = compressor.compress(uncompressed, compressed);
+
+            LZ4FastDecompressor decompressor = ESLZ4Decompressor.INSTANCE;
+            byte[] output = new byte[uncompressed.length];
+            int forkedDestinationBytes = decompressor.decompress(compressed, output);
+
+            assertEquals(unForkedDestinationBytes, forkedDestinationBytes);
+            assertArrayEquals(uncompressed, output);
+        }
+    }
+}

libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4Tests.java

@@ -0,0 +1,510 @@
+/*
+ * Copyright 2020 Adrien Grand and the lz4-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+
+package org.elasticsearch.lz4;
+
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
+ * net.jpountz.lz4.LZ4Test.
+ *
+ * It modifies the test case to remove unneeded tests (safe decompressor, native libs, etc). Additionally,
+ * we only test our compressor/decompressor and the pure java "safe" lz4-java compressor/decompressor.
+ * Finally, on any "round-trip" tests we compress data using the safe lz4-java instance and compare that the
+ * compression is the same as the compressor instance we are testing.
+ */
+public class ESLZ4Tests extends AbstractLZ4TestCase {
+
+    // Modified to only test ES decompressor instances
+    static LZ4FastDecompressor[] FAST_DECOMPRESSORS = new LZ4FastDecompressor[] {
+        ESLZ4Decompressor.INSTANCE
+    };
+
+    // Modified to not test any SAFE_DECOMPRESSORS, as we do not support it
+    static LZ4SafeDecompressor[] SAFE_DECOMPRESSORS = new LZ4SafeDecompressor[0];
+
+    // Modified to delete testMaxCompressedLength which requires native library. Additionally, we do not
+    // modify the maxCompressedLength logic
+
+    private static byte[] getCompressedWorstCase(byte[] decompressed) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        int len = decompressed.length;
+        if (len >= LZ4Constants.RUN_MASK) {
+            baos.write(LZ4Constants.RUN_MASK << LZ4Constants.ML_BITS);
+            len -= LZ4Constants.RUN_MASK;
+            while (len >= 255) {
+                baos.write(255);
+                len -= 255;
+            }
+            baos.write(len);
+        } else {
+            baos.write(len << LZ4Constants.ML_BITS);
+        }
+        try {
+            baos.write(decompressed);
+        } catch (IOException e) {
+            throw new AssertionError();
+        }
+        return baos.toByteArray();
+    }
+
+    public void testEmpty() {
+        testRoundTrip(new byte[0]);
+    }
+
+    public void testUncompressWorstCase(LZ4FastDecompressor decompressor) {
+        final int len = randomInt(100 * 1024);
+        final int max = randomIntBetween(1, 255);
+        byte[] decompressed = randomArray(len, max);
+        byte[] compressed = getCompressedWorstCase(decompressed);
+        byte[] restored = new byte[decompressed.length];
+        int cpLen = decompressor.decompress(compressed, 0, restored, 0, decompressed.length);
+        assertEquals(compressed.length, cpLen);
+        assertArrayEquals(decompressed, restored);
+    }
+
+    public void testUncompressWorstCase() {
+        for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) {
+            testUncompressWorstCase(decompressor);
+        }
+    }
+
+    public void testUncompressWorstCase(LZ4SafeDecompressor decompressor) {
+        final int len = randomInt(100 * 1024);
+        final int max = randomIntBetween(1, 256);
+        byte[] decompressed = randomArray(len, max);
+        byte[] compressed = getCompressedWorstCase(decompressed);
+        byte[] restored = new byte[decompressed.length];
+        int uncpLen = decompressor.decompress(compressed, 0, compressed.length, restored, 0);
+        assertEquals(decompressed.length, uncpLen);
+        assertArrayEquals(decompressed, restored);
+    }
+
+    // Modified to delete testUncompressSafeWorstCase (we do not have "safe" decompressor)
+
+    // Modified to only test 1 (fast) decompressor
+    public void testRoundTrip(byte[] data, int off, int len,
+                              LZ4Compressor compressor,
+                              LZ4FastDecompressor decompressor) {
+        for (Tester<?> tester : Arrays.asList(Tester.BYTE_ARRAY, Tester.BYTE_BUFFER, Tester.BYTE_ARRAY_WITH_LENGTH,
+            Tester.BYTE_BUFFER_WITH_LENGTH)) {
+            testRoundTrip(tester, data, off, len, compressor, decompressor);
+        }
+        if (data.length == len && off == 0) {
+            for (SrcDestTester<?> tester : Arrays.asList(SrcDestTester.BYTE_ARRAY, SrcDestTester.BYTE_BUFFER,
+                SrcDestTester.BYTE_ARRAY_WITH_LENGTH,
+                SrcDestTester.BYTE_BUFFER_WITH_LENGTH)) {
+                testRoundTrip(tester, data, compressor, decompressor);
+            }
+        }
+    }
+
+    // Modified to only test 1 (fast) decompressor
+    public <T> void testRoundTrip(
+        Tester<T> tester,
+        byte[] data, int off, int len,
+        LZ4Compressor compressor,
+        LZ4FastDecompressor decompressor) {
+        final int maxCompressedLength = tester.maxCompressedLength(len);
+        // "maxCompressedLength + 1" for the over-estimated compressed length test below
+        final T compressed = tester.allocate(maxCompressedLength + 1);
+        final int compressedLen = tester.compress(compressor,
+            tester.copyOf(data), off, len,
+            compressed, 0, maxCompressedLength);
+
+        // Modified to compress using an unforked lz4-java compressor and verify that the results are same.
+        T expectedCompressed = tester.allocate(maxCompressedLength + 1);
+        LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor();
+        final int expectedCompressedLen = tester.compress(unForkedCompressor,
+            tester.copyOf(data), off, len,
+            expectedCompressed, 0, maxCompressedLength);
+        assertEquals(expectedCompressedLen, compressedLen);
+        assertArrayEquals(tester.copyOf(expectedCompressed, 0, expectedCompressedLen), tester.copyOf(compressed, 0, compressedLen));
+
+
+        // test decompression
+        final T restored = tester.allocate(len);
+        assertEquals(compressedLen, tester.decompress(decompressor, compressed, 0, restored, 0, len));
+        assertArrayEquals(Arrays.copyOfRange(data, off, off + len), tester.copyOf(restored, 0, len));
+
+        // make sure it fails if the compression dest is not large enough
+        tester.fill(restored, randomByte());
+        final T compressed2 = tester.allocate(compressedLen-1);
+        try {
+            final int compressedLen2 = tester.compress(compressor,
+                tester.copyOf(data), off, len,
+                compressed2, 0, compressedLen - 1);
+            // Compression can succeed even with the smaller dest
+            // because the compressor is allowed to return different compression results
+            // even when it is invoked with the same input data.
+            // In this case, just make sure the compressed data can be successfully decompressed.
+            assertEquals(compressedLen2, tester.decompress(decompressor, compressed2, 0, restored, 0, len));
+            assertArrayEquals(Arrays.copyOfRange(data, off, off + len), tester.copyOf(restored, 0, len));
+        } catch (LZ4Exception e) {
+            // OK
+        }
+
+        if (tester != Tester.BYTE_ARRAY_WITH_LENGTH && tester != Tester.BYTE_BUFFER_WITH_LENGTH) {
+            // LZ4DecompressorWithLength will succeed in decompression
+            // because it ignores destLen.
+
+            if (len > 0) {
+                // decompression dest is too small
+                try {
+                    tester.decompress(decompressor, compressed, 0, restored, 0, len - 1);
+                    fail();
+                } catch (LZ4Exception e) {
+                    // OK
+                }
+            }
+
+            // decompression dest is too large
+            final T restored2 = tester.allocate(len+1);
+            try {
+                final int cpLen = tester.decompress(decompressor, compressed, 0, restored2, 0, len + 1);
+                fail("compressedLen=" + cpLen);
+            } catch (LZ4Exception e) {
+                // OK
+            }
+        }
+
+        // Modified to delete "safe" decompressor tests
+    }
+
+    // Modified to only test 1 (fast) decompressor
+    public <T> void testRoundTrip(SrcDestTester<T> tester,
+                                  byte[] data,
+                                  LZ4Compressor compressor,
+                                  LZ4FastDecompressor decompressor) {
+        final T original = tester.copyOf(data);
+        final int maxCompressedLength = tester.maxCompressedLength(data.length);
+        final T compressed = tester.allocate(maxCompressedLength);
+        final int compressedLen = tester.compress(compressor,
+            original,
+            compressed);
+        if (original instanceof ByteBuffer) {
+            assertEquals(data.length, ((ByteBuffer)original).position());
+            assertEquals(compressedLen, ((ByteBuffer)compressed).position());
+            ((ByteBuffer)original).rewind();
+            ((ByteBuffer)compressed).rewind();
+        }
+
+        // test decompression
+        final T restored = tester.allocate(data.length);
+        assertEquals(compressedLen, tester.decompress(decompressor, compressed, restored));
+        if (original instanceof ByteBuffer) {
+            assertEquals(compressedLen, ((ByteBuffer)compressed).position());
+            assertEquals(data.length, ((ByteBuffer)restored).position());
+        }
+        assertArrayEquals(data, tester.copyOf(restored, 0, data.length));
+        if (original instanceof ByteBuffer) {
+            ((ByteBuffer)compressed).rewind();
+            ((ByteBuffer)restored).rewind();
+        }
+
+        // Modified to delete "safe" decompressor tests
+    }
+
+    // Modified to delete unnecessary method
+
+    public void testRoundTrip(byte[] data, int off, int len) {
+        // Modified to only test safe instance and forked instance
+        for (LZ4Compressor compressor : Arrays.asList(LZ4Factory.safeInstance().fastCompressor(), ESLZ4Compressor.INSTANCE)) {
+            for (LZ4FastDecompressor decompressor : Arrays.asList(LZ4Factory.safeInstance().fastDecompressor(),
+                ESLZ4Decompressor.INSTANCE)) {
+                testRoundTrip(data, off, len, compressor, decompressor);
+            }
+        }
+    }
+
+    public void testRoundTrip(byte[] data) {
+        testRoundTrip(data, 0, data.length);
+    }
+
+    public void testRoundTrip(String resource) throws IOException {
+        final byte[] data = readResource(resource);
+        testRoundTrip(data);
+    }
+
+    public void testRoundtripGeo() throws IOException {
+        // Modified path to point at resource
+        testRoundTrip("calgary/geo.binary");
+    }
+
+    public void testRoundtripBook1() throws IOException {
+        // Modified path to point at resource
+        testRoundTrip("calgary/book1");
+    }
+
+    public void testRoundtripPic() throws IOException {
+        // Modified path to point at resource
+        testRoundTrip("calgary/pic.binary");
+    }
+
+    public void testNullMatchDec() {
+        // 1 literal, 4 matchs with matchDec=0, 8 literals
+        final byte[] invalid = new byte[] { 16, 42, 0, 0, (byte) 128, 42, 42, 42, 42, 42, 42, 42, 42 };
+        // decompression should neither throw an exception nor loop indefinitely
+        for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) {
+            decompressor.decompress(invalid, 0, new byte[13], 0, 13);
+        }
+        for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) {
+            decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0);
+        }
+    }
+
+    public void testEndsWithMatch() {
+        // 6 literals, 4 matchs
+        final byte[] invalid = new byte[] { 96, 42, 43, 44, 45, 46, 47, 5, 0 };
+        final int decompressedLength = 10;
+
+        for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) {
+            try {
+                // it is invalid to end with a match, should be at least 5 literals
+                decompressor.decompress(invalid, 0, new byte[decompressedLength], 0, decompressedLength);
+                assertTrue(decompressor.toString(), false);
+            } catch (LZ4Exception e) {
+                // OK
+            }
+        }
+
+        for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) {
+            try {
+                // it is invalid to end with a match, should be at least 5 literals
+                decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0);
+                assertTrue(false);
+            } catch (LZ4Exception e) {
+                // OK
+            }
+        }
+    }
+
+    public void testEndsWithLessThan5Literals() {
+        // 6 literals, 4 matchs
+        final byte[] invalidBase = new byte[] { 96, 42, 43, 44, 45, 46, 47, 5, 0 };
+
+        for (int i = 1; i < 5; ++i) {
+            final byte[] invalid = Arrays.copyOf(invalidBase, invalidBase.length + 1 + i);
+            invalid[invalidBase.length] = (byte) (i << 4); // i literals at the end
+
+            for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) {
+                try {
+                    // it is invalid to end with a match, should be at least 5 literals
+                    decompressor.decompress(invalid, 0, new byte[20], 0, 20);
+                    assertTrue(decompressor.toString(), false);
+                } catch (LZ4Exception e) {
+                    // OK
+                }
+            }
+
+            for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) {
+                try {
+                    // it is invalid to end with a match, should be at least 5 literals
+                    decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0);
+                    assertTrue(false);
+                } catch (LZ4Exception e) {
+                    // OK
+                }
+            }
+        }
+    }
+
+    // Modified to delete testWriteToReadOnlyBuffer. We only compress to byte arrays so this test is
+    // unnecessary
+
+    public void testAllEqual() {
+        // Modified to not use @Repeat
+        for (int i = 0; i < 5; ++i) {
+            final int len = randomBoolean() ? randomInt(20) : randomInt(100000);
+            final byte[] buf = new byte[len];
+            Arrays.fill(buf, randomByte());
+            testRoundTrip(buf);
+        }
+    }
+
+    public void testMaxDistance() {
+        final int len = randomIntBetween(1 << 17, 1 << 18);
+        final int off = randomInt(len - (1 << 16) - (1 << 15));
+        final byte[] buf = new byte[len];
+        for (int i = 0; i < (1 << 15); ++i) {
+            buf[off + i] = randomByte();
+        }
+        System.arraycopy(buf, off, buf, off + 65535, 1 << 15);
+        testRoundTrip(buf);
+    }
+
+    public void testRandomData() {
+        // Modified to not use @Repeat
+        for (int i = 0; i < 10; ++i) {
+            final int n = randomIntBetween(1, 15);
+            final int off = randomInt(1000);
+            final int len = randomBoolean() ? randomInt(1 << 16) : randomInt(1 << 20);
+            final byte[] data = randomArray(off + len + randomInt(100), n);
+            testRoundTrip(data, off, len);
+        }
+    }
+
+    // https://github.com/jpountz/lz4-java/issues/12
+    public void testRoundtripIssue12() {
+        byte[] data = new byte[]{
+            14, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 72, 14, 72, 14, 85, 3, 72, 14, 72, 14, 72, 14, 72, 14, 72, 14, 72, 14, 85, 3, 72,
+            14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 50, 64, 0, 46, -1, 0, 0, 0, 29, 3, 85,
+            8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3,
+            0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113,
+            0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113,
+            0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 50, 64, 0, 47, -105, 0, 0, 0, 30, 3, -97, 6, 0, 68, -113,
+            0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85,
+            8, -113, 0, 68, -97, 3, 0, 2, -97, 6, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97,
+            6, 0, 68, -113, 0, 120, 64, 0, 48, 4, 0, 0, 0, 31, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72,
+            33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72,
+            43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72,
+            28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72,
+            35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72,
+            41, 72, 32, 72, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            0, 39, 24, 32, 34, 124, 0, 120, 64, 0, 48, 80, 0, 0, 0, 31, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72,
+            35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72,
+            41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72,
+            40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72,
+            31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72,
+            26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72,
+            37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72,
+            36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72,
+            20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72,
+            22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72,
+            38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72,
+            29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72,
+            27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 50, 64, 0, 49, 20, 0, 0, 0, 32, 3, -97, 6, 0,
+            68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97,
+            6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2,
+            3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2,
+            3, -97, 6, 0, 50, 64, 0, 50, 53, 0, 0, 0, 34, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97,
+            6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3,
+            -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97,
+            3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3,
+            85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0,
+            2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3,
+            -97, 6, 0, 50, 64, 0, 51, 85, 0, 0, 0, 36, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97,
+            6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, -97, 5, 0, 2, 3, 85, 8, -113, 0, 68,
+            -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0,
+            68, -113, 0, 2, 3, -97, 6, 0, 50, -64, 0, 51, -45, 0, 0, 0, 37, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6,
+            0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97,
+            6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 120, 64, 0, 52, -88, 0, 0,
+            0, 39, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72,
+            13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85,
+            5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85,
+            5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85,
+            5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 72,
+            13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, -19, -24, -101, -35
+        };
+        testRoundTrip(data, 9, data.length - 9);
+    }
+
+    private static void assertCompressedArrayEquals(String message, byte[] expected, byte[] actual) {
+        int off = 0;
+        int decompressedOff = 0;
+        while (true) {
+            if (off == expected.length) {
+                break;
+            }
+            final Sequence sequence1 = readSequence(expected, off);
+            final Sequence sequence2 = readSequence(actual, off);
+            assertEquals(message + ", off=" + off + ", decompressedOff=" + decompressedOff, sequence1, sequence2);
+            off += sequence1.length;
+            decompressedOff += sequence1.literalLen + sequence1.matchLen;
+        }
+    }
+
+    private static Sequence readSequence(byte[] buf, int off) {
+        final int start = off;
+        final int token = buf[off++] & 0xFF;
+        int literalLen = token >>> 4;
+        if (literalLen >= 0x0F) {
+            int len;
+            while ((len = buf[off++] & 0xFF) == 0xFF) {
+                literalLen += 0xFF;
+            }
+            literalLen += len;
+        }
+        off += literalLen;
+        if (off == buf.length) {
+            return new Sequence(literalLen, -1, -1, off - start);
+        }
+        int matchDec = (buf[off++] & 0xFF) | ((buf[off++] & 0xFF) << 8);
+        int matchLen = token & 0x0F;
+        if (matchLen >= 0x0F) {
+            int len;
+            while ((len = buf[off++] & 0xFF) == 0xFF) {
+                matchLen += 0xFF;
+            }
+            matchLen += len;
+        }
+        matchLen += 4;
+        return new Sequence(literalLen, matchDec, matchLen, off - start);
+    }
+
+    private static class Sequence {
+        final int literalLen, matchDec, matchLen, length;
+
+        private Sequence(int literalLen, int matchDec, int matchLen, int length) {
+            this.literalLen = literalLen;
+            this.matchDec = matchDec;
+            this.matchLen = matchLen;
+            this.length = length;
+        }
+
+        @Override
+        public String toString() {
+            return "Sequence [literalLen=" + literalLen + ", matchDec=" + matchDec
+                + ", matchLen=" + matchLen + "]";
+        }
+
+        @Override
+        public int hashCode() {
+            return 42;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            Sequence other = (Sequence) obj;
+            if (literalLen != other.literalLen)
+                return false;
+            if (matchDec != other.matchDec)
+                return false;
+            if (matchLen != other.matchLen)
+                return false;
+            return true;
+        }
+
+    }
+}

libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/README

@@ -0,0 +1 @@
+Data from the Calgary corpus (http://corpus.canterbury.ac.nz/descriptions/#calgary).

libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/book1

@@ -0,0 +1,1377 @@
+<Y 1874>
+<A T. HARDY>
+<T Madding Crowd(Penguin 1978)>
+<C i>
+<P 51>
+DESCRIPTION OF FARMER OAK -- AN INCIDENT
+When Farmer Oak smiled, the corners of his mouth
+spread till they were within an unimportant distance of
+his ears, his eyes were reduced to chinks, and diverging
+wrinkles appeared round them, extending upon his
+countenance like the rays in a rudimentary sketch of
+the rising sun.
+His Christian name was Gabriel, and on working
+days he was a young man of sound judgment, easy
+motions, proper dress, and general good character. On
+Sundays he was a man of misty views, rather given to
+postponing, and hampered by his best clothes and
+umbrella : upon the whole, one who felt himself to
+occupy morally that vast middle space of Laodicean
+neutrality which lay between the Communion people
+of the parish and the drunken section, -- that is, he went
+to church, but yawned privately by the time the con+
+gegation reached the Nicene creed,- and thought of
+what there would be for dinner when he meant to be
+listening to the sermon. Or, to state his character as
+it stood in the scale of public opinion, when his friends
+and critics were in tantrums, he was considered rather a
+bad man ; when they were pleased, he was rather a good
+man ; when they were neither, he was a man whose
+moral colour was a kind of pepper-and-salt mixture.
+Since he lived six times as many working-days as
+Sundays, Oak's appearance in his old clothes was most
+peculiarly his own -- the mental picture formed by his
+neighbours in imagining him being always dressed in
+that way. He wore a low-crowned felt hat, spread out
+at the base by tight jamming upon the head for security
+in high winds, and a coat like Dr. Johnson's ; his lower
+extremities being encased in ordinary leather leggings
+and boots emphatically large, affording to each foot a
+roomy apartment so constructed that any wearer might
+stand in a river all day long and know nothing of
+<P 52>
+damp -- their maker being a conscientious man who
+endeavoured to compensate for any weakness in his cut
+by unstinted dimension and solidity.
+Mr. Oak 'carried 'about him, by way of watch,+
+what may be called a small silver clock; in other
+words, it was a watch as to shape and intention, and
+a small clock as to size. This instrument being several
+years older than Oak's grandfather, had the peculiarity
+of going either too fast or not at all. The smaller
+of its hands, too, occasionally slipped round on the
+pivot, and thus, though the minutes were told with
+precision, nobody could be quite certain of the hour
+they belonged to. The stopping peculiarity of his
+watch Oak remedied by thumps and shakes, and he
+escaped any evil consequences from the other two
+defects by constant comparisons with and observations
+of the sun and stars, and by pressing his face close
+to the glass of his neighbours' windows, till he could
+discern the hour marked by the green-faced timekeepers
+within. It may be mentioned that Oak's fob being
+difficult of access, by reason of its somewhat high
+situation in the waistband of his trousers (which also
+lay at a remote height under his waistcoat), the watch
+was as a necessity pulled out by throwing the body to
+one-side, compressing the- mouth and face to a mere
+mass of- ruddy flesh- on account -of the exertion, and
+drawing up the watch by its chain, like a bucket from a
+well.
+But some thoughtfull persons, who had seen him
+walking across one of his fields on a certain December
+morning -- sunny and exceedingly mild -- might have
+regarded Gabriel Oak in other aspects than these. In
+his face one might notice that many of the hues and
+curves of youth had tarried on to manhood: there even
+remained in his remoter crannies some relics of the boy.
+His height and breadth would have been sufficient to
+make his presence imposing, had they been exhibited
+with due consideration. But there is a way some men
+have, rural and urban alike, for which the mind is more
+responsible than flesh and sinew : it is a way of curtail+
+ing their dimensions by their manner of showing them.
+And from a quiet modesty that would have become a
+vestal which seemed continually to impress upon him
+<P 53>
+that he had no great claim on the world's room, Oak
+walked unassumingly and with a faintly perceptible
+bend, yet distinct from a bowing of the shoulders.
+This may be said to be a defect in an individual if he
+depends for his valuation more upon his appearance
+than upon his capacity to wear well, which Oak did not.
+He had just reached the time of life at which " young'
+is ceasing to be the prefix of "man ' in speaking of one.
+He was at the brightest period of masculine growth,
+for his intellect and his emotions were clearly separated :
+he had passed the time during which the influence of
+youth indiscriminately mingles them in the character
+of impulse, and he had not yet arrived at the stage
+wherein they become united again, in the character of
+prejudice, by the influence of a wife and family. In
+short, he was twenty-eight, and a bachelor.
+The field he was in this morning sloped to a
+ridge called Norcombe Hill. Through a spur of this
+hill ran the highway between Emminster and Chalk+
+Newton. Casually glancing over the hedge, Oak saw
+coming down the incline before him an ornamental
+spring waggon, painted yellow and gaily marked,
+drawn by two horses, a waggoner walking alongside
+bearing a whip perpendicularly. The waggon was
+laden with household goods and window plants, and
+on the apex of the whole sat a woman, 'young-'and
+attractive. Gabriel had not beheld the sight for more
+than half a minute, when the vehicle was brought to a
+standstill just beneath his eyes.
+" The tailboard of the waggon is gone, Miss,' said the
+waggoner.
+"Then I heard it fall,' said the girl, in a soft, though
+not particularly low voice. "I heard a noise I could
+not account for when we were coming up the hill.'
+"I'll run back.' +
+" Do,' she answered. +
+The sensible horses stood -- perfectly still, and the
+waggoner's steps sank fainter and fainter in the distance.
+The girl on the summit of the load sat motionless,
+surrounded by tables and chairs with their legs upwards,
+backed by an oak settle, and ornamented in front by
+pots of geraniums, myrtles, and cactuses, together with
+<P 54>
+a caged canary -- all probably from the windows of the
+house just vacated. There was also a cat in a willow
+basket, from the partly-opened lid of which she gazed
+with half-closed eyes, and affectionately-surveyed the
+small birds around.
+The handsome girl waited for some time idly in her
+place, and the only sound heard in the stillness-was -the
+hopping of the canary up-and down the perches of its
+prison. Then she looked attentively downwards. It
+was not at the bird, nor at the cat; it was at an oblong
+package tied in paper, and lying between them. She
+turned her head to learn if the waggoner were coming.
+He was not yet in sight; and her-eyes crept back to
+the package, her thoughts seeming to run 'upon what
+was inside it. At length she drew the article into her
+lap, and untied the paper covering; a small swing
+looking-glass was disclosed, in which she proceeded to
+survey herself attentively. She parted her lips and
+smiled.
+It was a fine morning, and the sun lighted up to a
+scarlet glow the crimson jacket she wore, and painted
+a soft lustre upon her bright face and dark hair. The
+myrtles, geraniums, and cactuses packed around her
+were fresh and green, and at such a leafless season they
+invested the whole concern of horses, waggon, furniture,
+and girl with a peculiar vernal charm. What possessed
+her to indulge in such a performance in the sight of the
+sparrows, blackbirds, and unperceived farmer who were
+alone its spectators, -- whether the smile began as a
+factitious one, to test her capacity in that art, -- nobody
+knows ; it ended certainly in a real smile. She blushed
+at herself, and seeing her reflection blush, blushed the
+more.
+The change from the customary spot and necessary
+occasion of such an act -- from the dressing hour in a
+bedroom to a time of travelling out of doors -- lent to
+the idle deed a novelty it did not intrinsically possess.
+The picture was a delicate one. Woman's prescriptive
+infirmity had stalked into the sunlight, which had
+clothed it in the freshness of an originality. A
+cynical inference was irresistitle by Gabriel Oak as he
+regarded the scene, generous though he fain would have
+been. There was no necessity whatever for her looking
+in the glass. She did not adjust her hat, or pat her
+<P 55>
+hair, or press a dimple into shape, or do one thing to
+signify that any such intention had been her motive in
+taking up the glass. She simply observed herself as a
+fair product of Nature in the feminine kind, her thoughts
+seeming to glide into far-off though likely dramas in
+which men would play a part -- vistas of probable
+triumphs -- the smiles being of a phase suggesting that
+hearts were imagined as lost and won. Still, this was
+but conjecture, and the whole series of actions was so
+idly put forth as to make it rash to assert that intention
+had any part in them at all.
+The waggoner's steps were heard returning. She
+put the glass in the paper, and the whole again into its
+place.
+When the waggon had passed on, Gabriel withdrew
+from his point of espial, and descending into the road,
+followed the vehicle to the turnpike-gate some way
+beyond the bottom of the hill, where the object of his
+contemplation now halted for the payment of toll. About
+twenty steps still remained between him and the gate,
+when he heard a dispute. lt was a difference con+
+cerning twopence between the persons with the waggon
+and the man at the toll-bar.
+" Mis'ess's niece is upon the top of the things, and
+she says that's enough that I've offered ye, you great
+miser, and she won't pay any more.' These were the
+waggoner's words.
+"Very well ; then mis'ess's niece can't pass,' said the
+turnpike-keeper, closing the gate.
+Oak looked from one to the other of the disputants,
+and fell into a reverie. There was something in the
+tone of twopence remarkably insignificant. Threepence
+had a definite value as money -- it was an appreciable
+infringement on a day's wages, and, as such, a higgling
+matter ; but twopence --  -- " Here,' he said, stepping
+forward and handing twopence to the gatekeeper ; "let
+the young woman pass.' He looked up at her then;
+she heard his words, and looked down.
+Gabriel's features adhered throughout their form so
+exactly to the middle line between the beauty of St.
+John and the ugliness of Judas Iscariot, as represented
+in a window of the church he attended, that not a single
+lineament could be selected and called worthy either of
+distinction or notoriety. The redjacketed and dark+
+<P 56>
+haired maiden seemed to think so too, for she carelessly
+glanced over him, and told her man to drive on. She
+might have looked her thanks to Gabriel on a minute
+scale, but she did not speak them; more probably she
+felt none, for in gaining her a passage he had lost her
+her point, and we know how women take a favour of
+that kind.
+The gatekeeper surveyed the retreating vehicle.
+" That's a handsome maid ' he said to Oak
+" But she has her faults,' said Gabriel.
+" True, farmer. '
+"And the greatest of them is -- well, what it is
+always.'
+" Beating people down ? ay, 'tis so.'
+"O no.'
+" What, then ? '
+Gabriel, perhaps a little piqued by the comely
+traveller's indifference, glanced back to where he had
+witnessed her performance over the hedge, and said,
+" Vanity.'
+<C ii>
+<P 57>
+NIGHT -- THE FLOCK -- AN INIERIOR -- ANOTHER INTERIOR
+IT was nearly midnight on the eve of St. Thomas"s, the
+shortest day in the year. A desolating wind wandered
+from the north over the hill whereon Oak had watched
+the yellow waggon and its occupant in the sunshine of
+a few days earlier.
+  Norcombe Hill -- not far from lonely Toller-Down
+ -- was one of the spots which suggest to a passer-by
+that he is in the presence of a shape approaching the
+indestructible as nearly as any to be found on earth.
+It was a featureless convexity of chalk and soil -- an
+ordinary specimen of those smoothly-outlined protuber+
+ances of the globe which may remain undisturbed on
+some great day of confusion, when far grander heights
+and dizzy granite precipices topple down.
+The hill was covered on its northern side by an
+ancient and decaying plantation of beeches, whose
+upper verge formed a line over the crest, fringing its
+arched curve against the sky, like a mane. To-night
+these trees sheltered the southern slope from the keenest
+blasts, which smote the wood and floundered through
+it with a sound as of grumbling, or gushed over its
+crowning boughs in a weakened moan. The dry leaves
+in the ditch simmered and boiled in the same breezes,
+a tongue of air occasionally ferreting out a few, and
+sending them spinning across the grass. A group or
+two of the latest in date amongst the dead multitude
+had remained till this very mid-winter time on the twigs
+which bore them and in falling rattled against the trunks
+with smart taps:
+Betwenne this half-wooded, half naked hill, and the
+vague still horizon that its summit indistinctly com+
+manded, was a mysterious sheet of fathomless shade
+ -- the sounds from which suggested that what it con+
+cealed bore some reduced resemblance to features here.
+<P 58>
+The thin grasses, more or less coating the hill, were
+touched by the wind in breezes of differing powers, and
+almost of differing natures -- one rubbing the blades
+heavily, another raking them piercingly, another brushing
+them like a soft broom. The instinctive act of human+
+kind was to stand and listen, and learn how the trees
+to each other in the regular antiphonies of a cathedral
+choir; how hedges and other shapes to leeward them
+caught the note, lowering it to the tenderest sob; and
+how the hurrying gust then plunged into the south, to
+be heard no more.
+The sky was clear -- remarkably clear -- and the
+twinkling of all the stars seemed to be but throbs of
+one body, timed by a common pulse. The North Star
+was directly in the wind's eye, and since evening the
+Bear had swung round it outwardly to the east, till he
+was now at a right angle with the meridian. A
+difference of colour in the stars -- oftener read of than
+seen in England-was really perceptible here. The
+sovereign brilliancy of Sirius pierced the eye with a steely
+glitter, the star called Capella was yellow, Aldebaran and
+Betelgueux shone with a fiery red.
+To persons standing alone on a hill during a clear
+midnight such as this, the roll of the world eastward is
+almost a palpable movement. The sensation may be
+caused by the panoramic glide of the stars past earthly
+objects, which is perceptible in a few minutes of still+
+ness, or by the better outlook upon space that a hill
+affords, or by the wind, or by the solitude ; but whatever
+be its origin, the impression of riding along is vivid and
+abiding. The poetry of motion is a phrase much in
+use, and to enjoy the epic form of that gratification it
+is necessary to stand on a hill at a small hour of the
+night, and, having first expanded with a sense of differ+
+ence from the mass of civilised mankind, who are
+dreamwrapt and disregardful of all such proceedings at
+this time, long and quietly watch your stately progress
+through the stars. After such a nocturnal reconnoitre
+it is hard to get back to earth, and to believe that the
+consciousness of such majestic speeding is derived from
+a tiny human frame.
+Suddenly an unexpected series of sounds began to
+<P 59>
+be heard
+in this place up against the sky. They had a
+clearness which was to be found nowhere in the wind,
+and a sequence which was to be found nowhere in
+nature. They were the notes of Farmer Oak's flute.
+The tune was not floating unhindered into the open
+air : it seemed muffled in some way, and was altogether
+too curtailed in power to spread high or wide. It came
+from the direction of a small dark object under the
+plantation hedge -- a shepherd's hut -- now presenting
+an outline to which an uninitiated person might have
+been puzzled to attach either meaning or use.
+The image as a whole was that of a small Noah's
+Ark on a small Ararat, allowing the traditionary outlines
+and general form of the Ark which are followed by toy+
+makers -- and by these means are established in men's
+imaginations among their firmest, because earliest im+
+pressions -- to pass as an approximate pattern. The
+hut stood on little wheels, which raised its floor about a
+foot from the ground. Such shepherds' huts are dragged
+into the fields when the lambing season comes on, to
+shelter the shepherd in his- enforced nightly attendance.
+It was only latterly that people had begun to call
+Gabriel !Farmer' Oak. During the twelvemonth pre+
+ceding this time he had been enabled by sustained
+efforts of industry and chronic good spirits to lease the
+small shepp farm of which Norcombe Hill was a portion,
+and stock it with two hundred sheep. Previously he
+had been a bailiff for a short time, and earlier still a
+shepherd only, having from his childhood assisted his
+father in tending the floeks of large proprietors, till old
+Gabriel sank to rest.
+This venture, unaided and alone, into the paths of
+farming as master and not as man, with an advance of
+sheep not yet paid for, was a critical juncture with
+Gabriel Oak, and he recognised his position clearly.
+The first movement in his new progress was the lambing
+of his ewes, and sheep having been his speciality from
+his "youth, he wisely refrained from deputing -- the task
+of tending them at this season to a hireling or a novice.
+The wind continued to beat-about the corners of the
+hut, but the flute-playing ceased. A rectangular space
+of light
+<P 60>
+appeared in the side of the hut, and in the
+opening the outline of Farmer Oak's figure. He carried
+a lantern in his hand, and closing the door behind him,
+came forward and busied himself about this nook of the
+field for nearly twenty minutes, the lantern light appear+
+ing and disappearing here and there, and brightening
+him or darkening him as he stood before or behind it.
+Oak's motions, though they had a quiet-energy, were
+slow, and their deliberateness accorded well with his
+occupation. Fitness being the basis of beauty, nobody
+could-have denied that his steady swings and turns"
+in and- about the flock had elements of grace, Yet,
+although if occasion demanded he could do or think a
+thing with as mercurial a dash as can the men of towns
+who are more to the manner born, his special power,
+morally, physically, and mentally, was static, owing
+little or nothing to momentum as a rule.
+A close examination of the ground hereabout, even
+by the wan starlight only, revealed how a portion of
+what would have been casually called a wild slope had
+been appropriated by Farmer Oak for his great purpose
+this winter. Detached hurdles thatched with straw
+were stuck into the ground at various scattered points,
+amid and under which the whitish forms of his meek
+ewes moved and rustled. The ring of the sheep-bell,
+which had been silent during his absence, recommenced,
+in tones that had more mellowness than clearness, owing
+to an increasing growth of surrounding wool. This
+continued till Oak withdrew again from the flock. He
+ --  returned to the hut, bringing in his arms a new-born
+lamb, consisting of four legs large enough for a full+
+grown sheep, united by a seemingly inconsiderable mem+
+brane about half the substance of the legs collectively,
+which constituted the animal's entire body just at present.
+The little speck of life he placed on a wisp of hay
+before the small stove, where a can of milk was simmer+
+ing. Oak extinguished the lantern by blowing into it
+and then pinching the snuff, the cot being lighted
+by a candle suspended by a twisted wire. A rather
+hard couch, formed of a few corn sacks thrown carelessly
+down, covered half the floor of this little
+<P 61>
+habitation, and
+here the young man stretched himself along, loosened
+his woollen cravat, and closed his eyes. In about the
+time a person unaccustomed to bodily labour would have
+decided upon which side to lie, Farmer Oak was asleep.
+The inside of the hut, as it now presented itself, was
+cosy and alluring, and the scarlet handful of fire in
+addition to the candle, reflecting its own genial colour
+upon whatever it could reach, flung associations of
+enjoyment even over utensils and tools. In the corner
+stood the sheep-crook, and along a shelf at one side
+were ranged bottles and canisters of the simple prepara+
+tions pertaining to ovine surgery and physic; spirits of
+wine, turpentine, tar, magnesia, ginger, and castor-oil
+being the chief. On a triangular shelf across the corner
+stood bread, bacon, cheese, and a cup for ale or cider,
+which was supplied from a flagon beneath. Beside the
+provisions lay the flute whose notes had lately been
+called forth by the lonely watcher to beguile a tedious
+hour. The house was ventilated by two round holes,
+like the lights of a ship's cabin, with wood slides+
+The lamb, revived by the warmth' began to bleat'
+instant meaning, as expected sounds will. Passing
+from the profoundest sleep to the most alert wakefulness
+with the same ease that had accompanied the reverse
+operation, he looked at his watch, found that the hour+
+hand had shifted again, put on his hat, took the lamb
+in his arms, and carried it into the darkness. After
+placing the little creature with its mother, he stood and
+carefully examined the sky, to ascertain the time of
+night from the altitudes of the stars.
+The Dog-star and Aldebaran, pointing to the restless
+Pleiades, were half-way up the Southern sky, and between
+them hung Orion, which gorgeous constellation never
+burnt more vividly than now, as it soared forth above
+the rim of the landscape. Castor and Pollux will
+the north-west; far away through the plantation Vega
+and Cassiopeia's chair stood daintily poised on the
+uppermost boughs.
+<P 62>
+"One o'clock,' said Gabriel.
+Being a man not without a frequent consciousness
+that there was some charm in this life he led, he stood
+still after looking at the sky as a useful instrument, and
+regarded it in an appreciative spirit, as a work of art
+superlatively beautiful. For a moment he seemed
+impressed with the speaking loneliness of the scene, or
+rather with the complete abstraction from all its compass
+of the sights and sounds of man. Human shapes,interferences,
+troubles, and joys were all as if they were not, and there
+seemed to be on the shaded hemisphere of the globe no sentient being
+save himself; he could fancy them all gone round to the sunny side.
+  Occupied this, with eyes stretched afar, Oak gradually per+
+ceived that what he had previously taken to be a star low
+down behind the outskirts of the plantation was in reality no
+such thing. It was an artificial light, almost close at hand.
+ To find themselves utterly alone at night where company
+is desirable and expected makes some people fearful; but a
+case more trying by far to the nerves is to discover some
+mysterious companionship when intuition, sensation, memory,
+analogy, testimony, probability, induction -- every kind of
+evidence in the logician's list -- have united to persuade con+
+sciousness that it is quite in isolation.
+ Farmer Oak went towards the plantation and pushed
+through its lower boughs to the windy side. A dim mass under
+the slope reminded him that a shed occupied a place here,
+the site being a cutting into the slope of the hill, so that at
+its back part the roof was almost level with the ground. In
+front it was formed of board nailed to posts and covered with
+tar as apreservative. Through crevices in the roof and side
+spread streaks and spots of light, a combination of which made
+the radiance that had attracted him. Oak stepped up behind,
+where,leaning down upon the roof and putting his eye close
+to a hole, he could see into the interior clearly.
+ The place contained two women and two cows. By the side
+of the latter a steaming bran-mash stood in a bucket. One
+of the women was past middle age. Her companion was ap+
+parently young and graceful; he could form no decided opinion
+<P 63>
+upon her looks, her position being almost beneath his eye, so
+that he saw her in a bird's-eye view, as Milton's Satan first saw
+Paradise. She wore no bonnet or het, but had enveloped her+
+self in a large cloak, which was carelessly flung over her head
+as a covering.
+ "There, now we'll go home," said the elder of the two, resting
+ her knuckles upon her hips, and looking at their goings-on as
+a whole. "I do hope Daisy will fetch round again now. I have
+never been more frightened in my life, but I don't mind break+
+ing my rest if she recovers."
+ The young woman, whose eyelids were apparently inclined
+to fall together on the smallest provocation of silence,yawned
+in sympathy.
+ "I wish we were rich enough to pay a man to do these
+things," she said.
+ "As we are not, we must do them ourselves," said the other;
+"for you must help me if you stay."
+"Well, my hat is gone, however," continued the younger. "It
+went over the hedge, I think. The idea of such a slight wind
+catching it."
+ The cow standing erect was of the Devon breed, and was
+encased in a tight warm hide of rich Indian red, as absolutely
+uniform from eyes to tail as if the animal had been dipped in
+a dye of that colour, her long back being mathematically level.
+The other was spotted,grey and white. Beside her Oak now
+noticed a little calf about a day old, looking idiotically at
+the two women, which showed that it had not long been
+accustomed to the phenomenon of eyesight, and often turn+
+ing to the lantern, which it apparently mistook for the moon.
+inherited instinct having as yet had little time for correction
+by experience. Between the sheep and the cows Lucina had
+been busy on Norcombe hill lately.
+ "I think we had better send for some oatmeal," said the
+"Yes, aunt; and I'll ride over for it as soon as it is
+light. '
+" But there's no side-saddle.'
+<P 64>
+"I can ride on the other : trust me.'
+Oak, upon hearing these remarks, became more
+curious to observe her features, but this prospect being
+denied him by the hooding efect of the cloak, and by his
+aerial position, he felt himself drawing upon his fancy
+for their details. In making even horizontal and clear
+inspections we colour and mould according to the warts
+within us whatever our eyes bring in. Had Gabriel
+been able from the first to get a distinct view of her +
+countenance, his estimate of it as very handsome or
+slightly so would have been as his soul required a
+divinity at the moment or was ready supplied with one.
+Having for some time known the want of a satisfactory
+form to fill an increasing void within him, his position
+moreover affording the widest scope for his fancy, he
+painted her a beauty.
+By one of those whimsical coincidences in which
+Nature, like a busy mother, seems to spare a moment
+from her unremitting labours to turn and make her
+children smile, the girl now dropped the cloak, and
+forth tumbled ropes of black hair over a red jacket.
+Oak knew her instantly as the heroine of the yellow
+waggon, myrtles, and looking-glass : prosily, as the
+woman who owed him twopence.
+They placed the calf beside its mother again, took
+up the lantern, and went out, the light sinking down
+the hill till it was no more than a nebula. Gabriel
+Oak returned to his flock.
+<C iii>
+<P 65>
+A GIRL ON HORSEBACK -- CONVERSATION
+THE sluggish day began to break. Even its position
+terrestrially is one of the elements of a new interest,
+and for no particular reason save that the incident of
+the night had occurred there, Oak went again into
+the plantation. Lingering and musing here, he heard
+the steps of a horse at the foot of the hill, and soon
+there appeared in view an auburn pony with a girl on
+its back, ascending by the path leading past the cattle+
+shed. She was the young woman of the night before.
+Gabriel instantly thought of the hat she had mentioned
+as having lost in the wind; possibly she had come to
+look for it. He hastily scanned the ditch and after
+walking about ten yards along it, found the hat among the
+leaves. Gabriel took it in his hand and returned to his
+hut. Here he ensconced himself, and peeped through
+the loophole in the direction of the riders approach.
+She came up and looked around -- then on the other
+side of the hedge. Gabriel was about to advance and
+restore the missing article when an unexpected per+
+formance induced him to suspend the action for the
+present. The path, after passing the cowshed, bisected
+the plantation. It was not a bridle-path -- merely a
+pedestrian's track, and the boughs spread horizontally
+at a height not greater than seven feet above the ground,
+which made it impossible to ride erect beneath them.
+The girl, who wore no riding-habit, looked around for
+a moment, as if to assure herself that all humanity was
+out of view, then dexterously dropped backwards flat
+upon the pony's back, her head over its tail, her feet
+against its shoulders, and her eyes to the sky. The
+rapidity of her glide into this position was that of a
+kingfisher -- its noiselessness that of a hawk. Gabriel's
+eyes had scarcely been able to follow her. The tall lank
+pony seemed used to such doings, and ambled
+<P 66>
+along unconcerned. Thus she passed under the level boughs.
+The performer seemed quite at home anywhere
+between a horse's head and its tail, and the necessity
+for this abnormal attitude having ceased with the
+passage of the plantation, she began to adopt another,
+even more obviously convenient than the first. She had
+no side-saddle, and it was very apparent that a firm
+seat upon the smooth leather beneath her was un+
+attainable sideways. Springing to her accustomed
+perpendicular like a bowed sapling, and satisfying her,
+self that nobody was in sight, she seated herself in the
+manner demanded by the saddle, though hardly expected
+of the woman, and trotted off in the direction of Tewnell
+Mill.
+Oak was amused, perhaps a little astonished, and
+hanging up the hat in his hut, went again among his
+ewes. An hour passed, the girl returned, properly
+seated now, with a bag of bran in front of her. On
+nearing the cattle-shed she was met by a boy bringing
+a milking-pail, who held the reins of the pony whilst
+she slid off. The boy led away the horse, leaving the
+pail with the young woman.
+Soon soft spirts alternating with loud spirts came
+in regular succession from within the shed, the obvious
+sounds of a person milking a cow. Gabriel took the
+lost hat in his hand, and waited beside the path she
+would follow in leaving the hill.
+She came, the pail in one hand, hanging against her
+knee. The left arm was extended as a balance, enough
+of it being shown bare to make Oak wish that the event
+ha happened in the summer, when the whole would
+have been revealed. There was a bright air and manner
+about her now, by which she seemed to imply that the
+desirability of her existence could not be questioned;
+and this rather saucy assumption failed in being offensive,
+because a beholder felt it to be, upon the whole, true.
+Like exceptional emphasis in the tone of a genius, that
+which would have made mediocrity ridiculous was an
+addition to recognised power. It was with some
+surprise that she saw Gabriel's face rising like the
+moon behind the hedge.
+The adjustment of the farmer's hazy conceptions of
+her
+<P 67>
+charms to the portrait of herself she now presented
+him with was less a diminuition than a difference. The
+starting-point selected by the judgment was. her height
+She seemed tall, but the pail was a small one, and the
+hedge diminutive; hence, making allowance for error
+by comparison with these, she could have been not
+above the height to be chosen by women as best. All
+features of consequence were severe and regular. It
+may have been observed by persons who go about the
+shires with eyes for beauty, that in Englishwoman a
+classically-formed face is seldom found to be united
+with a figure of the same pattern, the highly-finished
+features being generally too large for the remainder of
+the frame ; that a graceful and proportionate figure of
+eight heads usually goes off into random facial curves.
+Without throwing a Nymphean tissue over a milkmaid,
+let it be said that here criticism checked itself as out
+of place, and looked at her proportions with a long
+consciousness of pleasure. From the contours of her
+figure in its upper part, she must have had a beautiful
+neek and shoulders ; but since her infancy nobody had
+ever seen them. Had she been put into a low dress
+she would have run and thrust her head into a bush.
+Yet she was not a shy girl by any means; it was merely
+her instinct to draw the line dividing the seen from the
+unseen higher than they do it in towns.
+That the girl's thoughts hovered about her face
+and form as soon as she caught Oak's eyes conning the
+same page was natural, and almost certain. The self+
+consciousness shown would have been vanity if a little
+more pronounced, dignity if a little less. Rays of male
+vision seem to have a tickling effect upon virgin faces
+in rural districts ; she brushed hers with her hand, as if
+Gabriel had been irritating its pink surface by actual
+touch, and the free air of her previous movements was
+reduced at the same time to a chastened phase of
+itself. Yet it was the man who blushed, the maid not
+at all.
+" I found a hat,' said Oak.
+" It is mine,' said she, and, from a sense of proportion,
+kept down to a small smile an inclination to laugh dis+
+tinctly : "it flew away last night.'
+" One o'clock this morning ? '
+<P 68>
+" Well -- it was.' She was surprised. " How did you
+know ? ' she said.
+" I was here.'
+" You are Farmer Oak, are you not ? '
+" That or thereabouts. I'm lately come to this place.'
+" A large farm ? ' she inquired, casting her eyes round,
+and swinging back her hair, which was black in the
+shaded hollows of its mass; but it being now an hour
+past sunrise, the rays touched its prominent curves with
+a colour of their own.
+" No ; not large. About a hundred.' (In speaking
+of farms the word "acres ' is omitted by the natives, by
+analogy to such old expressions as "a stag of ten.')
+' "I wanted my hat this morning,' she went on.
+had to ride to Tewnell Mill.'
+"Yes you had.'
+"How do you know?'
+"I saw you!
+"Where?' she inquired, a misgiving bringing every
+muscle of her lineaments and frame to a standstill.
+"Here-going through the plantation, and all down
+the hill,' said Farmer Oak, with an aspect excessively
+knowing with regard to some matter in his mind, as he
+gazed at a remote point in the direction named, and then
+turned back to meet his colloquist's eyes.
+A perception caused him to withdraw his own eyes
+from hers as suddenly as if he had been caught in a
+theft. Recollection of the strange antics she had
+indulged in when passing through the trees, was suc+
+ceeded in the girl by a nettled palpitation, and that' by
+a hot face. It was a time to see a woman redden who
+was not given to reddening s a rule; not a point in
+the milkmaid but was of the deepest rose-colour. From
+the Maiden's Blush, through all varieties of the Provence
+down to the Crimson Tuscany, the countenance of Oak's
+acquaintance quickly graduated ; whereupon he, in con+
+siderateness, turned away his head.
+The sympathetic man still looked the other way, and
+wondered when she would recover coolness sufficient to
+justify him in facing her again. He heard what seemed
+to be the flitting of a
+<P 69>
+dead leaf upon the breeze, and
+looked. She had gone away.
+With an air between that of Tragedy and Comedy !
+Gabriel returned to his work.
+Five mornings and evenings passed. The young
+woman came regularly to milk the healthy cow or to
+attend to the sick one, but never allowed her vision to
+stray in the direction of Oak's person. His want of
+tact had deeply offended her -- not by seeing what he
+could not help, but by letting her know that he had
+seen it. For, as without law there is no sin, without
+eyes there is no indecorum; and she appeared to feel
+that Gabriel's espial had made her an indecorous woman
+without her own connivance. It was food for great regret
+with him; it was also a contretemps which touched into
+life a latent heat he had experienced in that direction.
+The acquaintanceship might, however, have ended in
+a slow forgetting, but for an incident which occurred at
+the end of the same week. One afternoon it began to
+freeze, and the frost increased with evening, which drew
+on like a stealthy tightening of bonds. It was a time
+when in cottages the breath of the sleepers freezes to
+the sheets; when round the drawing-room fire of a
+thick-walled mansion the sitters' backs are cold, even
+whilst their faces are all aglow. Many a small bird went
+to bed supperless that night among the bare boughs.
+As the milking-hour drew near, Oak kept his usual
+watch upon the cowshed. At last he felt cold, and
+shaking an extra quantity of bedding round the yeaning
+ewes he entered the hut and heaped more fuel upon
+the stove. The wind came in at the bottom of the door,
+and to prevent it Oak laid a sack there and wheeled the
+cot round a little more to the south. Then the wind
+spouted in at a ventilating hole -- of which there was one
+on each side of the hut.
+Gabriel had always known that when the fire was
+lighted and the door closed one of these must be kept
+open -- that chosen being always on the side away from
+the wind. Closing the slide to windward, he turned to
+open the other; on second -- -thoughts the farmer con+
+sidered that he would first sit down leaving both
+closed for a minute or two, till the temperature of the
+hut was a little raised. He sat down.
+<P 70>
+His head began to ache in an unwonted manner, and,
+fancying himself weary by reason of the broken rests of
+the preceding nights, Oak decided to get up, open the
+slide, and then allow himself to fall asleep. He fell
+asleep, however, without having performed the necessary
+preliminary.
+How long he remained unconseious Gabriel never
+knew. During the first stages of his return to percep+
+tion peculiar deeds seemed to be in course of enactment.
+His dog was howling, his head was aching fearfully --
+somebody was pulling him about, hands were loosening
+his neckerchief.
+On opening his eyes he found that evening had sunk
+to dusk in a strange manner of unexpectedness. The
+young girl with the remarkably pleasant lips and white
+teeth was beside him. More than this -- astonishingly
+more -- his head was upon her lap, his face and neck
+were disagreeably wet, and her fingers were unbuttoning
+his collar.
+"Whatever is the matter?' said Oak, vacantly.
+She seemed to experience mirth, but of too insignifi+
+cant a kind to start enjoyment.
+"Nothing now', she answered, "since you are not
+dead It is a wonder you were not,suffocated in this
+hut of yours.'
+"Ah, the hut ! ' murmured Gabriel. "I gave ten
+pounds for that hut. But I'll sell it, and sit under
+thatched hurdles as they did in old times, curl up
+to sleep in a lock of straw! It played me nearly the
+same trick the other day .! ' Gabriel, by way of emphasis,
+brought down his fist upon the floor.
+"It was not exactly the fault of the hut,' she ob+
+served in a tone which showed her to be that novelty
+among women -- one who finished a thought before
+beginning the sentence which was to convey it. " You
+should I think, have considered, and not have been so
+foolish as to leave the slides closed.'
+"Yes I suppose I should,' said Oak, absently. He
+was endeavouring to catch and appreciate the sensation
+of being thus with her, his head upon her dress, before
+the event passed on into the heap of bygone things.
+He wished she knew his impressions ; but he would as
+soon have thought of carrying an odour in a net as of
+attempting to convey the intangibilities
+<P 71>
+of his feeling
+in the coarse meshes of language. So he remained
+silent.
+She made him sit up, and then Oak began wiping
+his face and shaking himself like a Samson. "How
+can I thank 'ee ? ' he said at last, gratefully, some of the
+natural rusty red having returned to his face.
+ " Oh, never mind that,' said the girl, smiling, and
+allowing her smile to hold good for Gabriel's next
+remark, whatever that might prove to be.
+"How did you find me?"
+"I heard your dog howling and scratching at the
+door of the hut when I came to the milking (it was so
+lucky, Daisy's milking is almost over for the season, and
+ I shall not come here after this week or the next). The
+dog saw me, and jumped over to me, and laid hold of
+my skirt. I came across and looked round the hut the
+very first thing to see if the slides were closed. My
+uncle has a hut like this one, and I have heard him tell
+his shepherd not to go to sleep without leaving a slide
+open. I opened the door, and there you were like
+dead. I threw the milk over you, as there was no
+water, forgetting it was warm, and no use.'
+"I wonder if I should have died ? ' Gabriel said, in a
+low voice, which was rather meant to travel back to
+himself than to her.
+"O no," the girl replied. She seemed to prefer a
+less tragic probability ; to have saved a man from death
+'involved talk that should harmonise with the dignity of
+such a deed -- and she shunned it.
+"I believe you saved my life, Miss --  -- I don!t know
+your name. I know your aunt's, but not yours.'
+" I would just as soon not tell it -- rather not. There
+is no reason either why I should, as you probably will
+never have much to do with me.'
+ " Still, I should like to know.'
+" You can inquire at my aunt's -- she will tell you.'
+'My name is Gabriel Oak.'
+"And mine isn't. You seem fond of yours in
+speaking it so decisively, Gabriel Oak.'
+<P 72>
+" You see, it is the only one I shall ever have, and I
+must make the most of it.'
+" I always think mine sounds odd and disagreeable.'
+"I should think you might soon get a new one.'
+"Mercy ! -- how many opinions you keep about you
+concerning other people, Gabriel Oak.
+"Well Miss-excuse the words-I thought you
+would like them But I can't match you I know in
+napping out my mind upon my tongue. I never was
+very clever in my inside. But I thank you. Come
+give me your hand!'
+She hesitated, somewhat disconcerted at Oak's old+
+fashioned earnest conclusion. to a dialogue lightly
+carried on."Very well,' she said, and gave him her
+hand, compressing her lips to a demure impassivity.
+He held it but an instant, and in his fear of being too
+demonstrative, swerved to the opposite extreme, touching
+her fingers with the lightness of a small-hearted person.
+" I am sorry,' he said, the instant after.
+" What for?'
+"You may have it again if you like; there it is.'
+She gave him her hand again.
+Oak held it longer this time -- indeed, curiously long.
+"How soft it is -- being winter time, too -- not chapped
+or rough or anything!' he said.
+"There -- that's long enough,' said she, though with+
+out pulling it away "But I suppose you are thinking
+you would like to kiss it? You may if you want to.'
+"I wasn't thinking of any such thing,' said Gabriel,
+simply ; "but I will'
+"That you won't!' She snatched back her hand.
+Gabriel felt himself guilty of another want of tact.
+"Now find out my name,' she said, teasingly; and
+withdrew.
+<C iv>
+<P 73>
+GABRIEL'S RESOLVE -- THE VISIT -- THE MISTAKE
+THE only superiority in women that is tolerable to the
+rival sex is, as a rule, that of the unconscious kind ; but
+a superiority which recognizes itself may sometimes
+please by suggesting possibilities of capture to the
+subordinated man.
+This well-favoured and comely girl soon made appre+
+ciable inroads upon the emotional constitution of young
+Farmer Oak.
+Love, being an extremely exacting usurer (a sense of
+exorbitant profit, spiritually, by an exchange of hearts,
+being at the bottom of pure passions, as that of exorbi+
+tant profit, bodily or materially, is at the bottom of
+those of lower atmosphere), every morning Oak's feelings
+were as sensitive as the money-market in calculations
+upon his chances. His dog waited for his meals in a
+way so like that in which Oak waited for the girl's
+presence, that the farmer was quite struck with the
+resemblance, felt it lowering, and would not look at the
+dog. However, he continued to watch through the
+hedge for her regular coming, and thus his sentiments
+towards her were ideepened without any corresponding
+effect being produced upon herself. Oak had nothing
+finished and ready to say as yet, and not being able
+to frame love phrases which end where they begin ;
+passionate tales -- +
+     -- -Full of sound and fury
+   -- -signifting nothing -- +
+he said no word at all.
+By making inquiries he found that the girl's name
+was Bathsheba Everdene, and that the cow would go
+dry in about seven days. He dreaded the eight day.
+At last the eighth day came. The cow had ceased
+to give milk for that year, and Bathsheba Everdene
+came up the hill no more. Gabriel had reached a
+pitch of existence he never
+<P 74>
+could have anticipated a
+short time before. He liked saying 'Bathsheba' as a
+private enjoyment instead of whistling; turned over his
+taste to black hair, though he had sworn by brown ever
+since he was a boy, isolated himself till the space he
+filled in a possible strength in an actual weakness. Marriage
+transforms a distraction into a support, the power of
+which should be, and happily often is, in direct pro+
+portion to the degree of imbecility it supplants. Oak
+began now to see light in this direction, and said to
+himself, "I'll make her my wife, or upon my soul I shall
+be good for nothing .! '
+All this while he was perplexing himself about an
+errand on which he might consistently visit the cottage
+of Bathsheba's aunt.
+He found his opportunity in the death of a ewe,
+mother of a living lamb. On a day which had a
+summer face and a winter constitution-a fine January
+morning, when there was just enough blue sky visible to
+make cheerfully-disposed people wish for more, and an
+occasional gleam of silvery sunshine, Oak put the lamb
+into a respectable Sunday basket, and stalked across the
+fields to the house of Mrs. Hurst, the aunt -- George,
+the dog walking behind, with a countenance of great
+concern at the serious turn pastoral affairs seemed to be
+taking.
+Gabriel had watched the blue wood-smoke curling
+from the chimney with strange meditation. At evening
+he had fancifully traced it down the chimney to the
+spot of its origin -- seen the hearth and Bathsheba
+beside it -- beside it in her out-door dress; for the
+clothes she had worn on the hill were by association
+equally with her person included in the compass of his
+affection; they seemed at this early time of his love a
+necessary ingredient of the sweet mixture called Bath+
+sheba Everdene.
+He had made a toilet of a nicely-adjusted kind -- of a
+nature between the carefully neat and the carelessly
+ornate -- of a degree between fine-market-day and wet+
+Sunday selection. He thoroughly cleaned his silver
+watch-chain with whiting, put new lacing straps to his
+boots, looked to the brass eyelet-holes,
+<P 75>
+ went to the
+inmost heart of the plantation for a new walking-stick,
+and trimmed it vigorously on his way back; took a new
+handkerchief from the bottom of his clothes-box, put
+on the light waistcoat patterned all over with sprigs
+of an elegant flower uniting the beauties of both rose
+and lily without the defects of either, and used all the
+hair-oil he possessed upon his usually dry, sandy, and
+inextricably curly hair, till he had deepened it to a
+splendidly novel colour, between that of guano and
+Roman cement, making it stick to his head like mace
+round a nutmeg, or wet seaweed round a boulder after
+the ebb.
+Nothing disturbed the stillness of the cottage save
+ the chatter of a knot of sparrows on the eaves; one
+might fancy scandal and rumour to be no less the
+staple topic of these little coteries on roofs than of
+those under them. It seemed that the omen was an
+unpropitious one, for, as the rather untoward commence+
+ment of Oak's overtures, just as he arrived by the garden
+gate, he saw a cat inside, going into various arched shapes
+and fiendish convulsions at the sight of his dog George.
+The dog took no notice , for he had arrived at an age
+at which all superfluous barking was cynically avoided
+as a waste of breath -- -in fact he never barked even
+at the sheep except to order, when it was done with
+an absolutely neutral countenance, as a sort of Com+
+mination-service, which, though offensive, had to be
+gone through once now and then to frighten the flock
+for their own good.
+A voice came from behind some laurel-bushes into
+which the cat had run:
+"Poor dear! Did a nasty brute of a dog want to
+kill it; -- did he poor dear !'
+"I beg your pardon,' said Oak to the voice, 'but
+George was walking on behind me with a temper as
+mild as milk.'
+Almost before he had ceased speaking, Oak was
+seized with a misgiving as to whose ear was the recipient
+of his answer. Nobody appeared, and he heard the
+person retreat among the bushes.
+Gabriel meditated, and so deeply that he brought
+small furrows into his forehead by sheer force of
+reverie. Where the
+<P 76>
+issue of an interview is as likely
+to be a vast change for the worse as for the better,
+any initial difference from expectation causes nipping
+sensations of failure. Oak went up to the door a little
+abashed : his mental rehearsal and the reality had had
+no common grounds of opening.
+Bathsheba's aunt was indoors. " Will you tell Miss
+Everdene that somebody would be glad to speak to
+her ?'said Mr. Oak. (Calling one's self merely Some+
+body, without giving a name, is not to be taken as
+an example of the ill-breeding of the rural world: it
+springs from a refined modesty, of which townspeople,
+with their cards and announcements, have no notion
+whatever.)
+Bathsheba was out. The voice had evidently been
+hers.
+" Will you come in, Mr. Oak ? '
+"Oh, thank 'ee, said Gabriel, following her to the
+fireplace. "I've brought a lamb for Miss Everdene.
+I thought she might like one to rear; girls do.'
+" She might,' said Mrs. Hurst, musingly ; " though
+she's only a visitor here. If you will wait a minute,
+Bathsheba will be in.'
+" Yes, I will wait,' said Gabriel, sitting down. " The
+lamb isn't really the business I came about, Mrs. Hurst.
+In short, I was going to ask her if she'd like to be
+married.'
+"And were you indeed ?'
+" Yes. Because if she would, I should be very glad
+to marry her. D'ye know if she's got any other young
+man hanging about her at all ?'
+"Let me think," said Mrs. Hurst, poking the fire
+superfluously.... " Yes -- bless you, ever so many young
+men. You see, Farmer Oak, she's so good-looking, and
+an excellent scholar besides -- she was going to be a
+governess once, you know, only she was too wild. Not
+that her young men ever come here -- but, Lord, in the
+nature of women, she must have a dozen ! '
+" That's unfortunate,' said Farmer Oak, contemplating
+a crack in the stone floor with sorrow. "I'm only an
+every-day sort of man, and my only chance was in being
+the first comer... , Well, there's no use in my waiting,
+for that was all I came about: so I'll take myself off
+home-along, Mrs. Hurst.'
+When Gabriel had gone about two hundred yards
+along the
+<P 77>
+down, he heard a "hoi-hoi .! " uttered behind
+him, in a piping note of more treble quality than that
+in which the exclamation usually embodies itself when
+shouted across a field. He looked round, and saw a girl
+racing after him, waving a white handkerchief.
+Oak stood still -- and the runner drew nearer. It was
+Bathsheba Everdene. Gabriel's colour deepened: hers
+was already deep, not, as it appeared, from emotion,
+but from running.
+"Farmer Oak -- I -- ' she said, pausing for want of
+breath pulling up in front of him with a slanted face
+and putting her hand to her side.
+"I have just called to see you ' said Gabriel, pending
+her further speech.
+"Yes-I know that,! she said panting like a robin,
+her face red and moist from her exertions, like a peony
+petal before the sun dries off the dew. "I didn't know
+you had come to ask to have me, or I should have come
+in from the garden instantly. I ran after you to say --
+that my aunt made a mistake in sending you away from
+courting me --  --  -- '
+Gabriel expanded."I'm sorry to have made you
+run so fast, my dear,' he said, with a grateful sense of
+favours to come. "Wait a bit till you've found your
+breath.'
+" -- It was quite a mistake-aunt's telling you I had
+a young man "already,'- Bathsheba went on. " I haven't
+a sweetheart at all -- and I never had one, and I thought
+that, as times go with women, it was such a pity to send
+you away thinking that I had several.'
+"Really and truly I am glad to hear that.!' said .=
+Farmer Oak, smiling one of his long special smiles, and
+blushing with gladness. He held out his hand to take
+hers, which, when she had eased her side by pressing
+it there, was prettily extended upon her bosom to still
+her loud-beating heart. Directly he seized it she put
+it behind her, so that it slipped through his fingers like
+an eel. "
+"I have a nice snug little farm,' said Gabriel, with
+half a degree less assurance than when he had seized
+her hand.
+"Yes ; you have.'
+"A man has advanced me money to begin with, but
+still, it
+<P 78>
+will soon be paid off and though I am only an
+every-day sort of man, I have got on a little since I was
+a boy.' Gabriel uttered "a little' in a tone to-show
+her that it was the complacent form of "a great deal.'
+He continued : " When we be married, I am quite sure
+I can work twice as hard as I do now.'
+ He went forward and stretched out his arm again.
+Bathsheba had overtaken him at a point beside which
+stood a low stunted holly bush, now laden with red
+berries. Seeing his advance take the form of an attitude
+threatening a possible enclosure, if not compression, of
+her person, she edged off round the bush.
+" Why, Farmer Oak,' she said, over the top, looking
+at him with rounded eyes, "I never said I was going to
+marry you.'
+" Well -- that is a tale .! ' said Oak,  with dismay. " To
+run after anybody like this, and then say you don"t
+want him ! '
+"What I meant to tell you was only this,' she said
+eagerly, and yet half conscious of the absurdity of the
+position she had made for herself -- "that nobody has
+got me yet as a sweetheart, instead of my having a
+dozen, as my aunt said; I hate to be thought men's
+property in that way, though possibly I shall be had
+some day. Why, if I'd wanted you I shouldn't have
+run after you like this ; 'twould have'been the forwardest
+thing ! But there was no harm in 'hurrying to correct
+a piece of false news that had been told you.'
+"Oh, no -- no harm at all." But there is such a thing
+as being too generous in expressing a judgment impuls+
+ively, and Oak added with a more appreciative sense
+of all the circumstances -- ' Well, I am not quite certain
+it was no harm.'
+"Indeed, I hadn't time to think before starting
+whether I wanted to marry or not, for you'd have been
+gone over the hill.'
+" Come,' said Gabriel, freshening again ; "think a
+minute or two. I'll wait a while, Miss Everdene. Will
+you marry me? Do, Bathsheba. I love you far more
+than common!'
+"I'll try to think,' she observed, rather more timor+
+ously ; "if I can think out of doors; my mind spreads
+away so.'
+"But you can give a guess.'
+<P 79>
+"Then give me time.' Bathsheba looked thought+
+fully into the distance, away from the direction in which
+Gabriel stood.
+"I can make you happy,' said he to the back of her
+head, across the bush. "You shallo have as piano in a
+year or two -- -farmers' wives are getting to have pianos
+now --  and I'll practise up the flute right well to play
+with you in the evenings.'
+" Yes ; I should like that.'
+"And have one of those little ten-pound" gigs for
+market -- and nice flowers, and birds -- cocks and hens
+I mean, because they be useful,' continued Gabriel,
+feeling balanced between poetry and practicality.
+"I should like it very much.'
+"And a frame for cucumbers -- like a gentlman and
+lady.'
+"Yes.'
+"And when the wedding was over, we'd have it put
+in the newspaper list of marriages.'
+" Dearly I should like that ! '
+"And the babies in the births -- every man jack of
+'em! And at home by the fire, whenever you look up,
+there I shall be -- and whenever I look up' there will
+be you.'
+"Wait wait and don't be improper .!'
+Her countenance fell, and she was silent awhile.
+He regarded the red berries between them over and
+over again, to such an extent, that holly seemed in
+his after life to be a cypher signifying a proposal of
+marriage. Bathsheba decisively turned to him.
+"No;' 'tis no use,' she said. 'I don't want to marry
+you. '
+' Try.'
+"I have tried hard all the time I've been thinking;
+for a marriage would be very nice in one sense.
+People would talk about me, and think I had won my
+battle, and I should feel triumphant, and' all that,
+But a husband --  -- ' - +
+" Well .! '
+" Why, he'd always be there, as you say; whenever
+I looked up, there he'd be.'
+" Of course he would -- I, that is.'
+<P 80>
+" Well, what I mean is that I shouldn't mind being
+a bride at a wedding, if I could be one without having
+a husband. But since a woman can't show off in that
+way by herself, I shan't marry -- at least yet.'
+' That's a terrible wooden story.'
+At this criticism of her statement Bathsheba made
+an addition to her dignity by a slight sweep away
+from him.
+"Upon my heart and soul, I don't know what a
+maid can say stupider than that,' said Oak. "But
+dearest,' he continued in a palliative voice, "don't be
+like it !.' Oak sighed a deep honest sigh -- none the
+less so in that, being like the sigh of a pine plantation,
+it was rather noticeable as a disturbance of the atmo+
+sphere. " Why won't you have me ? ' he appealed,
+creeping round the holly to reach her side.
+" I cannot,' she said, retreating.
+"But why ?' he persisted, standing still at last in
+despair of ever reaching her, and facing over the
+bush.
+' Because I don't love you.'
+" Yes, but --  -- '
+She contracted a yawn to an inoffensive smallness,
+so that it was hardly ill-mannered at all. "I don't love
+you,' she said.'
+"But I love you -- and, as for myself, I am content
+to be liked.'
+" O Mr. Oak -- that's very fine ! You'd get to
+despise me.'
+"Never,' said Mr Oak, so earnestly that he seemed
+to be coming, by the forceof his words, straight
+through the bush and into her arms. "I shall do one
+thing in this life -- one thing certain -- that is, love you,
+and long for you, and keep wanting you till I die.' His
+voice had a genuine pathos now, and his large brown
+hands perceptibly trembled.
+"It seems dreadfully wrong not to have you when
+you feel so much!' she said with a little distress, and
+looking hopeleely around for some means of escape
+from her moral dilemma. " H(ow I wish I hadn't run
+after you!' However she seemed to have a short cut
+for getting back to cheerfulness, and set her face to
+signify archness. "It wouldn't do, Mr Oak. I want
+somebody to tame me; I am too independent ; and
+you would never be able to, I know.'
+<P 81>
+Oak cast his eyes down the field in a way implying
+that it was useless to attempt argument.
+" Mr. Oak,' she said, with luminous distinctness and
+common sense, " you are better off than I. I have
+hardly a penny in the world -- I am staying with my
+aunt for my bare sustenance. I am better educated
+than you -- and I don't love you a bit: that's my side
+of the case. Now yours: you are a farmer just begin+
+ing; and you ought in common prudence, if you marry
+at all (which you should certainly not think of doing
+at present) to marry a woman with money, who would
+admiration.
+"That's the very thing I had been thinking myself !'
+he naively said.
+Farmer Oak had one-and-a-half Christian character
+istics too many to succeed with Bathsheba : his humility,
+and a superfluous moiety of honesty. Bathsheba was
+decidedly disconcerted,
+"Well, then, why did you come and disturb me?'
+she said, almost angrily, if not quite, an enlarging red
+spot rising in each cheek.
+" I can't do what I think would be -- would be --  -- '
+" Right ? '
+" No : wise.'
+" You have made an admission now, Mr. Oak,' she
+exclaimed, with even more hauteur, and rocking her
+head disdainfully. 'After that, do you think I could
+marry you? Not if I know it.'
+He broke in passionately ! "But don't mistake me
+like that! Because I am open enough to own what
+every man in my shoes would have thought of, you
+make your colours come up your face, and get crabbed
+with me. That about your not being good enough for
+me is nonsense. You speak like a lady -- all the parish
+notice it, and your uncle at Weatherbury is, I have
+heerd, a large farmer -- much larger than ever I shall
+be. May I call in the evening, or will you walk along
+with me o' Sundays? I don't want you to make-up
+your mind at once, if you'd rather not.'
+<P 82>
+" No -- no -- I cannot. Don't press me any more --
+don't. I don't love you -- so 'twould be ridiculous,'
+she said, with a laugh.
+No man likes to see his emotions the sport of a
+merry-go-round of skittishness. " Very well,' said Oak,
+firmly, with the bearing of one who was going to give '
+his days and nights to Ecclesiastes for ever. "Then
+I'll ask you no more.'
+<C v>
+<P 83>
+DEPARTURE OF BATHSHEBA -- A PASTORAL TRAGEDY
+THE news which one day reached Gabriel, that Bath+
+sheba Everdene had left the neighbourhood, had an
+influence upon him which might have surprised any
+who never suspected that the more emphatic the renun+
+ciation the less absolute its character.
+It may have been observed that there is no regula
+path for getting out of love as there is for getting in.
+Some people look upon marriage as a short cut that way,
+but it has been known to fail. Separation, which was
+the means that chance offered to Gabriel Oak by
+Bathsheba's disappearance though effectual with people
+of certain humours is apt to idealise the removcd object
+with others -- notably those whose affection, placid and
+regular as it may be flows deep and long. Oak belonged
+to the even-tempered order of humanity, and felt the
+secret fusion of himself in Bathsheba to be burning with
+a finer flame now that she was gone -- that was all.
+His incipient friendship with her aunt-had been
+nipped by the failure of his suit, and all that Oak learnt
+of Bathsheba's movements was done indirectly. It ap+
+peared that she had gone to a place called Weatherbury,
+more than twenty miles off, but in what capacity --
+whether as a visitor, or permanently, he could not
+discover.
+Gabriel had two dogs. George, the elder, exhibited
+an ebony-tipped nose, surrounded by a narrow margin
+of pink flesh, and a coat marked in random splotches
+approximating in colour to white and slaty grey ; but the
+grey, after years of sun and rain, had been scorched and
+washed out of the more prominent locks, leaving them
+of a reddish-brown, as if the blue component of the grey
+had faded, like the indigo from the same kind of colour in
+Turner's pictures. In substance it had originally been
+hair, but long contact with sheep seemed
+<P 84>
+to be turning
+it by degrees into wool of a poor quality and staple.
+This dog had originally belonged to a shepherd of
+inferior morals and dreadful temper, and the result was
+that George knew the exact degrees of condemnation
+signified by cursing and swearing of all descriptions
+better than the wickedest old man in the neighbourhood.
+Long experience had so precisely taught the animal the
+difference between such exclamations as 'Come in .! '
+and 'D --  --  ye, come in !.' that he knew to a hair's
+breadth the rate of trotting back from the ewes' tails
+that each call involved, if a staggerer with the sheep
+crook was to be escaped. Though old, he was clever
+and trustworthy still.
+The young dog, George's son, might possibly have
+been the image of his mother, for there was not much
+resemblance between him and George. He was learn+
+ing the sheep-keeping business, so as to follow on at
+the flock when the other should die, but had got no
+further than the rudiments as yet -- still finding an
+insuperable difculty in distinguishing between doing a
+thing well enough and doing it too well. So earnest
+and yet so wrong-headed was this young dog (he had no,
+name in particular, and answered with perfect readiness
+to any pleasant interjection), that if sent behind the
+flock to help them on, he did it so thoroughly that he
+would have chased them across the whole county with
+the greatest pleasure if not called off or reminded when
+to step by the example of old George.
+Thus much for the dogs. On the further side of
+Norcombe Hill was a chalk-pit, from which chalk had
+been drawn for generations, and spread over adjacent
+farms. Two hcdges converged upon it in the form of
+a V, but without quite meeting. The narrow opening
+left, which was immediately over the brow of the pit,
+was protected by a rough railing.
+One night, when Farmer Oak had returned to, his
+house, believing there would be no further necessity for
+his attendance on the down, he called as usual to the
+dogs, previously to shutting them up in the outhouse till
+next morning. Only one responded -- old George ; the
+other-could not be found, either in the house, lane, or
+garden. - Gabriel then remembered
+<P 85>
+that he had left the
+two dogs on the hill eating a dead lamb (a kind of meat
+he usually kept from them, except when other food-ran
+finished his meal, he went indoors to the luxury of a bed,
+which latterly he had only enjoyed on Sundays.
+It was a still, moist night. Just before dawn he was
+assisted in waking by the abnormal reverberation of
+familiar music. To the shepherd, the note of the sheep'
+chronic sound that only makes itself noticed by ceasing
+ever distant, that all is well in the fold. In the solemn
+This exceptional ringing may be caused in two ways -- +
+by the rapid feeding of the sheep 

server/build.gradle

@@ -27,6 +27,7 @@ dependencies {
   api project(':libs:elasticsearch-secure-sm')
   api project(':libs:elasticsearch-x-content')
   api project(":libs:elasticsearch-geo")
+  api project(":libs:elasticsearch-lz4")
 
   implementation project(':libs:elasticsearch-plugin-classloader')
 
@@ -50,9 +51,6 @@ dependencies {
   api project(":libs:elasticsearch-cli")
   api 'com.carrotsearch:hppc:0.8.1'
 
-  // LZ4
-  api 'org.lz4:lz4-java:1.8.0'
-
   // time handling, remove with java 8 time
   api "joda-time:joda-time:${versions.joda}"
 
@@ -249,11 +247,6 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.common.geometry.S2LatLng'
     )
     ignoreMissingClasses 'javax.xml.bind.DatatypeConverter'
-
-    ignoreViolations(
-      // from java-lz4
-      'net.jpountz.util.UnsafeUtils'
-    )
 }
 
 tasks.named("dependencyLicenses").configure {

server/src/main/java/org/elasticsearch/transport/Compression.java

@@ -8,10 +8,16 @@
 
 package org.elasticsearch.transport;
 
+import net.jpountz.lz4.LZ4Compressor;
 import net.jpountz.lz4.LZ4Factory;
 
+import net.jpountz.lz4.LZ4FastDecompressor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.lz4.ESLZ4Compressor;
+import org.elasticsearch.lz4.ESLZ4Decompressor;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -21,12 +27,13 @@ public class Compression {
     public enum Scheme {
         LZ4,
         DEFLATE;
-        
+
         static final Version LZ4_VERSION = Version.V_7_14_0;
         static final int HEADER_LENGTH = 4;
         private static final byte[] DEFLATE_HEADER = new byte[]{'D', 'F', 'L', '\0'};
         private static final byte[] LZ4_HEADER = new byte[]{'L', 'Z', '4', '\0'};
         private static final int LZ4_BLOCK_SIZE;
+        private static final boolean USE_FORKED_LZ4;
 
         static {
             String blockSizeString = System.getProperty("es.transport.compression.lz4_block_size");
@@ -39,6 +46,8 @@ public class Compression {
             } else {
                 LZ4_BLOCK_SIZE = 64 * 1024;
             }
+
+            USE_FORKED_LZ4 = Booleans.parseBoolean(System.getProperty("es.compression.use_forked_lz4", "true"));
         }
 
         public static boolean isDeflate(BytesReference bytes) {
@@ -68,9 +77,23 @@ public class Compression {
             return true;
         }
 
+        public static LZ4FastDecompressor lz4Decompressor() {
+            if (USE_FORKED_LZ4) {
+                return ESLZ4Decompressor.INSTANCE;
+            } else {
+                return LZ4Factory.safeInstance().fastDecompressor();
+            }
+        }
+
         public static OutputStream lz4OutputStream(OutputStream outputStream) throws IOException {
             outputStream.write(LZ4_HEADER);
-            return new ReuseBuffersLZ4BlockOutputStream(outputStream, LZ4_BLOCK_SIZE, LZ4Factory.safeInstance().fastCompressor());
+            LZ4Compressor lz4Compressor;
+            if (USE_FORKED_LZ4) {
+                lz4Compressor = ESLZ4Compressor.INSTANCE;
+            } else {
+                lz4Compressor = LZ4Factory.safeInstance().fastCompressor();
+            }
+            return new ReuseBuffersLZ4BlockOutputStream(outputStream, LZ4_BLOCK_SIZE, lz4Compressor);
         }
     }
 
@@ -79,5 +102,4 @@ public class Compression {
         INDEXING_DATA,
         FALSE
     }
-
 }

server/src/main/java/org/elasticsearch/transport/Lz4TransportDecompressor.java

@@ -24,7 +24,6 @@
 package org.elasticsearch.transport;
 
 import net.jpountz.lz4.LZ4Exception;
-import net.jpountz.lz4.LZ4Factory;
 import net.jpountz.lz4.LZ4FastDecompressor;
 
 import org.apache.lucene.util.BytesRef;
@@ -120,7 +119,7 @@ public class Lz4TransportDecompressor implements TransportDecompressor {
     private boolean hasSkippedESHeader = false;
 
     public Lz4TransportDecompressor(PageCacheRecycler recycler) {
-        this.decompressor = LZ4Factory.safeInstance().fastDecompressor();
+        this.decompressor = Compression.Scheme.lz4Decompressor();
         this.recycler = recycler;
         this.pages = new ArrayDeque<>(4);
     }