Add microbenchmarking infrastructure (#18891)

With this commit we add a benchmarks project that contains the necessary build
infrastructure and an example benchmark. It is added as a separate project to avoid
interfering with the regular build too much (especially sanity checks) and to keep
the microbenchmarks isolated.

Microbenchmarks are generated with `gradle :benchmarks:jmhJar` and can be run with
` gradle :benchmarks:jmh`.

We intentionally do not use the
[jmh-gradle-plugin](https://github.com/melix/jmh-gradle-plugin) as it causes all
sorts of problems (dependencies are not properly excluded, not all JMH parameters
can be set) and it adds another abstraction layer that is not needed.

Closes #18242

benchmarks/README.md

@@ -0,0 +1,63 @@
+# Elasticsearch Microbenchmark Suite
+
+This directory contains the microbenchmark suite of Elasticsearch. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/).
+
+## Purpose
+
+We do not want to microbenchmark everything but the kitchen sink and should typically rely on our 
+[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with 
+[Rally](http://github.com/elastic/rally). Microbenchmarks are intended for performance-critical components to spot performance 
+regressions. The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
+
+## Getting Started
+
+Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result.
+
+## Running Microbenchmarks
+
+Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
+```
+ 
+Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
+
+If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have any other special requirements 
+generate the uberjar with `gradle :benchmarks:jmhJar` and run the it directly with:
+
+```
+java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar
+```
+
+JMH supports lots of command line parameters. Add `-h` to the command above for more information about the available command line options.
+
+## Adding Microbenchmarks
+
+Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the 
+[JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/).
+
+In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and 
+end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`.
+
+## Tips and Best Practices
+
+To get realistic results, you should exercise care when running your benchmarks. Here are a few tips:
+
+### Do
+
+* Ensure that the system executing your microbenchmarks has as little load as possible and shutdown every process that can cause unnecessary 
+  runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance.
+* Ensure to run enough warmup iterations to get into a stable state. If you are unsure, don't change the defaults.
+* Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`.
+* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the 
+  `performance` CPU governor.
+* Vary problem input size with `@Param`.
+* Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
+** Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
+   your results. If so, try to force a GC between runs (`-gc true`).
+** Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
+* Have your benchmarks peer-reviewed.
+
+### Don't
+
+* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.e. with `-prof perfasm`.
+* Run run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
+* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.

benchmarks/build.gradle

@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+buildscript {
+    repositories {
+        maven {
+            url 'https://plugins.gradle.org/m2/'
+        }
+    }
+    dependencies {
+        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+    }
+}
+
+apply plugin: 'elasticsearch.build'
+// build an uberjar with all benchmarks
+apply plugin: 'com.github.johnrengelman.shadow'
+// have the shadow plugin provide the runShadow task
+apply plugin: 'application'
+
+archivesBaseName = 'elasticsearch-benchmarks'
+mainClassName = 'org.openjdk.jmh.Main'
+
+// never try to invoke tests on the benchmark project - there aren't any
+check.dependsOn.remove(test)
+// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
+task test(type: Test, overwrite: true)
+
+dependencies {
+    compile("org.elasticsearch:elasticsearch:${version}") {
+        // JMH ships with the conflicting version 4.6 (JMH will not update this dependency as it is Java 6 compatible and joptsimple is one
+        // of the most recent compatible version). This prevents us from using jopt-simple in benchmarks (which should be ok) but allows us
+        // to invoke the JMH uberjar as usual.
+        exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
+    }
+    compile "org.openjdk.jmh:jmh-core:$versions.jmh"
+    compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
+    //TODO: Transitive dependencies of JMH. Add them here explicitly for the time being and find out why they are not included implicitly.
+    runtime 'net.sf.jopt-simple:jopt-simple:4.6'
+    runtime 'org.apache.commons:commons-math3:3.2'
+}
+
+compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+
+forbiddenApis {
+    // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
+    ignoreFailures = true
+}
+
+// No licenses for our benchmark deps (we don't ship benchmarks)
+dependencyLicenses.enabled = false
+
+thirdPartyAudit.excludes = [
+        // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)
+        'org.openjdk.jmh.profile.AbstractHotspotProfiler',
+        'org.openjdk.jmh.profile.HotspotThreadProfiler',
+        'org.openjdk.jmh.profile.HotspotClassloadingProfiler',
+        'org.openjdk.jmh.profile.HotspotCompilationProfiler',
+        'org.openjdk.jmh.profile.HotspotMemoryProfiler',
+        'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
+        'org.openjdk.jmh.util.Utils'
+]
+
+shadowJar {
+    classifier = 'benchmarks'
+}
+
+// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
+task jmhJar(
+        dependsOn: shadowJar,
+        description: 'Generates an uberjar with the microbenchmarks and all dependencies',
+        group: 'Benchmark'
+)
+
+task jmh(
+        dependsOn: runShadow,
+        description: 'Runs all microbenchmarks',
+        group: 'Benchmark'
+)

benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.joda.time.DateTimeZone;
+import org.joda.time.MutableDateTime;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.TemporalAmount;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+@State(Scope.Benchmark)
+public class DateBenchmark {
+    private long instant;
+
+    private MutableDateTime jodaDate = new MutableDateTime(0, DateTimeZone.UTC);
+
+    private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+
+    private ZonedDateTime javaDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneOffset.UTC);
+
+    private TemporalAmount diff = Duration.ofMillis(1L);
+
+    @Benchmark
+    public int mutableDateTimeSetMillisGetDayOfMonth() {
+        jodaDate.setMillis(instant++);
+        return jodaDate.getDayOfMonth();
+    }
+
+    @Benchmark
+    public int calendarSetMillisGetDayOfMonth() {
+        calendar.setTimeInMillis(instant++);
+        return calendar.get(Calendar.DAY_OF_MONTH);
+    }
+
+    @Benchmark
+    public int javaDateTimeSetMillisGetDayOfMonth() {
+        // all classes in java.time are immutable, we have to use a new instance
+        javaDateTime = javaDateTime.plus(diff);
+        return javaDateTime.getDayOfMonth();
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.openjdk.jmh.annotations.Benchmark;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class HelloBenchmark {
+    @Benchmark
+    public void benchmarkRuntimeOverhead() {
+        //intentionally left blank
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class AllocationBenchmark {
+    // Do NOT make any field final (even if it is not annotated with @Param)! See also
+    // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java
+
+    // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no
+    // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would
+    // need its own main method and we cannot execute more than one class with a main method per JAR.
+    @Param({
+        // indices, shards, replicas, nodes
+        "       10,      1,        0,     1",
+        "       10,      3,        0,     1",
+        "       10,     10,        0,     1",
+        "      100,      1,        0,     1",
+        "      100,      3,        0,     1",
+        "      100,     10,        0,     1",
+
+        "       10,      1,        0,    10",
+        "       10,      3,        0,    10",
+        "       10,     10,        0,    10",
+        "      100,      1,        0,    10",
+        "      100,      3,        0,    10",
+        "      100,     10,        0,    10",
+
+        "       10,      1,        1,    10",
+        "       10,      3,        1,    10",
+        "       10,     10,        1,    10",
+        "      100,      1,        1,    10",
+        "      100,      3,        1,    10",
+        "      100,     10,        1,    10",
+
+        "       10,      1,        2,    10",
+        "       10,      3,        2,    10",
+        "       10,     10,        2,    10",
+        "      100,      1,        2,    10",
+        "      100,      3,        2,    10",
+        "      100,     10,        2,    10",
+
+        "       10,      1,        0,    50",
+        "       10,      3,        0,    50",
+        "       10,     10,        0,    50",
+        "      100,      1,        0,    50",
+        "      100,      3,        0,    50",
+        "      100,     10,        0,    50",
+
+        "       10,      1,        1,    50",
+        "       10,      3,        1,    50",
+        "       10,     10,        1,    50",
+        "      100,      1,        1,    50",
+        "      100,      3,        1,    50",
+        "      100,     10,        1,    50",
+
+        "       10,      1,        2,    50",
+        "       10,      3,        2,    50",
+        "       10,     10,        2,    50",
+        "      100,      1,        2,    50",
+        "      100,      3,        2,    50",
+        "      100,     10,        2,    50"
+    })
+    public String indicesShardsReplicasNodes = "10,1,0,1";
+
+    public int numTags = 2;
+
+    private AllocationService strategy;
+    private ClusterState initialClusterState;
+
+    @Setup
+    public void setUp() throws Exception {
+        final String[] params = indicesShardsReplicasNodes.split(",");
+
+        int numIndices = toInt(params[0]);
+        int numShards = toInt(params[1]);
+        int numReplicas = toInt(params[2]);
+        int numNodes = toInt(params[3]);
+
+        strategy = Allocators.createAllocationService(Settings.builder()
+                .put("cluster.routing.allocation.awareness.attributes", "tag")
+                .build());
+
+        MetaData.Builder mb = MetaData.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            mb.put(IndexMetaData.builder("test_" + i)
+                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .numberOfShards(numShards)
+                    .numberOfReplicas(numReplicas)
+            );
+        }
+        MetaData metaData = mb.build();
+        RoutingTable.Builder rb = RoutingTable.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            rb.addAsNew(metaData.index("test_" + i));
+        }
+        RoutingTable routingTable = rb.build();
+        DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
+        for (int i = 1; i <= numNodes; i++) {
+            nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
+        }
+        initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes
+                (nb).build();
+    }
+
+    private int toInt(String v) {
+        return Integer.valueOf(v.trim());
+    }
+
+    @Benchmark
+    public ClusterState measureAllocation() {
+        ClusterState clusterState = initialClusterState;
+        while (clusterState.getRoutingNodes().hasUnassignedShards()) {
+            RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
+                    .shardsWithState(ShardRoutingState.INITIALIZING));
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+            result = strategy.reroute(clusterState, "reroute");
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        }
+        return clusterState;
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.gateway.GatewayAllocator;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public final class Allocators {
+    private static class NoopGatewayAllocator extends GatewayAllocator {
+        public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
+
+        protected NoopGatewayAllocator() {
+            super(Settings.EMPTY, null, null);
+        }
+
+        @Override
+        public void applyStartedShards(StartedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public void applyFailedShards(FailedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public boolean allocateUnassigned(RoutingAllocation allocation) {
+            return false;
+        }
+    }
+
+    private Allocators() {
+        throw new AssertionError("Do not instantiate");
+    }
+
+
+    public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
+        IllegalAccessException, InvocationTargetException {
+        return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings
+            .BUILT_IN_CLUSTER_SETTINGS));
+    }
+
+    public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
+        InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
+        return new AllocationService(settings,
+            defaultAllocationDeciders(settings, clusterSettings),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
+    }
+
+    public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
+        IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
+        List<AllocationDecider> list = new ArrayList<>();
+        // Keep a deterministic order of allocation deciders for the benchmark
+        for (Class<? extends AllocationDecider> deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
+            try {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class, ClusterSettings
+                    .class);
+                list.add(constructor.newInstance(settings, clusterSettings));
+            } catch (NoSuchMethodException e) {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class);
+                list.add(constructor.newInstance(settings));
+            }
+        }
+        return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0]));
+
+    }
+
+    public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
+        return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
+            DiscoveryNode.Role.DATA), Version.CURRENT);
+    }
+}

benchmarks/src/main/resources/log4j.properties

@@ -0,0 +1,8 @@
+# Do not log at all if it is not really critical - we're in a benchmark
+benchmarks.es.logger.level=ERROR
+log4j.rootLogger=${benchmarks.es.logger.level}, out
+
+log4j.appender.out=org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout=org.apache.log4j.PatternLayout
+log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n
+

buildSrc/version.properties

@@ -17,3 +17,6 @@ httpclient        = 4.5.2
 httpcore          = 4.4.4
 commonslogging    = 1.1.3
 commonscodec      = 1.10
+
+# benchmark dependencies
+jmh               = 1.12

settings.gradle

@@ -5,6 +5,7 @@ List projects = [
   'rest-api-spec',
   'core',
   'docs',
+  'benchmarks',
   'distribution:integ-test-zip',
   'distribution:zip',
   'distribution:tar',