Run some REST tests against a cluster running in docker containers (#39515)

* Run REST tests against a cluster running on docker

Closes #38053

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -122,7 +122,7 @@ public class TestFixturesPlugin implements Plugin<Project> {
 
         extension.fixtures
             .matching(fixtureProject -> fixtureProject.equals(project) == false)
-            .all(fixtureProject ->  project.evaluationDependsOn(fixtureProject.getPath()));
+            .all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath()));
 
         conditionTaskByType(tasks, extension, Test.class);
         conditionTaskByType(tasks, extension, getTaskClass("org.elasticsearch.gradle.test.RestIntegTestTask"));

distribution/docker/build.gradle

@@ -4,18 +4,20 @@ import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
 
-apply plugin: 'base'
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 configurations {
   dockerPlugins
   dockerSource
   ossDockerSource
+  restSpec
 }
 
 dependencies {
   dockerSource project(path: ":distribution:archives:linux-tar")
   ossDockerSource project(path: ":distribution:archives:oss-linux-tar")
+  restSpec project(':rest-api-spec')
 }
 
 ext.expansions = { oss, local ->
@@ -77,20 +79,65 @@ void addCopyDockerContextTask(final boolean oss) {
   }
 }
 
+def createAndSetWritable (Object... locations) {
+  locations.each { location ->
+    File file = file(location)
+    file.mkdirs()
+    file.setWritable(true, false)
+  }
+}
+
+task copyKeystore(type: Sync) {
+  from project(':x-pack:plugin:core')
+          .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+  into "${buildDir}/certs"
+  doLast {
+    file("${buildDir}/certs").setReadable(true, false)
+    file("${buildDir}/certs/testnode.jks").setReadable(true, false)
+  }
+}
+
 preProcessFixture {
+  if (TestFixturesPlugin.dockerComposeSupported()) {
+    dependsOn assemble
+  }
+  dependsOn copyKeystore
+  doLast {
+    // tests expect to have an empty repo
+    project.delete(
+            "${buildDir}/repo",
+            "${buildDir}/oss-repo"
+    )
+    createAndSetWritable(
+            "${buildDir}/repo",
+            "${buildDir}/oss-repo",
+            "${buildDir}/logs/default-1",
+            "${buildDir}/logs/default-2",
+            "${buildDir}/logs/oss-1",
+            "${buildDir}/logs/oss-2"
+    )
+  }
+}
+
+processTestResources {
+  from ({ zipTree(configurations.restSpec.singleFile) }) {
+    include 'rest-api-spec/api/**'
+  }
+  from project(':x-pack:plugin:core')
+          .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+  dependsOn configurations.restSpec
   // don't add the tasks to build the docker images if we have no way of testing them
   if (TestFixturesPlugin.dockerComposeSupported()) {
     dependsOn assemble
   }
 }
 
-postProcessFixture.doLast {
-  println "docker default distro is on port: ${ext."test.fixtures.elasticsearch-default.tcp.9200"}, " +
-          "oss is on: ${ext."test.fixtures.elasticsearch-oss.tcp.9200"}"
+task integTest(type: Test) {
+  maxParallelForks = '1'
+  include '**/*IT.class'
 }
 
-// TODO: Add some actual tests, this will just check that the TPC port in the container is up
-check.dependsOn postProcessFixture
+check.dependsOn integTest
 
 void addBuildDockerImage(final boolean oss) {
   final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {

distribution/docker/docker-compose.yml

@@ -1,17 +1,133 @@
 # Only used for testing the docker images
 version: '3'
 services:
-  elasticsearch-default:
+  elasticsearch-default-1:
     image: elasticsearch:test
     environment:  
+       - node.name=elasticsearch-default-1 
+       - cluster.initial_master_nodes=elasticsearch-default-1,elasticsearch-default-2
+       - discovery.seed_hosts=elasticsearch-default-2:9300 
        - cluster.name=elasticsearch-default
-       - discovery.type=single-node
+       - bootstrap.memory_lock=true
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false  
+       - xpack.security.enabled=true
+       - xpack.security.transport.ssl.enabled=true
+       - xpack.security.http.ssl.enabled=true
+       - xpack.security.authc.token.enabled=true
+       - xpack.security.audit.enabled=true
+       - xpack.security.authc.realms.file.file1.order=0   
+       - xpack.security.authc.realms.native.native1.order=1
+       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.http.ssl.verification_mode=certificate  
+       - xpack.security.transport.ssl.verification_mode=certificate  
+       - xpack.license.self_generated.type=trial
+    volumes: 
+       - ./build/repo:/tmp/es-repo
+       - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
+       - ./build/logs/default-1:/usr/share/elasticsearch/logs
+       - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
     ports:
       - "9200"
-  elasticsearch-oss:
-    image: elasticsearch-oss:test
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    entrypoint: /docker-test-entrypoint.sh
+  elasticsearch-default-2:
+    image: elasticsearch:test
+    environment:  
+       - node.name=elasticsearch-default-2
+       - cluster.initial_master_nodes=elasticsearch-default-1,elasticsearch-default-2
+       - discovery.seed_hosts=elasticsearch-default-1:9300 
+       - cluster.name=elasticsearch-default
+       - bootstrap.memory_lock=true
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false  
+       - xpack.security.enabled=true
+       - xpack.security.transport.ssl.enabled=true
+       - xpack.security.http.ssl.enabled=true
+       - xpack.security.authc.token.enabled=true
+       - xpack.security.audit.enabled=true
+       - xpack.security.authc.realms.file.file1.order=0   
+       - xpack.security.authc.realms.native.native1.order=1  
+       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.http.ssl.verification_mode=certificate  
+       - xpack.security.transport.ssl.verification_mode=certificate  
+       - xpack.license.self_generated.type=trial
+    volumes: 
+       - ./build/repo:/tmp/es-repo
+       - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
+       - ./build/logs/default-2:/usr/share/elasticsearch/logs
+       - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
+    ports:
+      - "9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    entrypoint: /docker-test-entrypoint.sh
+  elasticsearch-oss-1:
+    image: elasticsearch:test
+    environment:  
+       - node.name=elasticsearch-oss-1 
+       - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2
+       - discovery.seed_hosts=elasticsearch-oss-2:9300 
+       - cluster.name=elasticsearch-oss
+       - bootstrap.memory_lock=true
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false  
+    volumes: 
+       - ./build/oss-repo:/tmp/es-repo
+       - ./build/logs/oss-1:/usr/share/elasticsearch/logs
+    ports:
+      - "9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+  elasticsearch-oss-2:
+    image: elasticsearch:test
     environment:  
+       - node.name=elasticsearch-oss-2
+       - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2
+       - discovery.seed_hosts=elasticsearch-oss-1:9300 
        - cluster.name=elasticsearch-oss
-       - discovery.type=single-node
+       - bootstrap.memory_lock=true
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false  
+    volumes: 
+       - ./build/oss-repo:/tmp/es-repo
+       - ./build/logs/oss-2:/usr/share/elasticsearch/logs
     ports:
       - "9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1

distribution/docker/docker-test-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+cd /usr/share/elasticsearch/bin/
+./elasticsearch-users useradd x_pack_rest_user -p x-pack-test-password -r superuser || true 
+echo "testnode" > /tmp/password
+cat /tmp/password  | ./elasticsearch-keystore add -x -f -v 'xpack.security.transport.ssl.keystore.secure_password'
+cat /tmp/password  | ./elasticsearch-keystore add -x -f -v 'xpack.security.http.ssl.keystore.secure_password'
+/usr/local/bin/docker-entrypoint.sh | tee > /usr/share/elasticsearch/logs/console.log

distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.docker.test;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.CharArrays;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.CharBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Base64;
+
+public class DockerYmlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    private static final String USER = "x_pack_rest_user";
+    private static final String PASS = "x-pack-test-password";
+    private static final String KEYSTORE_PASS = "testnode";
+
+    public DockerYmlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        String distribution = getDistribution();
+        return new StringBuilder()
+            .append("localhost:")
+            .append(getProperty("test.fixtures.elasticsearch-" + distribution + "-1.tcp.9200"))
+            .append(",")
+            .append("localhost:")
+            .append(getProperty("test.fixtures.elasticsearch-" + distribution + "-2.tcp.9200"))
+            .toString();
+    }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
+    }
+
+    private String getDistribution() {
+        String distribution = System.getProperty("tests.distribution", "default");
+        if (distribution.equals("oss") == false && distribution.equals("default") == false) {
+            throw new IllegalArgumentException("supported values for tests.distribution are oss or default but it was " + distribution);
+        }
+        return distribution;
+    }
+
+    private boolean isOss() {
+        return getDistribution().equals("oss");
+    }
+
+    private String getProperty(String key) {
+        String value = System.getProperty(key);
+        if (value == null) {
+            throw new IllegalStateException("Could not find system properties from test.fixtures. " +
+                "This test expects to run with the elasticsearch.test.fixtures Gradle plugin");
+        }
+        return value;
+    }
+
+    @Before
+    public void waitForCluster() throws IOException {
+        super.initClient();
+        Request health = new Request("GET", "/_cluster/health");
+        health.addParameter("wait_for_nodes", "2");
+        health.addParameter("wait_for_status", "yellow");
+        client().performRequest(health);
+    }
+
+    static Path keyStore;
+
+    @BeforeClass
+    public static void getKeyStore() {
+        try {
+            keyStore = PathUtils.get(DockerYmlTestSuiteIT.class.getResource("/testnode.jks").toURI());
+        } catch (URISyntaxException e) {
+            throw new ElasticsearchException("exception while reading the store", e);
+        }
+        if (Files.exists(keyStore) == false) {
+            throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist.");
+        }
+    }
+
+    @AfterClass
+    public static void clearKeyStore() {
+        keyStore = null;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        if (isOss()) {
+            return super.restClientSettings();
+        }
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .put(ESRestTestCase.TRUSTSTORE_PATH, keyStore)
+            .put(ESRestTestCase.TRUSTSTORE_PASSWORD, KEYSTORE_PASS)
+            .build();
+    }
+
+    @Override
+    protected String getProtocol() {
+        if (isOss()) {
+            return "http";
+        }
+        return "https";
+    }
+
+    private static String basicAuthHeaderValue(String username, SecureString passwd) {
+        CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1);
+        byte[] charBytes = null;
+        try {
+            chars.put(username).put(':').put(passwd.getChars());
+            charBytes = CharArrays.toUtf8Bytes(chars.array());
+
+            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
+            String basicToken = Base64.getEncoder().encodeToString(charBytes);
+            return "Basic " + basicToken;
+        } finally {
+            Arrays.fill(chars.array(), (char) 0);
+            if (charBytes != null) {
+                Arrays.fill(charBytes, (byte) 0);
+            }
+        }
+    }
+}

distribution/docker/src/test/resources/rest-api-spec/test/10_info.yml

@@ -0,0 +1,11 @@
+---
+"Info":
+  - do:         {info: {}}
+  - is_true:    name
+  - is_true:    cluster_name
+  - is_true:    cluster_uuid
+  - is_true:    tagline
+  - is_true:    version
+  - is_true:    version.number
+  - match: { version.build_type: "docker" }
+

distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml

@@ -0,0 +1,108 @@
+---
+"Test cat nodes output":
+
+  - do:
+      cat.nodes: {}
+
+  - match:
+      $body: |
+        /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role          master          name
+        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+
+  - do:
+      cat.nodes:
+        v: true
+
+  - match:
+      $body: |
+        /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role     \s+  master   \s+   name  \n
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+
+  - do:
+      cat.nodes:
+        h: heap.current,heap.percent,heap.max
+        v: true
+
+  - match:
+      $body: |
+        /^      heap\.current          \s+  heap\.percent  \s+  heap\.max              \n
+           (\s+ \d+(\.\d+)?[ptgmk]?b   \s+  \d+            \s+  \d+(\.\d+)?[ptgmk]?b   \n)+  $/
+
+  - do:
+      cat.nodes:
+        h: heap.*
+        v: true
+
+  - match:
+      $body: |
+        /^      heap\.current          \s+  heap\.percent  \s+  heap\.max              \n
+           (\s+ \d+(\.\d+)?[ptgmk]?b   \s+  \d+            \s+  \d+(\.\d+)?[ptgmk]?b   \n)+  $/
+
+  - do:
+      cat.nodes:
+        h: file_desc.current,file_desc.percent,file_desc.max
+        v: true
+
+  - match:
+      # Windows reports -1 for the file descriptor counts.
+      $body: |
+        /^      file_desc\.current  \s+  file_desc\.percent  \s+  file_desc\.max   \n
+           (\s+ (-1|\d+)            \s+  \d+                 \s+  (-1|\d+)         \n)+  $/
+
+  - do:
+      cat.nodes:
+        h: http
+        v: true
+
+  - match:
+      $body: |
+        /^      http \n ((\d{1,3}\.){3}\d{1,3}:\d{1,5}\n)+  $/
+
+---
+"Additional disk information":
+  - do:
+      cat.nodes:
+        h: diskAvail,diskTotal,diskUsed,diskUsedPercent
+        v: true
+
+  - match:
+      # leading whitespace on columns and optional whitespace on values is necessary
+      # because `diskAvail` is right aligned and text representation of disk size might be
+      # longer so it's padded with leading whitespace
+      $body: |
+        /^  \s* diskAvail            \s+ diskTotal            \s+ diskUsed            \s+ diskUsedPercent            \n
+           (\s* \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b\s+ (100\.00 | \d{1,2}\.\d{2}) \n)+  $/
+
+  - do:
+      cat.nodes:
+        h: disk,dt,du,dup
+        v: true
+
+  - match:
+      # leading whitespace on columns and optional whitespace on values is necessary
+      # because `disk` is right aligned and text representation of disk size might be
+      # longer so it's padded with leading whitespace
+      $body: |
+        /^  \s* disk                 \s+ dt                   \s+ du                  \s+ dup                        \n
+           (\s* \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+(\.\d+)?[ptgmk]?b\s+ (100\.00 | \d{1,2}\.\d{2}) \n)+  $/
+
+---
+"Test cat nodes output with full_id set":
+
+  - do:
+      cat.nodes:
+        h: id
+  # check for a 4 char non-whitespace character string
+  - match:
+      $body: |
+        /^(\S{4}\n)+$/
+
+  - do:
+      cat.nodes:
+        h: id
+        full_id: true
+  # check for a 5+ char non-whitespace character string
+  - match:
+      $body: |
+        /^(\S{5,}\n)+$/
+

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -140,11 +140,7 @@ public abstract class ESRestTestCase extends ESTestCase {
             assert clusterHosts == null;
             assert hasXPack == null;
             assert nodeVersions == null;
-            String cluster = System.getProperty("tests.rest.cluster");
-            if (cluster == null) {
-                throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] "
-                        + "to which to send REST requests");
-            }
+            String cluster = getTestRestCluster();
             String[] stringUrls = cluster.split(",");
             List<HttpHost> hosts = new ArrayList<>(stringUrls.length);
             for (String stringUrl : stringUrls) {
@@ -182,6 +178,15 @@ public abstract class ESRestTestCase extends ESTestCase {
         assert hasXPack != null;
         assert nodeVersions != null;
     }
+
+    protected String getTestRestCluster() {
+        String cluster = System.getProperty("tests.rest.cluster");
+        if (cluster == null) {
+            throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] "
+                + "to which to send REST requests");
+        }
+        return cluster;
+    }
     
     /**
      * Helper class to check warnings in REST responses with sensitivity to versions