repository-s3 yamlRestTests to new cluster test framework (#102353)

- Replace docker-compose based s3-fixture usages to use plain java
- Support lazy evaluated system properties for local cluster definitions
- Make it work with configuration cache enabled
- Port Minio fixture usage to leverage test container testing library.

build-tools-internal/build.gradle

@@ -162,7 +162,7 @@ gradlePlugin {
     stringTemplate {
        id = 'elasticsearch.string-templates'
        implementationClass = 'org.elasticsearch.gradle.internal.StringTemplatePlugin'
-     }
+    }
     testFixtures {
       id = 'elasticsearch.test.fixtures'
       implementationClass = 'org.elasticsearch.gradle.internal.testfixtures.TestFixturesPlugin'

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -91,7 +91,6 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
         List<String> sourceSetConfigurationNames = List.of(
             sourceSet.getApiConfigurationName(),
             sourceSet.getImplementationConfigurationName(),
-            sourceSet.getImplementationConfigurationName(),
             sourceSet.getCompileOnlyConfigurationName(),
             sourceSet.getRuntimeOnlyConfigurationName()
         );

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java

@@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal;
 
 import org.gradle.api.DefaultTask;
 import org.gradle.api.tasks.Input;
-import org.gradle.api.tasks.Internal;
+import org.gradle.api.tasks.OutputDirectory;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.internal.file.Chmod;
 
@@ -39,7 +39,7 @@ public class EmptyDirTask extends DefaultTask {
         throw new UnsupportedOperationException();
     }
 
-    @Internal
+    @OutputDirectory
     public File getDir() {
         return dir;
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -113,7 +113,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
         configureArtifactTransforms(project);
 
         // Create configuration for aggregating historical feature metadata
-        Configuration featureMetadataConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {
+        FileCollection featureMetadataConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {
             c.setCanBeConsumed(false);
             c.setCanBeResolved(true);
             c.attributes(
@@ -127,7 +127,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
             });
         });
 
-        Configuration defaultDistroFeatureMetadataConfig = project.getConfigurations()
+        FileCollection defaultDistroFeatureMetadataConfig = project.getConfigurations()
             .create(DEFAULT_DISTRO_FEATURES_METADATA_CONFIGURATION, c -> {
                 c.setCanBeConsumed(false);
                 c.setCanBeResolved(true);

build-tools-internal/version.properties

@@ -41,6 +41,12 @@ junit5            = 5.7.1
 hamcrest          = 2.1
 mocksocket        = 1.2
 
+# test container dependencies
+testcontainer     = 1.19.2
+dockerJava        = 3.3.4
+ductTape          = 1.0.8
+commonsCompress   = 1.24.0
+
 # benchmark dependencies
 jmh               = 1.26
 

gradle/verification-metadata.xml

@@ -381,6 +381,26 @@
             <sha256 value="452e01bc961259e2a0f5d6f193a03cc8d29f560a433f7c8066158c97d3327af9" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.github.docker-java" name="docker-java-api" version="3.3.4">
+         <artifact name="docker-java-api-3.3.4.jar">
+            <sha256 value="650710c70160c1c651e4586a07e55c5f564436cf1f28d83737265ceb5d67696f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.github.docker-java" name="docker-java-core" version="3.3.4">
+         <artifact name="docker-java-core-3.3.4.jar">
+            <sha256 value="5e2e2b3f3e4d2f1aa954f9a29762c4301bb75b8e6acdf2437771b91aa63622c2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.github.docker-java" name="docker-java-transport" version="3.3.4">
+         <artifact name="docker-java-transport-3.3.4.jar">
+            <sha256 value="d4a0f11c6f95dfa5407c3418ef5fb59a6e985542abda3f6175f763e46daa12bd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.github.docker-java" name="docker-java-transport-zerodep" version="3.3.4">
+         <artifact name="docker-java-transport-zerodep-3.3.4.jar">
+            <sha256 value="78ca58ac36881034d230a6d47959ae8dc4762ee6ca5cd36b1ee6a10e8fd18d29" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.github.javaparser" name="javaparser-core" version="3.18.0">
          <artifact name="javaparser-core-3.18.0.jar">
             <sha256 value="df7e2d5b8319efd51d015ab5071a271f8463158bc2f0979d05393fe028d856a0" origin="Generated by Gradle"/>
@@ -4072,6 +4092,11 @@
             <sha256 value="938a2d08fe54050d7610b944d8ddc3a09355710d9e6be0aac838dbc04e9a2825" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.rnorth.duct-tape" name="duct-tape" version="1.0.8">
+         <artifact name="duct-tape-1.0.8.jar">
+            <sha256 value="31cef12ddec979d1f86d7cf708c41a17da523d05c685fd6642e9d0b2addb7240" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.skyscreamer" name="jsonassert" version="1.5.0">
          <artifact name="jsonassert-1.5.0.jar">
             <sha256 value="a310bc79c3f4744e2b2e993702fcebaf3696fec0063643ffdc6b49a8fb03ef39" origin="Generated by Gradle"/>
@@ -4132,6 +4157,11 @@
             <sha256 value="7966dcd73078250f38595223b1e807cd7566188a56236def031e265426056fc8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-simple" version="2.0.6">
+         <artifact name="slf4j-simple-2.0.6.jar">
+            <sha256 value="547f06226cfcbcca198669e7498f6a6d2a55a84de5dd6eb46579f11fc40fc5d8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.sonatype.plexus" name="plexus-cipher" version="1.4">
          <artifact name="plexus-cipher-1.4.jar">
             <sha256 value="5a15fdba22669e0fdd06e10dcce6320879e1f7398fbc910cd0677b50672a78c4" origin="Generated by Gradle"/>
@@ -4162,6 +4192,11 @@
             <sha256 value="5196a0da2c5a33d1a04e88fc7a9cc109501bc265b5bac8edd9984a1885070ad4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.testcontainers" name="testcontainers" version="1.19.2">
+         <artifact name="testcontainers-1.19.2.jar">
+            <sha256 value="0218fcbfe6358c99f2a12935858829ba86dcb092ce8ca9f58fd8401c3ca7a149" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.threeten" name="threetenbp" version="1.6.5">
          <artifact name="threetenbp-1.6.5.jar">
             <sha256 value="b2551604c0d6516428e3065213b74461240378a07622201a37ad32638ecb417c" origin="Generated by Gradle"/>

modules/build.gradle

@@ -8,7 +8,7 @@
 
 configure(subprojects.findAll { it.parent.path == project.path }) {
   group = 'org.elasticsearch.plugin' // for modules which publish client jars
-  apply plugin: 'elasticsearch.internal-testclusters'
+  // apply plugin: 'elasticsearch.internal-testclusters'
   apply plugin: 'elasticsearch.internal-es-plugin'
   apply plugin: 'elasticsearch.internal-test-artifact'
 

modules/repository-s3/build.gradle

@@ -1,11 +1,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
-import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
-import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
-
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -13,7 +9,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -46,6 +42,12 @@ dependencies {
   api 'javax.xml.bind:jaxb-api:2.2.2'
 
   testImplementation project(':test:fixtures:s3-fixture')
+  yamlRestTestImplementation project(":test:framework")
+  yamlRestTestImplementation project(':test:fixtures:s3-fixture')
+  yamlRestTestImplementation project(':test:fixtures:minio-fixture')
+  internalClusterTestImplementation project(':test:fixtures:minio-fixture')
+
+  yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 }
 
 restResources {
@@ -83,13 +85,6 @@ tasks.named('test').configure {
 
 boolean useFixture = false
 
-def fixtureAddress = { fixture, name, port ->
-  assert useFixture: 'closure should not be used without a fixture'
-  int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}"
-  assert ephemeralPort > 0
-  'http://127.0.0.1:' + ephemeralPort
-}
-
 // We test against two repositories, one which uses the usual two-part "permanent" credentials and
 // the other which uses three-part "temporary" or "session" credentials.
 
@@ -124,7 +119,6 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
   s3PermanentBucket = 'bucket'
   s3PermanentBasePath = 'base_path'
 
-  apply plugin: 'elasticsearch.test.fixtures'
   useFixture = true
 
 } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
@@ -159,6 +153,9 @@ if (!s3STSBucket && !s3STSBasePath) {
 }
 
 tasks.named("processYamlRestTestResources").configure {
+  from("src/test/resources") {
+    include "aws-web-identity-token-file"
+  }
   Map<String, Object> expansions = [
     'permanent_bucket'        : s3PermanentBucket,
     'permanent_base_path'     : s3PermanentBasePath + "_integration_tests",
@@ -182,197 +179,35 @@ tasks.named("internalClusterTest").configure {
 }
 
 tasks.named("yamlRestTest").configure {
-    systemProperty 'tests.rest.blacklist', (
-      useFixture ?
-        ['repository_s3/50_repository_ecs_credentials/*',
-         'repository_s3/60_repository_sts_credentials/*']
-        :
-        [
-          'repository_s3/30_repository_temporary_credentials/*',
-          'repository_s3/40_repository_ec2_credentials/*',
-          'repository_s3/50_repository_ecs_credentials/*',
-          'repository_s3/60_repository_sts_credentials/*'
-        ]
-    ).join(",")
-}
-
-if (useFixture) {
-  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
-  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
-  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')
-
-  normalization {
-    runtimeClasspath {
-      // ignore generated address file for the purposes of build avoidance
-      ignore 's3Fixture.address'
-    }
-  }
-}
-
-testClusters.matching { it.name == "yamlRestTest" }.configureEach {
-  keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
-  keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
-
-  keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
-  keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
-  keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
-
-  if (useFixture) {
-    setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
-    setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
-    setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
-
-    // to redirect InstanceProfileCredentialsProvider to custom auth point
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
-  } else {
-    println "Using an external service to test the repository-s3 plugin"
-  }
-}
-
-// MinIO
-if (useFixture) {
-  testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
-
-  tasks.register("yamlRestTestMinio", RestIntegTestTask) {
-    description = "Runs REST tests using the Minio repository."
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-
-    // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
-    systemProperty 'tests.rest.blacklist', [
-      'repository_s3/30_repository_temporary_credentials/*',
-      'repository_s3/40_repository_ec2_credentials/*',
-      'repository_s3/50_repository_ecs_credentials/*',
-      'repository_s3/60_repository_sts_credentials/*'
-    ].join(",")
-  }
-  tasks.named("check").configure { dependsOn("yamlRestTestMinio") }
-
-  testClusters.matching { it.name == "yamlRestTestMinio" }.configureEach {
-    keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
-    keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
-    setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
-    module tasks.named("explodedBundlePlugin")
-  }
-}
-
-// ECS
-if (useFixture) {
-  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
-  tasks.register("yamlRestTestECS", RestIntegTestTask.class) {
-    description = "Runs tests using the ECS repository."
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-    systemProperty 'tests.rest.blacklist', [
-      'repository_s3/10_basic/*',
-      'repository_s3/20_repository_permanent_credentials/*',
-      'repository_s3/30_repository_temporary_credentials/*',
-      'repository_s3/40_repository_ec2_credentials/*',
-      'repository_s3/60_repository_sts_credentials/*'
-    ].join(",")
-  }
-  tasks.named("check").configure { dependsOn("yamlRestTestECS") }
-
-  testClusters.matching { it.name == "yamlRestTestECS" }.configureEach {
-    setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
-    module tasks.named('explodedBundlePlugin')
-    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
-  }
-}
-
-// STS (Secure Token Service)
-if (useFixture) {
-  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-sts')
-  tasks.register("yamlRestTestSTS", RestIntegTestTask.class) {
-    description = "Runs tests with the STS (Secure Token Service)"
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-    systemProperty 'tests.rest.blacklist', [
-      'repository_s3/10_basic/*',
-      'repository_s3/20_repository_permanent_credentials/*',
-      'repository_s3/30_repository_temporary_credentials/*',
-      'repository_s3/40_repository_ec2_credentials/*',
-      'repository_s3/50_repository_ecs_credentials/*'
-    ].join(",")
-  }
-  tasks.named("check").configure { dependsOn("yamlRestTestSTS") }
-
-  testClusters.matching { it.name == "yamlRestTestSTS" }.configureEach {
-    module tasks.named("explodedBundlePlugin")
-
-    setting 's3.client.integration_test_sts.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-sts', '80')}" }, IGNORE_VALUE
-    systemProperty 'com.amazonaws.sdk.stsMetadataServiceEndpointOverride',
-      { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-sts', '80')}/assume-role-with-web-identity" }, IGNORE_VALUE
+  systemProperty("tests.use.fixture", Boolean.toString(useFixture))
+  systemProperty("s3PermanentAccessKey", s3PermanentAccessKey)
+  systemProperty("s3PermanentSecretKey", s3PermanentSecretKey)
+  systemProperty("s3TemporaryAccessKey", s3TemporaryAccessKey)
+  systemProperty("s3TemporarySecretKey", s3TemporarySecretKey)
+  systemProperty("s3EC2AccessKey", s3PermanentAccessKey)
 
-    File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file')
-    // The web identity token can be read only from the plugin config directory because of security restrictions
-    // Ideally we would create a symlink, but extraConfigFile doesn't support it
-    extraConfigFile 'repository-s3/aws-web-identity-token-file', awsWebIdentityTokenExternalLocation
-    environment 'AWS_WEB_IDENTITY_TOKEN_FILE', "$awsWebIdentityTokenExternalLocation"
-
-    // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the S3HttpFixtureWithSTS fixture
-    environment 'AWS_ROLE_ARN', 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole'
-    environment 'AWS_ROLE_SESSION_NAME', 'sts-fixture-test'
-  }
-}
-
-// Sanity test for STS Regional Endpoints
-if (useFixture) {
-  tasks.register("yamlRestTestRegionalSTS", RestIntegTestTask.class) {
-    description = "Runs tests with the Regional STS Endpoint"
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-    // Run just the basic sanity test to make sure ES starts up and loads the S3 repository with
-    // a regional endpoint without an error. It would be great to make actual requests against
-    // a test fixture, but setting the region means using a production endpoint
-    systemProperty 'tests.rest.blacklist', [
-      'repository_s3/20_repository_permanent_credentials/*',
-      'repository_s3/30_repository_temporary_credentials/*',
-      'repository_s3/40_repository_ec2_credentials/*',
-      'repository_s3/50_repository_ecs_credentials/*',
-      'repository_s3/60_repository_sts_credentials/*'
-    ].join(",")
-  }
-  tasks.named("check").configure { dependsOn("yamlRestTestRegionalSTS") }
-
-  testClusters.matching { it.name == "yamlRestTestRegionalSTS" }.configureEach {
-    module tasks.named("explodedBundlePlugin")
-
-    File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file')
-    extraConfigFile 'repository-s3/aws-web-identity-token-file', awsWebIdentityTokenExternalLocation
-    environment 'AWS_WEB_IDENTITY_TOKEN_FILE', "$awsWebIdentityTokenExternalLocation"
-    environment 'AWS_ROLE_ARN', 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole'
-    environment 'AWS_ROLE_SESSION_NAME', 'sts-fixture-test'
-    // Force the repository to set a regional production endpoint
-    environment 'AWS_STS_REGIONAL_ENDPOINTS', 'regional'
-    environment 'AWS_REGION', 'ap-southeast-2'
-  }
+  // ideally we could resolve an env path in cluster config as resource similar to configuring a config file
+  // not sure how common this is, but it would be nice to support
+  File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file')
+  // The web identity token can be read only from the plugin config directory because of security restrictions
+  // Ideally we would create a symlink, but extraConfigFile doesn't support it
+  nonInputProperties.systemProperty("awsWebIdentityTokenExternalLocation", awsWebIdentityTokenExternalLocation.getAbsolutePath())
 }
 
 // 3rd Party Tests
-TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
+tasks.register("s3ThirdPartyTest", Test) {
   SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
   SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
   setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
   setClasspath(internalTestSourceSet.getRuntimeClasspath())
   include '**/S3RepositoryThirdPartyTests.class'
+  // test container accesses ~/.testcontainers.properties read
+  systemProperty "tests.security.manager", "false"
   systemProperty 'test.s3.account', s3PermanentAccessKey
   systemProperty 'test.s3.key', s3PermanentSecretKey
   systemProperty 'test.s3.bucket', s3PermanentBucket
   nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
-  if (useFixture) {
-    nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
-  }
 }
-tasks.named("check").configure { dependsOn(s3ThirdPartyTest) }
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
@@ -405,3 +240,8 @@ tasks.named("thirdPartyAudit").configure {
           'javax.activation.DataHandler'
   )
 }
+
+tasks.named("check").configure {
+  dependsOn(tasks.withType(Test))
+}
+

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java

@@ -11,6 +11,7 @@ import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
 import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
 import com.amazonaws.services.s3.model.MultipartUpload;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -31,8 +32,11 @@ import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.fixtures.minio.MinioTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -48,8 +52,12 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
 
+    @ClassRule
+    public static MinioTestContainer minio = new MinioTestContainer(true);
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(S3RepositoryPlugin.class);
@@ -92,7 +100,7 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTes
         Settings.Builder settings = Settings.builder()
             .put("bucket", System.getProperty("test.s3.bucket"))
             .put("base_path", System.getProperty("test.s3.base", "testpath"));
-        final String endpoint = System.getProperty("test.s3.endpoint");
+        final String endpoint = minio.getAddress();
         if (endpoint != null) {
             settings.put("endpoint", endpoint);
         } else {

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3ClientYamlTestSuiteIT.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public abstract class AbstractRepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("tests.use.fixture", "true"));
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    public AbstractRepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+}

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -8,20 +8,67 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.s3.S3HttpFixture;
+import fixture.s3.S3HttpFixtureWithEC2;
+import fixture.s3.S3HttpFixtureWithSessionToken;
+
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
+    static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("tests.use.fixture", "true"));
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE);
+    public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(USE_FIXTURE);
+    public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2(USE_FIXTURE);
 
-public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    private static final String s3TemporarySessionToken = "session_token";
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))
+        .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey"))
+        .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey"))
+        .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey"))
+        .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken)
+        .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress)
+        .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress, (nodeSpec) -> USE_FIXTURE)
+        .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress, (n) -> USE_FIXTURE)
+        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress, (n) -> USE_FIXTURE)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster);
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters(
+            USE_FIXTURE
+                ? new String[] {
+                    "repository_s3/10_basic",
+                    "repository_s3/20_repository_permanent_credentials",
+                    "repository_s3/30_repository_temporary_credentials",
+                    "repository_s3/40_repository_ec2_credentials" }
+                : new String[] { "repository_s3/10_basic", "repository_s3/20_repository_permanent_credentials" }
+        );
+    }
 
     public RepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
     }
 }

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.s3.S3HttpFixtureWithECS;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
+    private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS(USE_FIXTURE);
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress)
+        .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint"))
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster);
+
+    @BeforeClass
+    public static void onlyWhenRunWithTestFixture() {
+        assumeTrue("Only run with fixture enabled", USE_FIXTURE);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters(new String[] { "repository_s3/50_repository_ecs_credentials" });
+    }
+
+    public RepositoryS3EcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioClientYamlTestSuiteIT.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.fixtures.minio.MinioTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+public class RepositoryS3MinioClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
+
+    public static MinioTestContainer minio = new MinioTestContainer(USE_FIXTURE);
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))
+        .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey"))
+        .setting("s3.client.integration_test_permanent.endpoint", () -> minio.getAddress())
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(minio).around(cluster);
+
+    @BeforeClass
+    public static void onlyWhenRunWithTestFixture() {
+        assumeTrue("Only run with fixture enabled", USE_FIXTURE);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters(new String[] { "repository_s3/10_basic", "repository_s3/20_repository_permanent_credentials" });
+    }
+
+    public RepositoryS3MinioClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RegionalStsClientYamlTestSuiteIT.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+public class RepositoryS3RegionalStsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file"))
+        .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation"))
+        // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the
+        // S3HttpFixtureWithSTS fixture
+        .environment("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/FederatedWebIdentityRole")
+        .environment("AWS_ROLE_SESSION_NAME", "sts-fixture-test")
+        .environment("AWS_STS_REGIONAL_ENDPOINTS", "regional")
+        .environment("AWS_REGION", "ap-southeast-2")
+        .build();
+
+    @BeforeClass
+    public static void onlyWhenRunWithTestFixture() {
+        assumeTrue("Only run with fixture enabled", USE_FIXTURE);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters(new String[] { "repository_s3/10_basic" });
+    }
+
+    public RepositoryS3RegionalStsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.s3.S3HttpFixture;
+import fixture.s3.S3HttpFixtureWithSTS;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
+    static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("tests.use.fixture", "true"));
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE);
+    private static final S3HttpFixtureWithSTS s3Sts = new S3HttpFixtureWithSTS(USE_FIXTURE);
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .setting("s3.client.integration_test_sts.endpoint", s3Sts::getAddress)
+        .systemProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", () -> s3Sts.getAddress() + "/assume-role-with-web-identity")
+        .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file"))
+        .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation"))
+        // // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the
+        // // S3HttpFixtureWithSTS fixture
+        .environment("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/FederatedWebIdentityRole")
+        .environment("AWS_ROLE_SESSION_NAME", "sts-fixture-test")
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Sts).around(cluster);
+
+    @BeforeClass
+    public static void onlyWhenRunWithTestFixture() {
+        assumeTrue("Only run with fixture enabled", USE_FIXTURE);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters(new String[] { "repository_s3/60_repository_sts_credentials" });
+    }
+
+    public RepositoryS3StsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

test/fixtures/minio-fixture/build.gradle

@@ -6,6 +6,27 @@
  * Side Public License, v 1.
  */
 apply plugin: 'elasticsearch.test.fixtures'
+apply plugin: 'java'
+apply plugin: 'elasticsearch.java'
 
 description = 'Fixture for MinIO Storage service'
 
+configurations.all {
+  transitive = false
+}
+
+dependencies {
+  testImplementation project(':test:framework')
+
+  api "junit:junit:${versions.junit}"
+  api "org.testcontainers:testcontainers:${versions.testcontainer}"
+  implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}"
+  runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}"
+  runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}"
+  runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}"
+  runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}"
+  runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}"
+  runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}"
+}

test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.fixtures.minio;
+
+import org.junit.rules.TestRule;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+
+public class MinioTestContainer extends GenericContainer<MinioTestContainer> implements TestRule {
+
+    private static final int servicePort = 9000;
+    private final boolean enabled;
+
+    public MinioTestContainer(boolean enabled) {
+        super(
+            new ImageFromDockerfile().withDockerfileFromBuilder(
+                builder -> builder.from("minio/minio:RELEASE.2021-03-01T04-20-55Z")
+                    .env("MINIO_ACCESS_KEY", "s3_test_access_key")
+                    .env("MINIO_SECRET_KEY", "s3_test_secret_key")
+                    .run("mkdir -p /minio/data/bucket")
+                    .cmd("server", "/minio/data")
+                    .build()
+            )
+        );
+        if (enabled) {
+            addExposedPort(servicePort);
+        }
+        this.enabled = enabled;
+    }
+
+    @Override
+    public void start() {
+        if (enabled) {
+            super.start();
+        }
+    }
+
+    public String getAddress() {
+        return "http://127.0.0.1:" + getMappedPort(servicePort);
+    }
+}

test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.fixtures.testcontainers;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+public class TestContainersThreadFilter implements ThreadFilter {
+    @Override
+    public boolean reject(Thread t) {
+        return t.getName().startsWith("testcontainers-") || t.getName().startsWith("ducttape");
+    }
+}

test/fixtures/s3-fixture/build.gradle

@@ -13,6 +13,9 @@ tasks.named("test").configure { enabled = false }
 
 dependencies {
   api project(':server')
+  api("junit:junit:${versions.junit}") {
+    transitive = false
+  }
   testImplementation project(':test:framework')
 }
 

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java

@@ -12,22 +12,44 @@ import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
 
 import org.elasticsearch.rest.RestStatus;
+import org.junit.rules.ExternalResource;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Arrays;
 import java.util.Objects;
 
-public class S3HttpFixture {
+public class S3HttpFixture extends ExternalResource {
 
     private final HttpServer server;
 
-    S3HttpFixture(final String[] args) throws Exception {
-        this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(args[0]), Integer.parseInt(args[1])), 0);
-        this.server.createContext("/", Objects.requireNonNull(createHandler(args)));
+    private boolean enabled;
+
+    public S3HttpFixture(boolean enabled) {
+        this(enabled, "bucket", "base_path_integration_tests", "s3_test_access_key");
+    }
+
+    public S3HttpFixture(boolean enabled, String... args) {
+        this(resolveAddress("localhost", 0), args);
+        this.enabled = enabled;
     }
 
-    final void start() throws Exception {
+    public S3HttpFixture(final String[] args) throws Exception {
+        this(resolveAddress(args[0], Integer.parseInt(args[1])), args);
+    }
+
+    public S3HttpFixture(InetSocketAddress inetSocketAddress, String... args) {
+        try {
+            this.server = HttpServer.create(inetSocketAddress, 0);
+            this.server.createContext("/", Objects.requireNonNull(createHandler(args)));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    final void startWithWait() throws Exception {
         try {
             server.start();
             // wait to be killed
@@ -37,10 +59,14 @@ public class S3HttpFixture {
         }
     }
 
+    public final void start() throws Exception {
+        server.start();
+    }
+
     protected HttpHandler createHandler(final String[] args) {
-        final String bucket = Objects.requireNonNull(args[2]);
-        final String basePath = args[3];
-        final String accessKey = Objects.requireNonNull(args[4]);
+        final String bucket = Objects.requireNonNull(args[0]);
+        final String basePath = args[1];
+        final String accessKey = Objects.requireNonNull(args[2]);
 
         return new S3HttpHandler(bucket, basePath) {
             @Override
@@ -59,7 +85,37 @@ public class S3HttpFixture {
         if (args == null || args.length < 5) {
             throw new IllegalArgumentException("S3HttpFixture expects 5 arguments [address, port, bucket, base path, access key]");
         }
-        final S3HttpFixture fixture = new S3HttpFixture(args);
-        fixture.start();
+        InetSocketAddress inetSocketAddress = resolveAddress(args[0], Integer.parseInt(args[1]));
+        final S3HttpFixture fixture = new S3HttpFixture(inetSocketAddress, Arrays.copyOfRange(args, 2, args.length));
+        fixture.startWithWait();
+    }
+
+    public String getAddress() {
+        return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort();
+    }
+
+    public void stop(int delay) {
+        server.stop(delay);
+    }
+
+    protected void before() throws Throwable {
+        if (enabled) {
+            start();
+        }
+    }
+
+    @Override
+    protected void after() {
+        if (enabled) {
+            stop(0);
+        }
+    }
+
+    private static InetSocketAddress resolveAddress(String address, int port) {
+        try {
+            return new InetSocketAddress(InetAddress.getByName(address), port);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
     }
 }

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java

@@ -11,6 +11,7 @@ import com.sun.net.httpserver.HttpHandler;
 
 import org.elasticsearch.rest.RestStatus;
 
+import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -22,14 +23,26 @@ public class S3HttpFixtureWithEC2 extends S3HttpFixtureWithSessionToken {
     private static final String EC2_PATH = "/latest/meta-data/iam/security-credentials/";
     private static final String EC2_PROFILE = "ec2Profile";
 
-    S3HttpFixtureWithEC2(final String[] args) throws Exception {
+    public S3HttpFixtureWithEC2(final String[] args) throws Exception {
         super(args);
     }
 
+    public S3HttpFixtureWithEC2(boolean enabled) {
+        this(enabled, "ec2_bucket", "ec2_base_path", "ec2_access_key", "ec2_session_token");
+    }
+
+    public S3HttpFixtureWithEC2(boolean enabled, String... args) {
+        super(enabled, args);
+    }
+
+    public S3HttpFixtureWithEC2(InetSocketAddress inetSocketAddress, String[] strings) {
+        super(inetSocketAddress, strings);
+    }
+
     @Override
     protected HttpHandler createHandler(final String[] args) {
-        final String ec2AccessKey = Objects.requireNonNull(args[4]);
-        final String ec2SessionToken = Objects.requireNonNull(args[5], "session token is missing");
+        final String ec2AccessKey = Objects.requireNonNull(args[2]);
+        final String ec2SessionToken = Objects.requireNonNull(args[3], "session token is missing");
         final HttpHandler delegate = super.createHandler(args);
 
         return exchange -> {
@@ -83,6 +96,6 @@ public class S3HttpFixtureWithEC2 extends S3HttpFixtureWithSessionToken {
             );
         }
         final S3HttpFixtureWithEC2 fixture = new S3HttpFixtureWithEC2(args);
-        fixture.start();
+        fixture.startWithWait();
     }
 }

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java

@@ -11,19 +11,32 @@ import com.sun.net.httpserver.HttpHandler;
 
 import org.elasticsearch.rest.RestStatus;
 
+import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
 public class S3HttpFixtureWithECS extends S3HttpFixtureWithEC2 {
 
-    private S3HttpFixtureWithECS(final String[] args) throws Exception {
+    public S3HttpFixtureWithECS(final String[] args) throws Exception {
         super(args);
     }
 
+    public S3HttpFixtureWithECS(boolean enabled) {
+        this(enabled, "ecs_bucket", "ecs_base_path", "ecs_access_key", "ecs_session_token");
+    }
+
+    public S3HttpFixtureWithECS(boolean enabled, String... args) {
+        super(enabled, args);
+    }
+
+    public S3HttpFixtureWithECS(InetSocketAddress inetSocketAddress, String[] strings) {
+        super(inetSocketAddress, strings);
+    }
+
     @Override
     protected HttpHandler createHandler(final String[] args) {
-        final String ecsAccessKey = Objects.requireNonNull(args[4]);
-        final String ecsSessionToken = Objects.requireNonNull(args[5], "session token is missing");
+        final String ecsAccessKey = Objects.requireNonNull(args[2]);
+        final String ecsSessionToken = Objects.requireNonNull(args[3], "session token is missing");
         final HttpHandler delegate = super.createHandler(args);
 
         return exchange -> {
@@ -47,6 +60,6 @@ public class S3HttpFixtureWithECS extends S3HttpFixtureWithEC2 {
             );
         }
         final S3HttpFixtureWithECS fixture = new S3HttpFixtureWithECS(args);
-        fixture.start();
+        fixture.startWithWait();
     }
 }

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSTS.java

@@ -11,6 +11,8 @@ import com.sun.net.httpserver.HttpHandler;
 
 import org.elasticsearch.rest.RestStatus;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.time.ZonedDateTime;
@@ -26,15 +28,34 @@ public class S3HttpFixtureWithSTS extends S3HttpFixture {
     private static final String ROLE_ARN = "arn:aws:iam::123456789012:role/FederatedWebIdentityRole";
     private static final String ROLE_NAME = "sts-fixture-test";
 
-    private S3HttpFixtureWithSTS(final String[] args) throws Exception {
+    public S3HttpFixtureWithSTS(boolean enabled) {
+        this(
+            enabled,
+            "sts_bucket",
+            "sts_base_path",
+            "sts_access_key",
+            "sts_session_token",
+            "Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDansFBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFOzTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ"
+        );
+    }
+
+    public S3HttpFixtureWithSTS(boolean enabled, String... args) {
+        super(enabled, args);
+    }
+
+    public S3HttpFixtureWithSTS(final String[] args) throws Exception {
         super(args);
     }
 
+    public S3HttpFixtureWithSTS(InetSocketAddress inetSocketAddress, String[] args) throws IOException {
+        super(inetSocketAddress, args);
+    }
+
     @Override
     protected HttpHandler createHandler(final String[] args) {
-        String accessKey = Objects.requireNonNull(args[4]);
-        String sessionToken = Objects.requireNonNull(args[5], "session token is missing");
-        String webIdentityToken = Objects.requireNonNull(args[6], "web identity token is missing");
+        String accessKey = Objects.requireNonNull(args[2]);
+        String sessionToken = Objects.requireNonNull(args[3], "session token is missing");
+        String webIdentityToken = Objects.requireNonNull(args[4], "web identity token is missing");
         final HttpHandler delegate = super.createHandler(args);
 
         return exchange -> {
@@ -106,6 +127,6 @@ public class S3HttpFixtureWithSTS extends S3HttpFixture {
             );
         }
         final S3HttpFixtureWithSTS fixture = new S3HttpFixtureWithSTS(args);
-        fixture.start();
+        fixture.startWithWait();
     }
 }

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java

@@ -11,6 +11,7 @@ import com.sun.net.httpserver.HttpHandler;
 
 import org.elasticsearch.rest.RestStatus;
 
+import java.net.InetSocketAddress;
 import java.util.Objects;
 
 import static fixture.s3.S3HttpHandler.sendError;
@@ -21,9 +22,21 @@ public class S3HttpFixtureWithSessionToken extends S3HttpFixture {
         super(args);
     }
 
+    public S3HttpFixtureWithSessionToken(boolean enabled) {
+        this(enabled, "session_token_bucket", "session_token_base_path_integration_tests", "session_token_access_key", "session_token");
+    }
+
+    public S3HttpFixtureWithSessionToken(boolean enabled, String... args) {
+        super(enabled, args);
+    }
+
+    public S3HttpFixtureWithSessionToken(InetSocketAddress inetSocketAddress, String[] args) {
+        super(inetSocketAddress, args);
+    }
+
     @Override
     protected HttpHandler createHandler(final String[] args) {
-        final String sessionToken = Objects.requireNonNull(args[5], "session token is missing");
+        final String sessionToken = Objects.requireNonNull(args[3], "session token is missing");
         final HttpHandler delegate = super.createHandler(args);
         return exchange -> {
             final String securityToken = exchange.getRequestHeaders().getFirst("x-amz-security-token");
@@ -46,6 +59,6 @@ public class S3HttpFixtureWithSessionToken extends S3HttpFixture {
             );
         }
         final S3HttpFixtureWithSessionToken fixture = new S3HttpFixtureWithSessionToken(args);
-        fixture.start();
+        fixture.startWithWait();
     }
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/SystemPropertyProvider.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.LocalClusterSpec;
+
+import java.util.Map;
+
+/**
+ * Functional interface for supplying system properties to an Elasticsearch node. This interface is designed to be implemented by tests
+ * and fixtures wanting to provide system properties to an {@link ElasticsearchCluster} in a dynamic fashion.
+ * Instances are evaluated lazily at cluster start time.
+ */
+public interface SystemPropertyProvider {
+
+    /**
+     * Returns a collection of system properties to apply to an Elasticsearch cluster node. This method is called when the cluster is
+     * started so implementors can return dynamic environment values that may or may not be based on the given node spec.
+     *
+     * @param nodeSpec the specification for the given node to apply settings to
+     * @return system property variables to add to the node
+     */
+    Map<String, String> get(LocalClusterSpec.LocalNodeSpec nodeSpec);
+}

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java

@@ -715,9 +715,9 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
             }
 
             String systemProperties = "";
-            if (spec.getSystemProperties().isEmpty() == false) {
-                systemProperties = spec.getSystemProperties()
-                    .entrySet()
+            Map<String, String> resolvedSystemProperties = new HashMap<>(spec.resolveSystemProperties());
+            if (resolvedSystemProperties.isEmpty() == false) {
+                systemProperties = resolvedSystemProperties.entrySet()
                     .stream()
                     .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
                     .map(p -> p.replace("${ES_PATH_CONF}", configDir.toString()))

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java

@@ -183,6 +183,7 @@ public abstract class AbstractLocalClusterSpecBuilder<T extends ElasticsearchClu
                 getKeystoreFiles(),
                 getKeystorePassword(),
                 getExtraConfigFiles(),
+                getSystemPropertyProviders(),
                 getSystemProperties(),
                 getJvmArgs()
             );

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java

@@ -11,6 +11,7 @@ package org.elasticsearch.test.cluster.local;
 import org.elasticsearch.test.cluster.EnvironmentProvider;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.SystemPropertyProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.Version;
 import org.elasticsearch.test.cluster.util.resource.Resource;
@@ -38,6 +39,7 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
     private final Map<String, Resource> keystoreFiles = new HashMap<>();
     private final Map<String, Resource> extraConfigFiles = new HashMap<>();
     private final Map<String, String> systemProperties = new HashMap<>();
+    private final List<SystemPropertyProvider> systemPropertyProviders = new ArrayList<>();
     private final List<String> jvmArgs = new ArrayList<>();
     private DistributionType distributionType;
     private Version version;
@@ -204,10 +206,31 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
         return cast(this);
     }
 
+    @Override
+    public T systemProperty(String key, Supplier<String> supplier) {
+        this.systemPropertyProviders.add(s -> Map.of(key, supplier.get()));
+        return cast(this);
+    }
+
+    public T systemProperty(SystemPropertyProvider systemPropertyProvider) {
+        this.systemPropertyProviders.add(systemPropertyProvider);
+        return cast(this);
+    }
+
+    @Override
+    public T systemProperty(String key, Supplier<String> value, Predicate<LocalClusterSpec.LocalNodeSpec> predicate) {
+        this.systemPropertyProviders.add(s -> predicate.test(s) ? Map.of(key, value.get()) : Map.of());
+        return cast(this);
+    }
+
     public Map<String, String> getSystemProperties() {
         return inherit(() -> parent.getSystemProperties(), systemProperties);
     }
 
+    public List<SystemPropertyProvider> getSystemPropertyProviders() {
+        return inherit(() -> parent.getSystemPropertyProviders(), systemPropertyProviders);
+    }
+
     @Override
     public T jvmArg(String arg) {
         this.jvmArgs.add(arg);

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java

@@ -34,4 +34,5 @@ public final class DefaultLocalClusterSpecBuilder extends AbstractLocalClusterSp
             )
         );
     }
+
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java

@@ -12,6 +12,7 @@ import org.elasticsearch.test.cluster.ClusterSpec;
 import org.elasticsearch.test.cluster.EnvironmentProvider;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.SystemPropertyProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.local.model.User;
 import org.elasticsearch.test.cluster.util.Version;
@@ -88,6 +89,7 @@ public class LocalClusterSpec implements ClusterSpec {
         private final Map<String, Resource> keystoreFiles;
         private final String keystorePassword;
         private final Map<String, Resource> extraConfigFiles;
+        private final List<SystemPropertyProvider> systemPropertyProviders;
         private final Map<String, String> systemProperties;
         private final List<String> jvmArgs;
         private Version version;
@@ -109,6 +111,7 @@ public class LocalClusterSpec implements ClusterSpec {
             Map<String, Resource> keystoreFiles,
             String keystorePassword,
             Map<String, Resource> extraConfigFiles,
+            List<SystemPropertyProvider> systemPropertyProviders,
             Map<String, String> systemProperties,
             List<String> jvmArgs
         ) {
@@ -128,6 +131,7 @@ public class LocalClusterSpec implements ClusterSpec {
             this.keystoreFiles = keystoreFiles;
             this.keystorePassword = keystorePassword;
             this.extraConfigFiles = extraConfigFiles;
+            this.systemPropertyProviders = systemPropertyProviders;
             this.systemProperties = systemProperties;
             this.jvmArgs = jvmArgs;
         }
@@ -184,10 +188,6 @@ public class LocalClusterSpec implements ClusterSpec {
             return extraConfigFiles;
         }
 
-        public Map<String, String> getSystemProperties() {
-            return systemProperties;
-        }
-
         public List<String> getJvmArgs() {
             return jvmArgs;
         }
@@ -278,6 +278,24 @@ public class LocalClusterSpec implements ClusterSpec {
             return resolvedEnvironment;
         }
 
+        /**
+         * Resolve node system properties. Order of precedence is as follows:
+         * <ol>
+         *     <li>SystemProperties from cluster configured {@link SystemPropertyProvider}</li>
+         *     <li>SystemProperties variables from node configured {@link SystemPropertyProvider}</li>
+         *     <li>SystemProperties variables cluster settings</li>
+         *     <li>SystemProperties variables node settings</li>
+         * </ol>
+         *
+         * @return resolved system properties for node
+         */
+        public Map<String, String> resolveSystemProperties() {
+            Map<String, String> resolvedSystemProperties = new HashMap<>();
+            systemPropertyProviders.forEach(p -> resolvedSystemProperties.putAll(p.get(this)));
+            resolvedSystemProperties.putAll(systemProperties);
+            return resolvedSystemProperties;
+        }
+
         /**
          * Returns a new {@link LocalNodeSpec} without the given {@link SettingsProvider}s. This is needed when resolving settings from a
          * settings provider to avoid infinite recursion.
@@ -308,6 +326,7 @@ public class LocalClusterSpec implements ClusterSpec {
                         n.keystoreFiles,
                         n.keystorePassword,
                         n.extraConfigFiles,
+                        n.systemPropertyProviders,
                         n.systemProperties,
                         n.jvmArgs
                     )

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java

@@ -11,6 +11,7 @@ package org.elasticsearch.test.cluster.local;
 import org.elasticsearch.test.cluster.EnvironmentProvider;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.SystemPropertyProvider;
 import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.Version;
@@ -121,6 +122,22 @@ interface LocalSpecBuilder<T extends LocalSpecBuilder<?>> {
      */
     T systemProperty(String property, String value);
 
+    /**
+     * Adds a system property to node JVM arguments computed by the given supplier
+     */
+    T systemProperty(String property, Supplier<String> supplier);
+
+    /**
+     * Adds a system property to node JVM arguments computed by the given supplier
+     * when the given predicate evaluates to {@code true}.
+     */
+    T systemProperty(String setting, Supplier<String> value, Predicate<LocalNodeSpec> predicate);
+
+    /**
+     * Register a {@link SystemPropertyProvider}.
+     */
+    T systemProperty(SystemPropertyProvider systemPropertyProvider);
+
     /**
      * Adds an additional command line argument to node JVM arguments.
      */