Add repository-url module and move URLRepository (#22752)

This is related to #22116. URLRepository requires SocketPermission
connect. This commit introduces a new module called "repository-url"
where URLRepository will reside. With the new module, permissions can
be removed from core.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -276,8 +276,7 @@ class ClusterFormationTasks {
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test',
-                'repositories.url.allowed_urls': 'http://snapshot.test*'
+                'node.attr.testattr'           : 'test'
         ]
         // we set min master nodes to the total number of nodes in the cluster and
         // basically skip initial state recovery to allow the cluster to form using a realistic master election

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -256,7 +256,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Base64.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Numbers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]fs[/\\]FsBlobStore.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]url[/\\]URLBlobStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]BytesArray.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]PagedBytesReference.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cache[/\\]Cache.java" checks="LineLength" />
@@ -437,7 +436,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]ChecksumBlobStoreFormat.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -81,7 +81,6 @@ import org.elasticsearch.monitor.process.ProcessService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
@@ -348,9 +347,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Node.NODE_INGEST_SETTING,
                     Node.NODE_ATTRIBUTES,
                     Node.NODE_LOCAL_STORAGE_SETTING,
-                    URLRepository.ALLOWED_URLS_SETTING,
-                    URLRepository.REPOSITORIES_URL_SETTING,
-                    URLRepository.SUPPORTED_PROTOCOLS_SETTING,
                     TransportMasterNodeReadAction.FORCE_LOCAL_SETTING,
                     AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,
                     BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX,

core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java

@@ -19,24 +19,22 @@
 
 package org.elasticsearch.repositories;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * Sets up classes for Snapshot/Restore.
  */
@@ -47,7 +45,6 @@ public class RepositoriesModule extends AbstractModule {
     public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins, NamedXContentRegistry namedXContentRegistry) {
         Map<String, Repository.Factory> factories = new HashMap<>();
         factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry));
-        factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry));
 
         for (RepositoryPlugin repoPlugin : repoPlugins) {
             Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry);

core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java

@@ -26,21 +26,22 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.common.io.FileTestUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.repositories.uri.URLRepository;
+import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.VersionUtils;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -61,27 +62,19 @@ import static org.hamcrest.Matchers.notNullValue;
 @ClusterScope(scope = Scope.TEST)
 public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
 
+    private static Path repoPath;
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        if (randomBoolean()) {
-            // Configure using path.repo
-            return Settings.builder()
-                    .put(super.nodeSettings(nodeOrdinal))
-                    .put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath())
-                    .build();
-        } else {
-            // Configure using url white list
-            try {
-                URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/");
-                return Settings.builder()
-                        .put(super.nodeSettings(nodeOrdinal))
-                        .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString())
-                        .build();
-            } catch (URISyntaxException ex) {
-                throw new IllegalArgumentException(ex);
-            }
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(Environment.PATH_REPO_SETTING.getKey(), repoPath)
+            .build();
+    }
 
-        }
+    @BeforeClass
+    public static void repoSetup() throws IOException {
+        repoPath = createTempDir("repositories");
     }
 
     public void testRestoreOldSnapshots() throws Exception {
@@ -151,13 +144,14 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
     }
 
     private void createRepo(String prefix, String version, String repo) throws Exception {
-        Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
-        URI repoFileUri = repoFile.toUri();
-        URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
+        Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
+        String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0];
+        Path fsRepoPath = repoPath.resolve(repoFileName);
+        FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null);
         logger.info("-->  creating repository [{}] for version [{}]", repo, version);
         assertAcked(client().admin().cluster().preparePutRepository(repo)
-                .setType("url").setSettings(Settings.builder()
-                        .put("url", repoJarUri.toString())));
+            .setType(MockRepository.TYPE).setSettings(Settings.builder()
+                .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString())));
     }
 
     private void testOldSnapshot(String version, String repo, String snapshot) throws IOException {
@@ -198,7 +192,7 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
                 equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"),
                 equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"),
                 equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}")
-        ));
+            ));
         assertThat(template.aliases().size(), equalTo(3));
         assertThat(template.aliases().get("alias1"), notNullValue());
         assertThat(template.aliases().get("alias2").filter().string(), containsString(version));

core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java

@@ -142,30 +142,6 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
         } catch (RepositoryException ex) {
             assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo"));
         }
-
-        String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString();
-        String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/");
-        logger.info("--> trying creating url repository with unsupported url protocol");
-        try {
-            client().admin().cluster().preparePutRepository("test-repo")
-                    .setType("url").setSettings(Settings.builder().put("url", unsupportedUrl))
-                    .get();
-            fail("Shouldn't be here");
-        } catch (RepositoryException ex) {
-            assertThat(ex.toString(),
-                either(containsString("unsupported url protocol [netdoc]"))
-                    .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9
-        }
-
-        logger.info("--> trying creating url repository with location that is not registered in path.repo setting");
-        try {
-            client().admin().cluster().preparePutRepository("test-repo")
-                    .setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL()))
-                    .get();
-            fail("Shouldn't be here");
-        } catch (RepositoryException ex) {
-            assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo"));
-        }
     }
 
     public void testRepositoryAckTimeout() throws Exception {

core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@@ -1442,63 +1442,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
     }
 
-    public void testUrlRepository() throws Exception {
-        Client client = client();
-
-        logger.info("-->  creating repository");
-        Path repositoryLocation = randomRepoPath();
-        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
-                .setType("fs").setSettings(Settings.builder()
-                        .put("location", repositoryLocation)
-                        .put("compress", randomBoolean())
-                        .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
-
-        createIndex("test-idx");
-        ensureGreen();
-
-        logger.info("--> indexing some data");
-        for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
-
-        logger.info("--> delete index");
-        cluster().wipeIndices("test-idx");
-
-        logger.info("--> create read-only URL repository");
-        assertAcked(client.admin().cluster().preparePutRepository("url-repo")
-                .setType("url").setSettings(Settings.builder()
-                        .put("url", repositoryLocation.toUri().toURL())
-                        .put("list_directories", randomBoolean())));
-        logger.info("--> restore index after deletion");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> list available shapshots");
-        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
-
-        logger.info("--> delete snapshot");
-        DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
-        assertAcked(deleteSnapshotResponse);
-
-        logger.info("--> list available shapshot again, no snapshots should be returned");
-        getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
-    }
-
     @TestLogging("_root:DEBUG")  // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
     // more logs we cannot find out why
     public void testReadonlyRepository() throws Exception {

modules/repository-url/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+    description 'Module for URL repository'
+    classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
+}
+
+integTest {
+    cluster {
+        setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    }
+}

core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java → modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java

@@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
     public URLBlobStore(Settings settings, URL path) {
         super(settings);
         this.path = path;
-        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
+        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size",
+            new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
     }
 
     /**

modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.repository.url;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.repositories.url.URLRepository;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin {
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+            URLRepository.ALLOWED_URLS_SETTING,
+            URLRepository.REPOSITORIES_URL_SETTING,
+            URLRepository.SUPPORTED_PROTOCOLS_SETTING
+        );
+    }
+
+    @Override
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+        return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry));
+    }
+}

core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java → modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.repositories.uri;
+package org.elasticsearch.repositories.url;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -127,8 +127,12 @@ public class URLRepository extends BlobStoreRepository {
                 // We didn't match white list - try to resolve against path.repo
                 URL normalizedUrl = environment.resolveRepoURL(url);
                 if (normalizedUrl == null) {
-                    logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
-                    throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey());
+                    String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " +
+                        "path.repo setting or by {} setting: [{}] ";
+                    logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
+                    String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or "
+                        + ALLOWED_URLS_SETTING.getKey();
+                    throw new RepositoryException(getMetadata().name(), exceptionMessage);
                 }
                 return normalizedUrl;
             }

modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.url;
+
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.NoSuchFileException;
+
+@SuppressForbidden(reason = "use http server")
+public class URLBlobStoreTests extends ESTestCase {
+
+    private static HttpServer httpServer;
+    private static String blobName;
+    private static byte[] message = new byte[512];
+    private URLBlobStore urlBlobStore;
+
+    @BeforeClass
+    public static void startHttp() throws Exception {
+        for (int i = 0; i < message.length; ++i) {
+            message[i] = randomByte();
+        }
+        blobName = randomAsciiOfLength(8);
+
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0);
+
+        httpServer.createContext("/indices/" + blobName, (s) -> {
+            s.sendResponseHeaders(200, message.length);
+            OutputStream responseBody = s.getResponseBody();
+            responseBody.write(message);
+            responseBody.close();
+        });
+
+        httpServer.start();
+    }
+
+    @AfterClass
+    public static void stopHttp() throws IOException {
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
+    @Before
+    public void storeSetup() throws MalformedURLException {
+        Settings settings = Settings.EMPTY;
+        String spec = "http://localhost:6001/";
+        urlBlobStore = new URLBlobStore(settings, new URL(spec));
+    }
+
+    public void testURLBlobStoreCanReadBlob() throws IOException {
+        BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
+        try (InputStream stream = container.readBlob(blobName)) {
+            byte[] bytes = new byte[message.length];
+            int read = stream.read(bytes);
+            assertEquals(message.length, read);
+            assertArrayEquals(message, bytes);
+        }
+    }
+
+    public void testNoBlobFound() throws IOException {
+        BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
+        String incorrectBlobName = "incorrect_" + blobName;
+        try (InputStream ignored = container.readBlob(incorrectBlobName)) {
+            fail("Should have thrown NoSuchFileException exception");
+            ignored.read();
+        } catch (NoSuchFileException e) {
+            assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage());
+        }
+    }
+}

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.url;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+import java.io.IOException;
+
+public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
+

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.url;
+
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+
+public class URLRepositoryTests extends ESTestCase {
+
+    public void testWhiteListingRepoURL() throws IOException {
+        String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath)
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+    }
+
+    public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
+        String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        try {
+            new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+            fail("RepositoryException should have been thrown.");
+        } catch (RepositoryException e) {
+            String msg = "[url] file url [" + repoPath
+                + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls";
+            assertEquals(msg, e.getMessage());
+        }
+    }
+
+    public void testMustBeSupportedProtocol() throws IOException {
+        Path directory = createTempDir();
+        String repoPath = directory.resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(Environment.PATH_REPO_SETTING.getKey(), directory.toString())
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        try {
+            new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+            fail("RepositoryException should have been thrown.");
+        } catch (RepositoryException e) {
+            assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
+        }
+    }
+
+}

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.url;
+
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class URLSnapshotRestoreTests extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(URLRepositoryPlugin.class);
+    }
+
+    public void testUrlRepository() throws Exception {
+        Client client = client();
+
+        logger.info("-->  creating repository");
+        Path repositoryLocation = randomRepoPath();
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+            .setType(FsRepository.TYPE).setSettings(Settings.builder()
+                .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
+                .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+                .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+
+        createIndex("test-idx");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client
+            .admin()
+            .cluster()
+            .prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        int actualTotalShards = createSnapshotResponse.getSnapshotInfo().totalShards();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(actualTotalShards));
+
+        SnapshotState state = client
+            .admin()
+            .cluster()
+            .prepareGetSnapshots("test-repo")
+            .setSnapshots("test-snap")
+            .get()
+            .getSnapshots()
+            .get(0)
+            .state();
+        assertThat(state, equalTo(SnapshotState.SUCCESS));
+
+        logger.info("--> delete index");
+        cluster().wipeIndices("test-idx");
+
+        logger.info("--> create read-only URL repository");
+        assertAcked(client.admin().cluster().preparePutRepository("url-repo")
+            .setType(URLRepository.TYPE).setSettings(Settings.builder()
+                .put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL())
+                .put("list_directories", randomBoolean())));
+        logger.info("--> restore index after deletion");
+        RestoreSnapshotResponse restoreSnapshotResponse = client
+            .admin()
+            .cluster()
+            .prepareRestoreSnapshot("url-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .execute()
+            .actionGet();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
+
+        logger.info("--> list available shapshots");
+        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
+        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
+
+        logger.info("--> delete snapshot");
+        DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
+        assertAcked(deleteSnapshotResponse);
+
+        logger.info("--> list available shapshot again, no snapshots should be returned");
+        getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
+        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
+    }
+}

modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml

@@ -0,0 +1,32 @@
+# Integration tests for URL Repository component
+#
+"URL Repository plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.modules.0.name: repository-url }
+
+---
+setup:
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo1
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test1"
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo2
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test2"

modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml

@@ -0,0 +1,16 @@
+"Repository can be registered":
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo1
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test1"
+  - is_true: acknowledged
+
+  - do:
+      snapshot.get_repository:
+        repository: test_repo1
+
+  - is_true : test_repo1

qa/rolling-upgrade/build.gradle

@@ -29,6 +29,7 @@ task oldClusterTest(type: RestIntegTestTask) {
     numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
 }
@@ -40,6 +41,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
   finalizedBy 'oldClusterTest#node0.stop'
@@ -52,6 +54,7 @@ task upgradedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
   // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion

rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml

@@ -5,17 +5,17 @@ setup:
       snapshot.create_repository:
         repository: test_repo_get_1
         body:
-          type: url
+          type: fs
           settings:
-            url: "http://snapshot.test1"
+            location: "test_repo_get_1_loc"
 
   - do:
       snapshot.create_repository:
         repository: test_repo_get_2
         body:
-          type: url
+          type: fs
           settings:
-            url: "http://snapshot.test2"
+            location: "test_repo_get_1_loc"
 
 ---
 "Get all repositories":

settings.gradle

@@ -31,6 +31,7 @@ List projects = [
   'modules:transport-netty4',
   'modules:reindex',
   'modules:percolator',
+  'modules:repository-url',
   'plugins:analysis-icu',
   'plugins:analysis-kuromoji',
   'plugins:analysis-phonetic',