Remove discovery-file plugin (#33257)

In #33241 we moved the file-based discovery functionality to core
Elasticsearch, but preserved the `discovery-file` plugin, and support for the
existing location of the `unicast_hosts.txt` file, for BWC reasons. This commit
completes the removal of this plugin.

docs/plugins/discovery-file.asciidoc

@@ -1,14 +0,0 @@
-[[discovery-file]]
-=== File-Based Discovery Plugin
-
-The functionality provided by the `discovery-file` plugin is now available in
-Elasticsearch without requiring a plugin. This plugin still exists to ensure
-backwards compatibility, but it will be removed in a future version.
-
-On installation, this plugin creates a file at
-`$ES_PATH_CONF/discovery-file/unicast_hosts.txt` that comprises comments that
-describe how to use it. It is preferable not to install this plugin and instead
-to create this file, and its containing directory, using standard tools.
-
-:plugin_name: discovery-file
-include::install_remove.asciidoc[]

docs/plugins/discovery.asciidoc

@@ -21,10 +21,6 @@ The Azure Classic discovery plugin uses the Azure Classic API for unicast discov
 
 The Google Compute Engine discovery plugin uses the GCE API for unicast discovery.
 
-<<discovery-file,File-based discovery>>::
-
-The File-based discovery plugin allows providing the unicast hosts list through a dynamically updatable file.
-
 [float]
 ==== Community contributed discovery plugins
 
@@ -38,5 +34,3 @@ include::discovery-ec2.asciidoc[]
 include::discovery-azure-classic.asciidoc[]
 
 include::discovery-gce.asciidoc[]
-
-include::discovery-file.asciidoc[]

docs/reference/cat/plugins.asciidoc

@@ -23,7 +23,6 @@ U7321H6 analysis-stempel        {version} The Stempel (Polish) Analysis plugin i
 U7321H6 analysis-ukrainian      {version} The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.
 U7321H6 discovery-azure-classic {version} The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism
 U7321H6 discovery-ec2           {version} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.
-U7321H6 discovery-file          {version} Discovery file plugin enables unicast discovery from hosts stored in a file.
 U7321H6 discovery-gce           {version} The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.
 U7321H6 ingest-attachment       {version} Ingest processor that uses Apache Tika to extract contents
 U7321H6 ingest-geoip            {version} Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database

docs/reference/migration/migrate_7_0/plugins.asciidoc

@@ -22,4 +22,12 @@ See {plugins}/repository-gcs-client.html#repository-gcs-client[Google Cloud Stor
 ==== Analysis Plugin changes
 
 * The misspelled helper method `requriesAnalysisSettings(AnalyzerProvider<T> provider)` has been
-renamed to `requiresAnalysisSettings`
+renamed to `requiresAnalysisSettings`
+
+==== File-based discovery plugin
+
+* This plugin has been removed since its functionality is now part of
+Elasticsearch and requires no plugin. The location of the hosts file has moved
+from `$ES_PATH_CONF/file-discovery/unicast_hosts.txt` to
+`$ES_PATH_CONF/unicast_hosts.txt`. See <<file-based-hosts-provider, the
+file-based hosts provider documentation>> for further information.

plugins/discovery-file/build.gradle

@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.elasticsearch.gradle.test.ClusterConfiguration
-import org.elasticsearch.gradle.test.ClusterFormationTasks
-import org.elasticsearch.gradle.test.NodeInfo
-
-esplugin {
-    description 'Discovery file plugin enables unicast discovery from hosts stored in a file.'
-    classname 'org.elasticsearch.discovery.file.FileBasedDiscoveryPlugin'
-}
-
-bundlePlugin {
-    from('config/discovery-file') {
-        into 'config'
-    }
-}
-
-task setupSeedNodeAndUnicastHostsFile(type: DefaultTask) {
-    mustRunAfter(precommit)
-}
-// setup the initial cluster with one node that will serve as the seed node
-// for unicast discovery
-ClusterConfiguration config = new ClusterConfiguration(project)
-config.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
-config.clusterName = 'discovery-file-test-cluster'
-List<NodeInfo> nodes = ClusterFormationTasks.setup(project, 'initialCluster', setupSeedNodeAndUnicastHostsFile, config)
-File srcUnicastHostsFile = file('build/cluster/unicast_hosts.txt')
-
-// write the unicast_hosts.txt file to a temporary location to be used by the second cluster
-setupSeedNodeAndUnicastHostsFile.doLast {
-    // write the unicast_hosts.txt file to a temp file in the build directory
-    srcUnicastHostsFile.setText(nodes.get(0).transportUri(), 'UTF-8')
-}
-
-// second cluster, which will connect to the first via the unicast_hosts.txt file
-integTestCluster {
-    dependsOn setupSeedNodeAndUnicastHostsFile
-    clusterName = 'discovery-file-test-cluster'
-    setting 'discovery.zen.hosts_provider', 'file'
-    extraConfigFile 'discovery-file/unicast_hosts.txt', srcUnicastHostsFile
-}
-
-integTestRunner.finalizedBy ':plugins:discovery-file:initialCluster#stop'
-

plugins/discovery-file/config/discovery-file/unicast_hosts.txt

@@ -1,20 +0,0 @@
-# The unicast_hosts.txt file contains the list of unicast hosts to connect to
-# for pinging during the discovery process, when using the file-based discovery
-# mechanism.  This file should contain one entry per line, where an entry is a
-# host/port combination.  The host and port should be separated by a `:`.  If
-# the port is left off, a default port of 9300 is assumed.  For example, if the
-# cluster has three nodes that participate in the discovery process:
-# (1) 66.77.88.99 running on port 9300 (2) 66.77.88.100 running on port 9305
-# and (3) 66.77.88.99 running on port 10005, then this file should contain the
-# following text:
-#
-#10.10.10.5
-#10.10.10.6:9305
-#10.10.10.5:10005
-#
-# For IPv6 addresses, make sure to put a bracket around the host part of the address,
-# for example: [2001:cdba:0000:0000:0000:0000:3257:9652]:9301 (where 9301 is the port).
-#
-# NOTE: all lines starting with a `#` are comments, and comments must exist
-# on lines of their own (i.e. comments cannot begin in the middle of a line)
-#

plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java

@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.file;
-
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.zen.UnicastHostsProvider;
-import org.elasticsearch.plugins.DiscoveryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.function.Supplier;
-
-public class FileBasedDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
-
-    private final DeprecationLogger deprecationLogger;
-    static final String DEPRECATION_MESSAGE
-        = "File-based discovery is now built into Elasticsearch and does not require the discovery-file plugin";
-
-    public FileBasedDiscoveryPlugin(Settings settings) {
-        deprecationLogger = new DeprecationLogger(Loggers.getLogger(this.getClass(), settings));
-    }
-
-    @Override
-    public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(TransportService transportService,
-                                                                            NetworkService networkService) {
-        deprecationLogger.deprecated(DEPRECATION_MESSAGE);
-        return Collections.emptyMap();
-    }
-}

plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java

@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.file;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-
-/**
- * Integration tests to make sure the file-based discovery plugin works in a cluster.
- */
-public class FileBasedDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-
-    public FileBasedDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
-}

plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginDeprecationTests.java

@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.file;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.elasticsearch.discovery.file.FileBasedDiscoveryPlugin.DEPRECATION_MESSAGE;
-
-public class FileBasedDiscoveryPluginDeprecationTests extends ESTestCase {
-    public void testDeprecationWarning() {
-        new FileBasedDiscoveryPlugin(Settings.EMPTY).getZenHostsProviders(null, null);
-        assertWarnings(DEPRECATION_MESSAGE);
-    }
-}

plugins/discovery-file/src/test/resources/rest-api-spec/test/discovery_file/10_basic.yml

@@ -1,13 +0,0 @@
-# Integration tests for file-based discovery
-#
-"Ensure cluster formed successfully with discovery file":
-    # make sure both nodes joined the cluster
-    - do:
-        cluster.health:
-            wait_for_nodes: 2
-
-    # make sure the cluster was formed with the correct name
-    - do:
-        cluster.state: {}
-
-    - match: { cluster_name: 'discovery-file-test-cluster' } # correct cluster name, we formed the cluster we expected to

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -224,10 +224,6 @@ fi
     install_and_check_plugin discovery ec2 aws-java-sdk-core-*.jar
 }
 
-@test "[$GROUP] install discovery-file plugin" {
-    install_and_check_plugin discovery file
-}
-
 @test "[$GROUP] install ingest-attachment plugin" {
     # we specify the version on the poi-3.17.jar so that the test does
     # not spuriously pass if the jar is missing but the other poi jars
@@ -364,10 +360,6 @@ fi
     remove_plugin discovery-ec2
 }
 
-@test "[$GROUP] remove discovery-file plugin" {
-    remove_plugin discovery-file
-}
-
 @test "[$GROUP] remove ingest-attachment plugin" {
     remove_plugin ingest-attachment
 }

server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java

@@ -49,23 +49,21 @@ public class FileBasedUnicastHostsProvider extends AbstractComponent implements
     public static final String UNICAST_HOSTS_FILE = "unicast_hosts.txt";
 
     private final Path unicastHostsFilePath;
-    private final Path legacyUnicastHostsFilePath;
 
     public FileBasedUnicastHostsProvider(Settings settings, Path configFile) {
         super(settings);
         this.unicastHostsFilePath = configFile.resolve(UNICAST_HOSTS_FILE);
-        this.legacyUnicastHostsFilePath = configFile.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE);
     }
 
     private List<String> getHostsList() {
         if (Files.exists(unicastHostsFilePath)) {
-            return readFileContents(unicastHostsFilePath);
-        }
-
-        if (Files.exists(legacyUnicastHostsFilePath)) {
-            deprecationLogger.deprecated("Found dynamic hosts list at [{}] but this path is deprecated. This list should be at [{}] " +
-                "instead. Support for the deprecated path will be removed in future.", legacyUnicastHostsFilePath, unicastHostsFilePath);
-            return readFileContents(legacyUnicastHostsFilePath);
+            try (Stream<String> lines = Files.lines(unicastHostsFilePath)) {
+                return lines.filter(line -> line.startsWith("#") == false) // lines starting with `#` are comments
+                    .collect(Collectors.toList());
+            } catch (IOException e) {
+                logger.warn(() -> new ParameterizedMessage("failed to read file [{}]", unicastHostsFilePath), e);
+                return Collections.emptyList();
+            }
         }
 
         logger.warn("expected, but did not find, a dynamic hosts list at [{}]", unicastHostsFilePath);
@@ -73,16 +71,6 @@ public class FileBasedUnicastHostsProvider extends AbstractComponent implements
         return Collections.emptyList();
     }
 
-    private List<String> readFileContents(Path path) {
-        try (Stream<String> lines = Files.lines(path)) {
-            return lines.filter(line -> line.startsWith("#") == false) // lines starting with `#` are comments
-                .collect(Collectors.toList());
-        } catch (IOException e) {
-            logger.warn(() -> new ParameterizedMessage("failed to read file [{}]", unicastHostsFilePath), e);
-            return Collections.emptyList();
-        }
-    }
-
     @Override
     public List<TransportAddress> buildDynamicHosts(HostsResolver hostsResolver) {
         final List<TransportAddress> transportAddresses = hostsResolver.resolveHosts(getHostsList(), 1);

server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java

@@ -52,11 +52,9 @@ import static org.elasticsearch.discovery.zen.FileBasedUnicastHostsProvider.UNIC
 
 public class FileBasedUnicastHostsProviderTests extends ESTestCase {
 
-    private boolean legacyLocation;
     private ThreadPool threadPool;
     private ExecutorService executorService;
     private MockTransportService transportService;
-    private Path configPath;
 
     @Before
     public void setUp() throws Exception {
@@ -108,24 +106,12 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
         assertEquals(9300, nodes.get(2).getPort());
     }
 
-    public void testBuildDynamicNodesLegacyLocation() throws Exception {
-        legacyLocation = true;
-        testBuildDynamicNodes();
-        assertDeprecatedLocationWarning();
-    }
-
     public void testEmptyUnicastHostsFile() throws Exception {
         final List<String> hostEntries = Collections.emptyList();
         final List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
         assertEquals(0, addresses.size());
     }
 
-    public void testEmptyUnicastHostsFileLegacyLocation() throws Exception {
-        legacyLocation = true;
-        testEmptyUnicastHostsFile();
-        assertDeprecatedLocationWarning();
-    }
-
     public void testUnicastHostsDoesNotExist() {
         final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
         final FileBasedUnicastHostsProvider provider = new FileBasedUnicastHostsProvider(settings, createTempDir().toAbsolutePath());
@@ -141,12 +127,6 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
         assertEquals(0, addresses.size());
     }
 
-    public void testInvalidHostEntriesLegacyLocation() throws Exception {
-        legacyLocation = true;
-        testInvalidHostEntries();
-        assertDeprecatedLocationWarning();
-    }
-
     public void testSomeInvalidHostEntries() throws Exception {
         final List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300", "192.168.0.1:9301");
         final List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
@@ -155,12 +135,6 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
         assertEquals(9301, addresses.get(0).getPort());
     }
 
-    public void testSomeInvalidHostEntriesLegacyLocation() throws Exception {
-        legacyLocation = true;
-        testSomeInvalidHostEntries();
-        assertDeprecatedLocationWarning();
-    }
-
     // sets up the config dir, writes to the unicast hosts file in the config dir,
     // and then runs the file-based unicast host provider to get the list of discovery nodes
     private List<TransportAddress> setupAndRunHostProvider(final List<String> hostEntries) throws IOException {
@@ -168,15 +142,9 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
         final Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
             .build();
-        if (randomBoolean()) {
-            configPath = homeDir.resolve("config");
-        } else {
-            configPath = createTempDir();
-        }
-        final Path discoveryFilePath = legacyLocation ? configPath.resolve("discovery-file") : configPath;
-        Files.createDirectories(discoveryFilePath);
-        final Path unicastHostsPath = discoveryFilePath.resolve(UNICAST_HOSTS_FILE);
-        try (BufferedWriter writer = Files.newBufferedWriter(unicastHostsPath)) {
+        final Path configPath = randomBoolean() ? homeDir.resolve("config") : createTempDir();
+        Files.createDirectories(configPath);
+        try (BufferedWriter writer = Files.newBufferedWriter(configPath.resolve(UNICAST_HOSTS_FILE))) {
             writer.write(String.join("\n", hostEntries));
         }
 
@@ -184,12 +152,4 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
             UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
                 TimeValue.timeValueSeconds(10)));
     }
-
-    private void assertDeprecatedLocationWarning() {
-        assertWarnings("Found dynamic hosts list at [" +
-            configPath.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE) +
-            "] but this path is deprecated. This list should be at [" +
-            configPath.resolve(UNICAST_HOSTS_FILE) +
-            "] instead. Support for the deprecated path will be removed in future.");
-    }
 }