Example stable plugin (#90805)

Add example stable plugin. This example plugin creates and tests
some trivial analysis components. Once we implement settings for
stable plugins, we will be able to make the example plugin a little bit
more interesting.

Co-authored-by: William Brafford <william.brafford@elastic.co>

docs/changelog/90805.yaml

@@ -0,0 +1,5 @@
+pr: 90805
+summary: Example stable plugin
+area: Infra/Plugins
+type: enhancement
+issues: []

plugins/examples/settings.gradle

@@ -23,6 +23,7 @@ gradle.rootProject {
       props.load(is)
       elasticsearchVersion = "${props.get('elasticsearch')}-SNAPSHOT"
       log4jVersion = props.get('log4j')
+      luceneVersion = props.get('lucene')
     }
   }
 }

plugins/examples/stable-analysis/build.gradle

@@ -0,0 +1,28 @@
+apply plugin: 'elasticsearch.stable-esplugin'
+apply plugin: 'elasticsearch.yaml-rest-test'
+
+esplugin {
+  name 'stable-analysis-plugin'
+  description 'An example analysis plugin using stable plugin api'
+}
+//TODO write module-info
+
+dependencies {
+
+  //TODO transitive dependency off and plugin-api dependency?
+  compileOnly "org.elasticsearch:elasticsearch-plugin-api:${elasticsearchVersion}"
+  compileOnly "org.elasticsearch:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
+  compileOnly "org.apache.lucene:lucene-analysis-common:${luceneVersion}"
+
+  //TODO for testing this also have to be declared
+  testImplementation "org.elasticsearch:elasticsearch-plugin-api:${elasticsearchVersion}"
+  testImplementation "org.elasticsearch:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
+  testImplementation "org.apache.lucene:lucene-analysis-common:${luceneVersion}"
+
+  testImplementation ('junit:junit:4.13.2'){
+    exclude group: 'org.hamcrest'
+  }
+  testImplementation 'org.hamcrest:hamcrest:2.2'
+
+}
+

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.example.analysis.lucene.ReplaceHash;
+import org.elasticsearch.example.analysis.lucene.Skip1TokenFilter;
+import org.elasticsearch.example.analysis.lucene.UnderscoreTokenizer;
+import org.elasticsearch.plugin.api.NamedComponent;
+
+@NamedComponent(name = "example_analyzer_factory")
+public class ExampleAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory {
+
+    @Override
+    //TODO guide lucene
+    public Analyzer create() {
+        return new CustomAnalyzer();
+    }
+
+    static class CustomAnalyzer extends Analyzer {
+
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName) {
+            var tokenizer = new UnderscoreTokenizer();
+            var tokenFilter = new Skip1TokenFilter(tokenizer);
+            return new TokenStreamComponents(r -> tokenizer.setReader(new ReplaceHash(r)), tokenFilter);
+        }
+    }
+}
+

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.util.SuppressForbidden;
+import org.elasticsearch.example.analysis.lucene.ReplaceHash;
+import org.elasticsearch.plugin.analysis.api.CharFilterFactory;
+import org.elasticsearch.plugin.api.NamedComponent;
+
+import java.io.Reader;
+
+@NamedComponent(name = "example_char_filter")
+public class ExampleCharFilterFactory implements CharFilterFactory {
+    @Override
+    public Reader create(Reader reader) {
+        return new ReplaceHash(reader);
+    }
+}
+
+

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.elasticsearch.example.analysis.lucene.AppendTokenFilter;
+import org.elasticsearch.example.analysis.lucene.Skip1TokenFilter;
+import org.elasticsearch.plugin.analysis.api.AnalysisMode;
+import org.elasticsearch.plugin.api.NamedComponent;
+
+@NamedComponent(name = "example_token_filter_factory")
+public class ExampleTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory {
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new Skip1TokenFilter(tokenStream);
+    }
+
+    @Override
+    public TokenStream normalize(TokenStream tokenStream) {
+        return new AppendTokenFilter(tokenStream, "1");
+    }
+
+    @Override
+    public AnalysisMode getAnalysisMode() {
+        return org.elasticsearch.plugin.analysis.api.TokenFilterFactory.super.getAnalysisMode();
+    }
+
+}
+

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.example.analysis.lucene.UnderscoreTokenizer;
+import org.elasticsearch.plugin.analysis.api.TokenizerFactory;
+import org.elasticsearch.plugin.api.NamedComponent;
+
+@NamedComponent(name = "example_tokenizer_factory")
+public class ExampleTokenizerFactory implements TokenizerFactory {
+    @Override
+    public Tokenizer create() {
+        return new UnderscoreTokenizer();
+    }
+
+}
+

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/AppendTokenFilter.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.elasticsearch.plugin.analysis.api.TokenFilterFactory;
+
+import java.io.IOException;
+
+public class AppendTokenFilter extends TokenFilter {
+    public static TokenFilterFactory factoryForSuffix(String suffix) {
+        return new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return suffix;
+            }
+
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                return new AppendTokenFilter(tokenStream, suffix);
+            }
+        };
+    }
+
+    private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+    private final char[] appendMe;
+
+    public AppendTokenFilter(TokenStream input, String appendMe) {
+        super(input);
+        this.appendMe = appendMe.toCharArray();
+    }
+
+    @Override
+    public boolean incrementToken() throws IOException {
+        if (false == input.incrementToken()) {
+            return false;
+        }
+        term.resizeBuffer(term.length() + appendMe.length);
+        System.arraycopy(appendMe, 0, term.buffer(), term.length(), appendMe.length);
+        term.setLength(term.length() + appendMe.length);
+        return true;
+    }
+}

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceHash.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.charfilter.MappingCharFilter;
+import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+
+import java.io.Reader;
+
+public class ReplaceHash extends MappingCharFilter {
+
+    public ReplaceHash(Reader in) {
+        super(charMap(), in);
+    }
+
+    private static NormalizeCharMap charMap() {
+        NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
+        builder.add("#", "3");
+        return builder.build();
+    }
+}

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/Skip1TokenFilter.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+import java.io.IOException;
+
+public class Skip1TokenFilter extends FilteringTokenFilter {
+
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+    public Skip1TokenFilter(TokenStream in) {
+        super(in);
+    }
+
+    @Override
+    protected boolean accept() throws IOException {
+        return termAtt.buffer()[0] != '1';
+    }
+}

plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/UnderscoreTokenizer.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.util.CharTokenizer;
+
+public class UnderscoreTokenizer extends CharTokenizer {
+
+    @Override
+    protected boolean isTokenChar(int c) {
+        return c != '_';
+    }
+}

plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ExampleCharFilterFactoryTests.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+
+import org.elasticsearch.plugin.analysis.api.CharFilterFactory;
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ExampleCharFilterFactoryTests {
+    @Test
+    public void exampleCharFilterIsAnnotatedWithName() {
+        CharFilterFactory charFilterFactory = new ExampleCharFilterFactory();
+        assertThat(charFilterFactory.name(), equalTo("example_char_filter"));
+    }
+}

plugins/examples/stable-analysis/src/yamlRestTest/java/org/elasticsearch/example/stable/client/ExampleStableAnalysisClientYamlTestSuiteIT.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.stable.client;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+//TODO javadoc, a default impl perhaps?
+public class ExampleStableAnalysisClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    public ExampleStableAnalysisClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        // The test executes all the test candidates by default
+        // see ESClientYamlSuiteTestCase.REST_TESTS_SUITE
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}

plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/10_loading.yml

@@ -0,0 +1,15 @@
+---
+"Plugin loaded":
+  - skip:
+      reason: "contains is a newly added assertion"
+      features: contains
+  - do:
+      cluster.state: {}
+
+  # Get master node id
+  - set: { master_node: master }
+
+  - do:
+      nodes.info: {}
+
+  - contains:  { nodes.$master.plugins: { name: stable-analysis-plugin } }

plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/20_char_filter.yml

@@ -0,0 +1,31 @@
+## Smoke tests for char filters included in the analysis-common module
+
+#todo this can be expanded with parameters once settings support is added
+---
+"Stable char_filter plugin. Replace # with 3":
+  - do:
+      indices.analyze:
+        body:
+          text: t#st
+          tokenizer: standard
+          char_filter:
+            - type: example_char_filter
+  - length: { tokens: 1 }
+  - match:  { tokens.0.token: "t3st" }
+
+  - do:
+      indices.analyze:
+        body:
+          text: t#st
+          explain: true
+          tokenizer: standard
+          char_filter:
+            - type: example_char_filter
+  - match:  { detail.custom_analyzer: true }
+  - length: { detail.charfilters.0.filtered_text: 1 }
+  - match:  { detail.charfilters.0.filtered_text.0: "t3st" }
+  - length: { detail.tokenizer.tokens: 1 }
+  - match:  { detail.tokenizer.tokens.0.token: "t3st" }
+  - match:  { detail.tokenizer.tokens.0.start_offset: 0 }
+  - match:  { detail.tokenizer.tokens.0.end_offset: 4 }
+  - match:  { detail.tokenizer.tokens.0.position: 0 }

plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/30_token_filter.yml

@@ -0,0 +1,28 @@
+## Smoke tests for char filters included in the analysis-common module
+
+#todo this can be expanded with parameters once settings support is added
+---
+"Stable token_filter plugin. Skip tokens starting with 1":
+  - do:
+      indices.analyze:
+        body:
+          text: 1test 2test 1test 3test
+          tokenizer: standard
+          filter:
+            - type: example_token_filter_factory
+  - length: { tokens: 2 }
+  - match:  { tokens.0.token: "2test" }
+  - match:  { tokens.1.token: "3test" }
+
+  - do:
+      indices.analyze:
+        body:
+          text: 1test 2test 1test 3test
+          explain: true
+          tokenizer: standard
+          filter:
+            - type: example_token_filter_factory
+  - match:  { detail.custom_analyzer: true }
+  - length: { detail.tokenfilters.0.tokens: 2 }
+  - match:  { detail.tokenfilters.0.tokens.0.token: "2test" }
+  - match:  { detail.tokenfilters.0.tokens.1.token: "3test" }

plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/40_tokenizer.yml

@@ -0,0 +1,26 @@
+## Smoke tests for char filters included in the analysis-common module
+
+#todo this can be expanded with parameters once settings support is added
+---
+"Stable tokenizer plugin. Tokenizes text by _":
+  - do:
+      indices.analyze:
+        body:
+          text: x_y_z
+          tokenizer: example_tokenizer_factory
+  - length: { tokens: 3 }
+  - match:  { tokens.0.token: "x" }
+  - match:  { tokens.1.token: "y" }
+  - match:  { tokens.2.token: "z" }
+
+  - do:
+      indices.analyze:
+        body:
+          text: x_y_z
+          explain: true
+          tokenizer: example_tokenizer_factory
+  - match:  { detail.custom_analyzer: true }
+  - length: { detail.tokenizer.tokens: 3 }
+  - match:  { detail.tokenizer.tokens.0.token: "x" }
+  - match:  { detail.tokenizer.tokens.1.token: "y" }
+  - match:  { detail.tokenizer.tokens.2.token: "z" }

plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/50_analyzer.yml

@@ -0,0 +1,24 @@
+## Smoke tests for char filters included in the analysis-common module
+
+#todo this can be expanded with parameters once settings support is added
+---
+"Stable analyzer provider plugin. It combines a underscore tokenizer, token filter which skips 1 and char filter which replaces # with 3":
+  - do:
+      indices.analyze:
+        body:
+          text: 1x_y_#z
+          analyzer: example_analyzer_factory
+  - length: { tokens: 2 }
+  - match:  { tokens.0.token: "y" }
+  - match:  { tokens.1.token: "3z" }
+
+  - do:
+      indices.analyze:
+        body:
+          text: 1x_y_#z
+          explain: true
+          analyzer: example_analyzer_factory
+  - match:  { detail.custom_analyzer: false } #?? why not custom?
+  - length: { detail.analyzer.tokens: 2 }
+  - match:  { detail.analyzer.tokens.0.token: "y" }
+  - match:  { detail.analyzer.tokens.1.token: "3z" }