Merge pull request #20396 from johtani/fix/fail_loading_non_prebuilt_tokenfilter_in_analyze_api

Can load non-PreBuiltTokenFilter in Analyze API

core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java

@@ -174,7 +174,7 @@ public final class AnalysisRegistry implements Closeable {
             Settings currentSettings = tokenizerSettings.get(tokenizer);
             return getAnalysisProvider("tokenizer", tokenizers, tokenizer, currentSettings.get("type"));
         } else {
-            return prebuiltAnalysis.tokenizerFactories.get(tokenizer);
+            return getTokenizerProvider(tokenizer);
         }
     }
 
@@ -202,7 +202,7 @@ public final class AnalysisRegistry implements Closeable {
                 return getAnalysisProvider("tokenfilter", tokenFilters, tokenFilter, typeName);
             }
         } else {
-            return prebuiltAnalysis.tokenFilterFactories.get(tokenFilter);
+            return getTokenFilterProvider(tokenFilter);
         }
     }
 
@@ -220,7 +220,7 @@ public final class AnalysisRegistry implements Closeable {
             Settings currentSettings = tokenFilterSettings.get(charFilter);
             return getAnalysisProvider("charfilter", charFilters, charFilter, currentSettings.get("type"));
         } else {
-            return prebuiltAnalysis.charFilterFactories.get(charFilter);
+            return getCharFilterProvider(charFilter);
         }
     }
 

core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.admin.indices;
 
+import org.apache.lucene.analysis.minhash.MinHashFilter;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
@@ -253,4 +254,18 @@ public class TransportAnalyzeActionTests extends ESTestCase {
             assertEquals(e.getMessage(), "failed to find global char filter under [foobar]");
         }
     }
+
+    public void testNonPreBuildTokenFilter() throws IOException {
+        AnalyzeRequest request = new AnalyzeRequest();
+        request.tokenizer("whitespace");
+        request.addTokenFilter("min_hash");
+        request.text("the quick brown fox");
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment);
+        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        int default_hash_count = 1;
+        int default_bucket_size = 512;
+        int default_hash_set_size = 1;
+        assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size());
+
+    }
 }