Optionally allow text similarity reranking to fail (#121784)

Add an option to allow reranking to fail, and the docs to pass through as-is.
Exposing the error to users and adding documentation is a later piece of work.

docs/changelog/121784.yaml

@@ -0,0 +1,5 @@
+pr: 121784
+summary: Optionally allow text similarity reranking to fail
+area: Search
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java

@@ -205,7 +205,7 @@ public class FieldBasedRerankerIT extends AbstractRerankerIT {
 
         @Override
         public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) {
-            return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize()) {
+            return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize(), false) {
                 @Override
                 protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                     float[] scores = new float[featureDocs.length];
@@ -346,7 +346,7 @@ public class FieldBasedRerankerIT extends AbstractRerankerIT {
         @Override
         public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) {
             if (this.throwingRankBuilderType == ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT)
-                return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize()) {
+                return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize(), false) {
                     @Override
                     protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                         throw new UnsupportedOperationException("rfc - simulated failure");

server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java

@@ -249,7 +249,7 @@ public class MockedRequestActionBasedRerankerIT extends AbstractRerankerIT {
             String inferenceText,
             float minScore
         ) {
-            super(size, from, windowSize);
+            super(size, from, windowSize, false);
             this.client = client;
             this.inferenceId = inferenceId;
             this.inferenceText = inferenceText;

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -197,6 +197,7 @@ public class TransportVersions {
     public static final TransportVersion SLM_UNHEALTHY_IF_NO_SNAPSHOT_WITHIN = def(9_010_0_00);
     public static final TransportVersion ESQL_SUPPORT_PARTIAL_RESULTS = def(9_011_0_00);
     public static final TransportVersion REMOVE_REPOSITORY_CONFLICT_MESSAGE = def(9_012_0_00);
+    public static final TransportVersion RERANKER_FAILURES_ALLOWED = def(9_013_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java

@@ -20,12 +20,14 @@ import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
 import org.elasticsearch.transport.Transport;
 
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -186,7 +188,7 @@ public class RankFeaturePhase extends SearchPhase {
             new ActionListener<>() {
                 @Override
                 public void onResponse(RankFeatureDoc[] docsWithUpdatedScores) {
-                    RankFeatureDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(docsWithUpdatedScores);
+                    RankDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(docsWithUpdatedScores, true);
                     SearchPhaseController.ReducedQueryPhase reducedRankFeaturePhase = newReducedQueryPhaseResults(
                         reducedQueryPhase,
                         topResults
@@ -196,12 +198,36 @@ public class RankFeaturePhase extends SearchPhase {
 
                 @Override
                 public void onFailure(Exception e) {
-                    context.onPhaseFailure(NAME, "Computing updated ranks for results failed", e);
+                    if (rankFeaturePhaseRankCoordinatorContext.failuresAllowed()) {
+                        // TODO: handle the exception somewhere
+                        // don't want to log the entire stack trace, it's not helpful here
+                        logger.warn("Exception computing updated ranks, continuing with existing ranks: {}", e.toString());
+                        // use the existing score docs as-is
+                        // downstream things expect every doc to have a score, so we need to infer a score here
+                        // if the doc doesn't otherwise have one. We can use the rank to infer a possible score instead (1/rank).
+                        ScoreDoc[] inputDocs = reducedQueryPhase.sortedTopDocs().scoreDocs();
+                        RankFeatureDoc[] rankDocs = new RankFeatureDoc[inputDocs.length];
+                        for (int i = 0; i < inputDocs.length; i++) {
+                            ScoreDoc doc = inputDocs[i];
+                            rankDocs[i] = new RankFeatureDoc(doc.doc, Float.isNaN(doc.score) ? 1f / (i + 1) : doc.score, doc.shardIndex);
+                        }
+                        RankDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(rankDocs, false);
+                        SearchPhaseController.ReducedQueryPhase reducedRankFeaturePhase = newReducedQueryPhaseResults(
+                            reducedQueryPhase,
+                            topResults
+                        );
+                        moveToNextPhase(rankPhaseResults, reducedRankFeaturePhase);
+                    } else {
+                        context.onPhaseFailure(NAME, "Computing updated ranks for results failed", e);
+                    }
                 }
             }
         );
         rankFeaturePhaseRankCoordinatorContext.computeRankScoresForGlobalResults(
-            rankPhaseResults.getAtomicArray().asList().stream().map(SearchPhaseResult::rankFeatureResult).toList(),
+            rankPhaseResults.getSuccessfulResults()
+                .flatMap(r -> Arrays.stream(r.rankFeatureResult().shardResult().rankFeatureDocs))
+                .filter(rfd -> rfd.featureData != null)
+                .toArray(RankFeatureDoc[]::new),
             rankResultListener
         );
     }
@@ -210,7 +236,6 @@ public class RankFeaturePhase extends SearchPhase {
         SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
         ScoreDoc[] scoreDocs
     ) {
-
         return new SearchPhaseController.ReducedQueryPhase(
             reducedQueryPhase.totalHits(),
             reducedQueryPhase.fetchHits(),

server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java

@@ -133,9 +133,8 @@ public abstract class RankBuilder implements VersionedNamedWriteable, ToXContent
         if (obj == null || getClass() != obj.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked")
         RankBuilder other = (RankBuilder) obj;
-        return Objects.equals(rankWindowSize, other.rankWindowSize()) && doEquals(other);
+        return rankWindowSize == other.rankWindowSize && doEquals(other);
     }
 
     protected abstract boolean doEquals(RankBuilder other);

server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java

@@ -12,13 +12,9 @@ package org.elasticsearch.search.rank.context;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
-import org.elasticsearch.search.rank.feature.RankFeatureResult;
-import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.List;
 
 import static org.elasticsearch.search.SearchService.DEFAULT_FROM;
 import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
@@ -33,11 +29,17 @@ public abstract class RankFeaturePhaseRankCoordinatorContext {
     protected final int size;
     protected final int from;
     protected final int rankWindowSize;
+    protected final boolean failuresAllowed;
 
-    public RankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize) {
+    public RankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize, boolean failuresAllowed) {
         this.size = size < 0 ? DEFAULT_SIZE : size;
         this.from = from < 0 ? DEFAULT_FROM : from;
         this.rankWindowSize = rankWindowSize;
+        this.failuresAllowed = failuresAllowed;
+    }
+
+    public boolean failuresAllowed() {
+        return failuresAllowed;
     }
 
     /**
@@ -48,12 +50,13 @@ public abstract class RankFeaturePhaseRankCoordinatorContext {
 
     /**
      * Preprocesses the provided documents: sorts them by score descending.
-     * @param originalDocs documents to process
+     *
+     * @param originalDocs   documents to process
+     * @param rerankedScores {@code true} if the document scores have been reranked
      */
-    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
-        return Arrays.stream(originalDocs)
-            .sorted(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed())
-            .toArray(RankFeatureDoc[]::new);
+    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs, boolean rerankedScores) {
+        Arrays.sort(originalDocs, Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
+        return originalDocs;
     }
 
     /**
@@ -64,16 +67,10 @@ public abstract class RankFeaturePhaseRankCoordinatorContext {
      * Once all the scores have been computed, we sort the results, perform any pagination needed, and then call the `onFinish` consumer
      * with the final array of {@link ScoreDoc} results.
      *
-     * @param rankSearchResults a list of rank feature results from each shard
+     * @param featureDocs       an array of rank feature results from each shard
      * @param rankListener      a rankListener to handle the global ranking result
      */
-    public void computeRankScoresForGlobalResults(
-        List<RankFeatureResult> rankSearchResults,
-        ActionListener<RankFeatureDoc[]> rankListener
-    ) {
-        // extract feature data from each shard rank-feature phase result
-        RankFeatureDoc[] featureDocs = extractFeatureDocs(rankSearchResults);
-
+    public void computeRankScoresForGlobalResults(RankFeatureDoc[] featureDocs, ActionListener<RankFeatureDoc[]> rankListener) {
         // generate the final `topResults` results, and pass them to fetch phase through the `rankListener`
         computeScores(featureDocs, rankListener.delegateFailureAndWrap((listener, scores) -> {
             for (int i = 0; i < featureDocs.length; i++) {
@@ -86,10 +83,12 @@ public abstract class RankFeaturePhaseRankCoordinatorContext {
     /**
      * Ranks the provided {@link RankFeatureDoc} array and paginates the results based on the `from` and `size` parameters. Filters out
      * documents that have a relevance score less than min_score.
+     *
      * @param rankFeatureDocs documents to process
+     * @param rerankedScores {@code true} if the document scores have been reranked
      */
-    public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs) {
-        RankFeatureDoc[] sortedDocs = preprocess(rankFeatureDocs);
+    public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs, boolean rerankedScores) {
+        RankFeatureDoc[] sortedDocs = preprocess(rankFeatureDocs, rerankedScores);
         RankFeatureDoc[] topResults = new RankFeatureDoc[Math.max(0, Math.min(size, sortedDocs.length - from))];
         for (int rank = 0; rank < topResults.length; ++rank) {
             topResults[rank] = sortedDocs[from + rank];
@@ -97,17 +96,4 @@ public abstract class RankFeaturePhaseRankCoordinatorContext {
         }
         return topResults;
     }
-
-    private RankFeatureDoc[] extractFeatureDocs(List<RankFeatureResult> rankSearchResults) {
-        List<RankFeatureDoc> docFeatures = new ArrayList<>();
-        for (RankFeatureResult rankFeatureResult : rankSearchResults) {
-            RankFeatureShardResult shardResult = rankFeatureResult.shardResult();
-            for (RankFeatureDoc rankFeatureDoc : shardResult.rankFeatureDocs) {
-                if (rankFeatureDoc.featureData != null) {
-                    docFeatures.add(rankFeatureDoc);
-                }
-            }
-        }
-        return docFeatures.toArray(new RankFeatureDoc[0]);
-    }
 }

server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java

@@ -775,7 +775,7 @@ public class RankFeaturePhaseTests extends ESTestCase {
     }
 
     private RankFeaturePhaseRankCoordinatorContext defaultRankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize) {
-        return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize) {
+        return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize, false) {
 
             @Override
             protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
@@ -785,20 +785,12 @@ public class RankFeaturePhaseTests extends ESTestCase {
             }
 
             @Override
-            public void computeRankScoresForGlobalResults(
-                List<RankFeatureResult> rankSearchResults,
-                ActionListener<RankFeatureDoc[]> rankListener
-            ) {
-                List<RankFeatureDoc> features = new ArrayList<>();
-                for (RankFeatureResult rankFeatureResult : rankSearchResults) {
-                    RankFeatureShardResult shardResult = rankFeatureResult.shardResult();
-                    features.addAll(Arrays.stream(shardResult.rankFeatureDocs).toList());
-                }
-                rankListener.onResponse(features.toArray(new RankFeatureDoc[0]));
+            public void computeRankScoresForGlobalResults(RankFeatureDoc[] featureDocs, ActionListener<RankFeatureDoc[]> rankListener) {
+                rankListener.onResponse(featureDocs);
             }
 
             @Override
-            public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs) {
+            public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs, boolean rerankedScores) {
                 Arrays.sort(rankFeatureDocs, Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
                 RankFeatureDoc[] topResults = new RankFeatureDoc[Math.max(0, Math.min(size, rankFeatureDocs.length - from))];
                 // perform pagination

server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java

@@ -687,7 +687,7 @@ public class SearchServiceSingleNodeTests extends ESSingleNodeTestCase {
                                     int from,
                                     Client client
                                 ) {
-                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE, false) {
                                         @Override
                                         protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                                             float[] scores = new float[featureDocs.length];
@@ -831,7 +831,7 @@ public class SearchServiceSingleNodeTests extends ESSingleNodeTestCase {
                                 int from,
                                 Client client
                             ) {
-                                return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE, false) {
                                     @Override
                                     protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                                         throw new IllegalStateException("should have failed earlier");
@@ -947,7 +947,7 @@ public class SearchServiceSingleNodeTests extends ESSingleNodeTestCase {
                                     int from,
                                     Client client
                                 ) {
-                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE, false) {
                                         @Override
                                         protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                                             float[] scores = new float[featureDocs.length];
@@ -1075,7 +1075,7 @@ public class SearchServiceSingleNodeTests extends ESSingleNodeTestCase {
                                     int from,
                                     Client client
                                 ) {
-                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE, false) {
                                         @Override
                                         protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                                             float[] scores = new float[featureDocs.length];

server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java

@@ -171,7 +171,7 @@ public class RankFeatureShardPhaseTests extends ESTestCase {
             // no work to be done on the coordinator node for the rank feature phase
             @Override
             public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) {
-                return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE, false) {
                     @Override
                     protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
                         throw new AssertionError("not expected");

test/framework/src/main/java/org/elasticsearch/search/rank/rerank/AbstractRerankerIT.java

@@ -30,7 +30,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasRank;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 /**
  * this base class acts as a wrapper for testing different rerankers, and their behavior when exceptions are thrown
@@ -190,7 +193,7 @@ public abstract class AbstractRerankerIT extends ESIntegTestCase {
                 .setFrom(10),
             response -> {
                 assertHitCount(response, 5L);
-                assertEquals(0, response.getHits().getHits().length);
+                assertThat(response.getHits().getHits(), emptyArray());
             }
         );
         assertNoOpenContext(indexName);
@@ -226,7 +229,7 @@ public abstract class AbstractRerankerIT extends ESIntegTestCase {
                 .setSize(2),
             response -> {
                 assertHitCount(response, 4L);
-                assertEquals(2, response.getHits().getHits().length);
+                assertThat(response.getHits().getHits(), arrayWithSize(2));
                 int rank = 1;
                 for (SearchHit searchHit : response.getHits().getHits()) {
                     assertThat(searchHit, hasId(String.valueOf(5 - (rank - 1))));
@@ -397,13 +400,13 @@ public abstract class AbstractRerankerIT extends ESIntegTestCase {
                 .setAllowPartialSearchResults(true)
                 .setSize(10),
             response -> {
-                assertTrue(response.getFailedShards() > 0);
+                assertThat(response.getFailedShards(), greaterThan(0));
                 assertTrue(
                     Arrays.stream(response.getShardFailures())
                         .allMatch(failure -> failure.getCause().getMessage().contains("rfs - simulated failure"))
                 );
                 assertHitCount(response, 5);
-                assertTrue(response.getHits().getHits().length == 0);
+                assertThat(response.getHits().getHits(), emptyArray());
             }
         );
         assertNoOpenContext(indexName);
@@ -496,7 +499,7 @@ public abstract class AbstractRerankerIT extends ESIntegTestCase {
         assertNoOpenContext(indexName);
     }
 
-    private void assertNoOpenContext(final String indexName) throws Exception {
+    protected void assertNoOpenContext(final String indexName) throws Exception {
         assertBusy(
             () -> assertThat(indicesAdmin().prepareStats(indexName).get().getTotal().getSearch().getOpenContexts(), equalTo(0L)),
             1,

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankFeaturePhaseRankCoordinatorContext.java

@@ -11,8 +11,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.Random;
 
 /**
@@ -24,7 +22,7 @@ public class RandomRankFeaturePhaseRankCoordinatorContext extends RankFeaturePha
     private final Integer seed;
 
     public RandomRankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize, Integer seed) {
-        super(size, from, rankWindowSize);
+        super(size, from, rankWindowSize, false);
         this.seed = seed;
     }
 
@@ -40,16 +38,4 @@ public class RandomRankFeaturePhaseRankCoordinatorContext extends RankFeaturePha
         }
         scoreListener.onResponse(scores);
     }
-
-    /**
-     * Sorts documents by score descending.
-     * @param originalDocs documents to process
-     */
-    @Override
-    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
-        return Arrays.stream(originalDocs)
-            .sorted(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed())
-            .toArray(RankFeatureDoc[]::new);
-    }
-
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java

@@ -24,20 +24,12 @@ import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorCont
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.search.rank.rerank.RerankingRankFeaturePhaseRankShardContext;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder.FIELD_FIELD;
-import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder.INFERENCE_ID_FIELD;
-import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder.INFERENCE_TEXT_FIELD;
-import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder.MIN_SCORE_FIELD;
-
 /**
  * A {@code RankBuilder} that enables ranking with text similarity model inference. Supports parameters for configuring the inference call.
  */
@@ -51,35 +43,26 @@ public class TextSimilarityRankBuilder extends RankBuilder {
         License.OperationMode.ENTERPRISE
     );
 
-    static final ConstructingObjectParser<TextSimilarityRankBuilder, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
-        String inferenceId = (String) args[0];
-        String inferenceText = (String) args[1];
-        String field = (String) args[2];
-        Integer rankWindowSize = args[3] == null ? DEFAULT_RANK_WINDOW_SIZE : (Integer) args[3];
-        Float minScore = (Float) args[4];
-
-        return new TextSimilarityRankBuilder(field, inferenceId, inferenceText, rankWindowSize, minScore);
-    });
-
-    static {
-        PARSER.declareString(constructorArg(), INFERENCE_ID_FIELD);
-        PARSER.declareString(constructorArg(), INFERENCE_TEXT_FIELD);
-        PARSER.declareString(constructorArg(), FIELD_FIELD);
-        PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
-        PARSER.declareFloat(optionalConstructorArg(), MIN_SCORE_FIELD);
-    }
-
     private final String inferenceId;
     private final String inferenceText;
     private final String field;
     private final Float minScore;
-
-    public TextSimilarityRankBuilder(String field, String inferenceId, String inferenceText, int rankWindowSize, Float minScore) {
+    private final boolean failuresAllowed;
+
+    public TextSimilarityRankBuilder(
+        String field,
+        String inferenceId,
+        String inferenceText,
+        int rankWindowSize,
+        Float minScore,
+        boolean failuresAllowed
+    ) {
         super(rankWindowSize);
         this.inferenceId = inferenceId;
         this.inferenceText = inferenceText;
         this.field = field;
         this.minScore = minScore;
+        this.failuresAllowed = failuresAllowed;
     }
 
     public TextSimilarityRankBuilder(StreamInput in) throws IOException {
@@ -89,6 +72,11 @@ public class TextSimilarityRankBuilder extends RankBuilder {
         this.inferenceText = in.readString();
         this.field = in.readString();
         this.minScore = in.readOptionalFloat();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.RERANKER_FAILURES_ALLOWED)) {
+            this.failuresAllowed = in.readBoolean();
+        } else {
+            this.failuresAllowed = false;
+        }
     }
 
     @Override
@@ -108,17 +96,14 @@ public class TextSimilarityRankBuilder extends RankBuilder {
         out.writeString(inferenceText);
         out.writeString(field);
         out.writeOptionalFloat(minScore);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.RERANKER_FAILURES_ALLOWED)) {
+            out.writeBoolean(failuresAllowed);
+        }
     }
 
     @Override
     public void doXContent(XContentBuilder builder, Params params) throws IOException {
-        // rankWindowSize serialization is handled by the parent class RankBuilder
-        builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId);
-        builder.field(INFERENCE_TEXT_FIELD.getPreferredName(), inferenceText);
-        builder.field(FIELD_FIELD.getPreferredName(), field);
-        if (minScore != null) {
-            builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore);
-        }
+        throw new UnsupportedOperationException("This should not be XContent serialized");
     }
 
     @Override
@@ -177,7 +162,8 @@ public class TextSimilarityRankBuilder extends RankBuilder {
             client,
             inferenceId,
             inferenceText,
-            minScore
+            minScore,
+            failuresAllowed
         );
     }
 
@@ -197,17 +183,22 @@ public class TextSimilarityRankBuilder extends RankBuilder {
         return minScore;
     }
 
+    public boolean failuresAllowed() {
+        return failuresAllowed;
+    }
+
     @Override
     protected boolean doEquals(RankBuilder other) {
         TextSimilarityRankBuilder that = (TextSimilarityRankBuilder) other;
         return Objects.equals(inferenceId, that.inferenceId)
             && Objects.equals(inferenceText, that.inferenceText)
             && Objects.equals(field, that.field)
-            && Objects.equals(minScore, that.minScore);
+            && Objects.equals(minScore, that.minScore)
+            && failuresAllowed == that.failuresAllowed;
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(inferenceId, inferenceText, field, minScore);
+        return Objects.hash(inferenceId, inferenceText, field, minScore, failuresAllowed);
     }
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContext.java

@@ -43,9 +43,10 @@ public class TextSimilarityRankFeaturePhaseRankCoordinatorContext extends RankFe
         Client client,
         String inferenceId,
         String inferenceText,
-        Float minScore
+        Float minScore,
+        boolean failuresAllowed
     ) {
-        super(size, from, rankWindowSize);
+        super(size, from, rankWindowSize, failuresAllowed);
         this.client = client;
         this.inferenceId = inferenceId;
         this.inferenceText = inferenceText;
@@ -126,19 +127,25 @@ public class TextSimilarityRankFeaturePhaseRankCoordinatorContext extends RankFe
 
     /**
      * Sorts documents by score descending and discards those with a score less than minScore.
-     * @param originalDocs documents to process
+     *
+     * @param originalDocs   documents to process
+     * @param rerankedScores {@code true} if the document scores have been reranked
      */
     @Override
-    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
-        List<RankFeatureDoc> docs = new ArrayList<>();
+    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs, boolean rerankedScores) {
+        if (rerankedScores == false) {
+            // just return, don't normalize or apply minScore to scores that haven't been modified
+            return originalDocs;
+        }
+        List<RankFeatureDoc> docs = new ArrayList<>(originalDocs.length);
         for (RankFeatureDoc doc : originalDocs) {
             if (minScore == null || doc.score >= minScore) {
                 doc.score = normalizeScore(doc.score);
                 docs.add(doc);
             }
         }
-        docs.sort(RankFeatureDoc::compareTo);
-        return docs.toArray(new RankFeatureDoc[0]);
+        docs.sort(null);
+        return docs.toArray(RankFeatureDoc[]::new);
     }
 
     protected InferenceAction.Request generateRequest(List<String> docFeatures) {

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java

@@ -44,6 +44,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
     public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id");
     public static final ParseField INFERENCE_TEXT_FIELD = new ParseField("inference_text");
     public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField FAILURES_ALLOWED_FIELD = new ParseField("allow_rerank_failures");
 
     public static final ConstructingObjectParser<TextSimilarityRankRetrieverBuilder, RetrieverParserContext> PARSER =
         new ConstructingObjectParser<>(TextSimilarityRankBuilder.NAME, args -> {
@@ -52,8 +53,16 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
             String inferenceText = (String) args[2];
             String field = (String) args[3];
             int rankWindowSize = args[4] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[4];
-
-            return new TextSimilarityRankRetrieverBuilder(retrieverBuilder, inferenceId, inferenceText, field, rankWindowSize);
+            boolean failuresAllowed = args[5] != null && (Boolean) args[5];
+
+            return new TextSimilarityRankRetrieverBuilder(
+                retrieverBuilder,
+                inferenceId,
+                inferenceText,
+                field,
+                rankWindowSize,
+                failuresAllowed
+            );
         });
 
     static {
@@ -66,6 +75,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
         PARSER.declareString(constructorArg(), INFERENCE_TEXT_FIELD);
         PARSER.declareString(constructorArg(), FIELD_FIELD);
         PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
+        PARSER.declareBoolean(optionalConstructorArg(), FAILURES_ALLOWED_FIELD);
 
         RetrieverBuilder.declareBaseParserFields(TextSimilarityRankBuilder.NAME, PARSER);
     }
@@ -84,18 +94,21 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
     private final String inferenceId;
     private final String inferenceText;
     private final String field;
+    private final boolean failuresAllowed;
 
     public TextSimilarityRankRetrieverBuilder(
         RetrieverBuilder retrieverBuilder,
         String inferenceId,
         String inferenceText,
         String field,
-        int rankWindowSize
+        int rankWindowSize,
+        boolean failuresAllowed
     ) {
         super(List.of(new RetrieverSource(retrieverBuilder, null)), rankWindowSize);
         this.inferenceId = inferenceId;
         this.inferenceText = inferenceText;
         this.field = field;
+        this.failuresAllowed = failuresAllowed;
     }
 
     public TextSimilarityRankRetrieverBuilder(
@@ -105,6 +118,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
         String field,
         int rankWindowSize,
         Float minScore,
+        boolean failuresAllowed,
         String retrieverName,
         List<QueryBuilder> preFilterQueryBuilders
     ) {
@@ -116,6 +130,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
         this.inferenceText = inferenceText;
         this.field = field;
         this.minScore = minScore;
+        this.failuresAllowed = failuresAllowed;
         this.retrieverName = retrieverName;
         this.preFilterQueryBuilders = preFilterQueryBuilders;
     }
@@ -132,6 +147,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
             field,
             rankWindowSize,
             minScore,
+            failuresAllowed,
             retrieverName,
             newPreFilterQueryBuilders
         );
@@ -163,7 +179,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
     @Override
     protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder sourceBuilder) {
         sourceBuilder.rankBuilder(
-            new TextSimilarityRankBuilder(this.field, this.inferenceId, this.inferenceText, this.rankWindowSize, this.minScore)
+            new TextSimilarityRankBuilder(field, inferenceId, inferenceText, rankWindowSize, minScore, failuresAllowed)
         );
         return sourceBuilder;
     }
@@ -181,6 +197,10 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
         return rankWindowSize;
     }
 
+    public boolean failuresAllowed() {
+        return failuresAllowed;
+    }
+
     @Override
     protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(RETRIEVER_FIELD.getPreferredName(), innerRetrievers.getFirst().retriever());
@@ -188,6 +208,9 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
         builder.field(INFERENCE_TEXT_FIELD.getPreferredName(), inferenceText);
         builder.field(FIELD_FIELD.getPreferredName(), field);
         builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize);
+        if (failuresAllowed) {
+            builder.field(FAILURES_ALLOWED_FIELD.getPreferredName(), failuresAllowed);
+        }
     }
 
     @Override
@@ -197,12 +220,13 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
             && Objects.equals(inferenceId, that.inferenceId)
             && Objects.equals(inferenceText, that.inferenceText)
             && Objects.equals(field, that.field)
-            && Objects.equals(rankWindowSize, that.rankWindowSize)
-            && Objects.equals(minScore, that.minScore);
+            && rankWindowSize == that.rankWindowSize
+            && Objects.equals(minScore, that.minScore)
+            && failuresAllowed == that.failuresAllowed;
     }
 
     @Override
     public int doHashCode() {
-        return Objects.hash(inferenceId, inferenceText, field, rankWindowSize, minScore);
+        return Objects.hash(inferenceId, inferenceText, field, rankWindowSize, minScore, failuresAllowed);
     }
 }

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilderTests.java

@@ -1,94 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.rank.textsimilarity;
-
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
-
-import java.io.IOException;
-
-import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
-
-public class TextSimilarityRankBuilderTests extends AbstractXContentSerializingTestCase<TextSimilarityRankBuilder> {
-
-    @Override
-    protected TextSimilarityRankBuilder createTestInstance() {
-        return new TextSimilarityRankBuilder(
-            "my-field",
-            "my-inference-id",
-            "my-inference-text",
-            randomIntBetween(1, 1000),
-            randomBoolean() ? null : randomFloat()
-        );
-    }
-
-    @Override
-    protected TextSimilarityRankBuilder mutateInstance(TextSimilarityRankBuilder instance) throws IOException {
-        String field = instance.field();
-        String inferenceId = instance.inferenceId();
-        String inferenceText = instance.inferenceText();
-        int rankWindowSize = instance.rankWindowSize();
-        Float minScore = instance.minScore();
-
-        int mutate = randomIntBetween(0, 4);
-        switch (mutate) {
-            case 0 -> field = field + randomAlphaOfLength(2);
-            case 1 -> inferenceId = inferenceId + randomAlphaOfLength(2);
-            case 2 -> inferenceText = inferenceText + randomAlphaOfLength(2);
-            case 3 -> rankWindowSize = randomValueOtherThan(instance.rankWindowSize(), this::randomRankWindowSize);
-            case 4 -> minScore = randomValueOtherThan(instance.minScore(), this::randomMinScore);
-            default -> throw new IllegalStateException("Requested to modify more than available parameters.");
-        }
-        return new TextSimilarityRankBuilder(field, inferenceId, inferenceText, rankWindowSize, minScore);
-    }
-
-    @Override
-    protected Writeable.Reader<TextSimilarityRankBuilder> instanceReader() {
-        return TextSimilarityRankBuilder::new;
-    }
-
-    @Override
-    protected TextSimilarityRankBuilder doParseInstance(XContentParser parser) throws IOException {
-        parser.nextToken();
-        assertEquals(parser.currentToken(), XContentParser.Token.START_OBJECT);
-        parser.nextToken();
-        assertEquals(parser.currentToken(), XContentParser.Token.FIELD_NAME);
-        assertEquals(parser.currentName(), TextSimilarityRankBuilder.NAME);
-        TextSimilarityRankBuilder builder = TextSimilarityRankBuilder.PARSER.parse(parser, null);
-        parser.nextToken();
-        assertEquals(parser.currentToken(), XContentParser.Token.END_OBJECT);
-        parser.nextToken();
-        assertNull(parser.currentToken());
-        return builder;
-    }
-
-    private int randomRankWindowSize() {
-        return randomIntBetween(0, 1000);
-    }
-
-    private float randomMinScore() {
-        return randomFloatBetween(-1.0f, 1.0f, true);
-    }
-
-    public void testParserDefaults() throws IOException {
-        String json = """
-            {
-              "field": "my-field",
-              "inference_id": "my-inference-id",
-              "inference_text": "my-inference-text"
-            }""";
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
-            TextSimilarityRankBuilder parsed = TextSimilarityRankBuilder.PARSER.parse(parser, null);
-            assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize());
-        }
-    }
-
-}

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContextTests.java

@@ -7,13 +7,13 @@
 
 package org.elasticsearch.xpack.inference.rank.textsimilarity;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 
+import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
@@ -31,7 +31,8 @@ public class TextSimilarityRankFeaturePhaseRankCoordinatorContextTests extends E
         mockClient,
         "my-inference-id",
         "some query",
-        0.0f
+        0.0f,
+        false
     );
 
     public void testComputeScores() {
@@ -43,17 +44,7 @@ public class TextSimilarityRankFeaturePhaseRankCoordinatorContextTests extends E
         featureDoc3.featureData("text 3");
         RankFeatureDoc[] featureDocs = new RankFeatureDoc[] { featureDoc1, featureDoc2, featureDoc3 };
 
-        subject.computeScores(featureDocs, new ActionListener<>() {
-            @Override
-            public void onResponse(float[] floats) {
-                assertArrayEquals(new float[] { 1.0f, 3.0f, 2.0f }, floats, 0.0f);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                fail();
-            }
-        });
+        subject.computeScores(featureDocs, assertNoFailureListener(f -> assertArrayEquals(new float[] { 1.0f, 3.0f, 2.0f }, f, 0.0f)));
         verify(mockClient).execute(
             eq(GetInferenceModelAction.INSTANCE),
             argThat(actionRequest -> ((GetInferenceModelAction.Request) actionRequest).getTaskType().equals(TaskType.RERANK)),
@@ -62,17 +53,7 @@ public class TextSimilarityRankFeaturePhaseRankCoordinatorContextTests extends E
     }
 
     public void testComputeScoresForEmpty() {
-        subject.computeScores(new RankFeatureDoc[0], new ActionListener<>() {
-            @Override
-            public void onResponse(float[] floats) {
-                assertArrayEquals(new float[0], floats, 0.0f);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                fail();
-            }
-        });
+        subject.computeScores(new RankFeatureDoc[0], assertNoFailureListener(f -> assertArrayEquals(new float[0], f, 0.0f)));
         verify(mockClient).execute(
             eq(GetInferenceModelAction.INSTANCE),
             argThat(actionRequest -> ((GetInferenceModelAction.Request) actionRequest).getTaskType().equals(TaskType.RERANK)),

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.rank.textsimilarity;
 
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rank.rerank.AbstractRerankerIT;
 import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
@@ -15,6 +16,14 @@ import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
 import java.util.Collection;
 import java.util.List;
 
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasRank;
+
 public class TextSimilarityRankMultiNodeTests extends AbstractRerankerIT {
 
     private static final String inferenceId = "inference-id";
@@ -23,17 +32,27 @@ public class TextSimilarityRankMultiNodeTests extends AbstractRerankerIT {
 
     @Override
     protected RankBuilder getRankBuilder(int rankWindowSize, String rankFeatureField) {
-        return new TextSimilarityRankBuilder(rankFeatureField, inferenceId, inferenceText, rankWindowSize, minScore);
+        return new TextSimilarityRankBuilder(rankFeatureField, inferenceId, inferenceText, rankWindowSize, minScore, false);
     }
 
     @Override
     protected RankBuilder getThrowingRankBuilder(int rankWindowSize, String rankFeatureField, ThrowingRankBuilderType type) {
+        return getThrowingRankBuilder(rankWindowSize, rankFeatureField, type, false);
+    }
+
+    protected RankBuilder getThrowingRankBuilder(
+        int rankWindowSize,
+        String rankFeatureField,
+        ThrowingRankBuilderType type,
+        boolean failuresAllowed
+    ) {
         return new TextSimilarityTestPlugin.ThrowingMockRequestActionBasedRankBuilder(
             rankWindowSize,
             rankFeatureField,
             inferenceId,
             inferenceText,
             minScore,
+            failuresAllowed,
             type.name()
         );
     }
@@ -51,6 +70,58 @@ public class TextSimilarityRankMultiNodeTests extends AbstractRerankerIT {
         // no-op
     }
 
+    public void testRerankerAllowedFailureNoExceptions() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        assertNoFailuresAndResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(10))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(20))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(30))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(40))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(50))
+            )
+                .setRankBuilder(
+                    getThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT,
+                        true
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10),
+            response -> {
+                assertHitCount(response, 5L);
+                int rank = 1;
+                for (SearchHit searchHit : response.getHits().getHits()) {
+                    int id = 5 - (rank - 1);
+                    assertThat(searchHit, hasId(String.valueOf(id)));
+                    assertThat(searchHit, hasRank(rank));
+                    assertNotNull(searchHit.getFields().get(searchField));
+                    assertEquals(id * 10, searchHit.getScore(), 0f);
+                    rank++;
+                }
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
     @Override
     protected boolean shouldCheckScores() {
         return false;

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java

@@ -57,7 +57,8 @@ public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTes
             randomAlphaOfLength(10),
             randomAlphaOfLength(20),
             randomAlphaOfLength(50),
-            randomIntBetween(100, 10000)
+            randomIntBetween(100, 10000),
+            randomBoolean()
         );
     }
 
@@ -117,29 +118,33 @@ public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTes
                 parser,
                 new RetrieverParserContext(new SearchUsage(), nf -> true)
             );
-            assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize());
-            assertEquals(DEFAULT_RERANK_ID, parsed.inferenceId());
+            assertThat(parsed.rankWindowSize(), equalTo(DEFAULT_RANK_WINDOW_SIZE));
+            assertThat(parsed.inferenceId(), equalTo(DEFAULT_RERANK_ID));
+            assertThat(parsed.failuresAllowed(), equalTo(false));
+
         }
     }
 
     public void testTextSimilarityRetrieverParsing() throws IOException {
-        String restContent = "{"
-            + "  \"retriever\": {"
-            + "    \"text_similarity_reranker\": {"
-            + "      \"retriever\": {"
-            + "        \"test\": {"
-            + "          \"value\": \"my-test-retriever\""
-            + "        }"
-            + "      },"
-            + "      \"field\": \"my-field\","
-            + "      \"inference_id\": \"my-inference-id\","
-            + "      \"inference_text\": \"my-inference-text\","
-            + "      \"rank_window_size\": 100,"
-            + "      \"min_score\": 20.0,"
-            + "      \"_name\": \"foo_reranker\""
-            + "    }"
-            + "  }"
-            + "}";
+        String restContent = """
+            {
+              "retriever": {
+                "text_similarity_reranker": {
+                  "retriever": {
+                    "test": {
+                      "value": "my-test-retriever"
+                    }
+                  },
+                  "field": "my-field",
+                  "inference_id": "my-inference-id",
+                  "inference_text": "my-inference-text",
+                  "rank_window_size": 100,
+                  "min_score": 20.0,
+                  "allow_rerank_failures": true,
+                  "_name": "foo_reranker"
+                }
+              }
+            }""";
         SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder();
         try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) {
             SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true);
@@ -147,6 +152,7 @@ public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTes
             TextSimilarityRankRetrieverBuilder parsed = (TextSimilarityRankRetrieverBuilder) source.retriever();
             assertThat(parsed.minScore(), equalTo(20f));
             assertThat(parsed.retrieverName(), equalTo("foo_reranker"));
+            assertThat(parsed.failuresAllowed(), equalTo(true));
             try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) {
                 SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent(
                     parseSerialized,

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java

@@ -139,7 +139,8 @@ public class TextSimilarityRankRetrieverTelemetryTests extends ESIntegTestCase {
                         "some_inference_id",
                         "some_inference_text",
                         "some_field",
-                        10
+                        10,
+                        false
                     )
                 )
             );

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java

@@ -21,14 +21,22 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
+import org.hamcrest.Matcher;
 import org.junit.Before;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasRank;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -49,7 +57,7 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
             Float minScore,
             int topN
         ) {
-            super(field, inferenceId + "-task-settings-top-" + topN, inferenceText, rankWindowSize, minScore);
+            super(field, inferenceId + "-task-settings-top-" + topN, inferenceText, rankWindowSize, minScore, false);
         }
     }
 
@@ -68,7 +76,7 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
             Float minScore,
             int inferenceResultCount
         ) {
-            super(field, inferenceId, inferenceText, rankWindowSize, minScore);
+            super(field, inferenceId, inferenceText, rankWindowSize, minScore, false);
             this.inferenceResultCount = inferenceResultCount;
         }
 
@@ -81,7 +89,8 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
                 client,
                 inferenceId,
                 inferenceText,
-                minScore
+                minScore,
+                failuresAllowed()
             ) {
                 @Override
                 protected InferenceAction.Request generateRequest(List<String> docFeatures) {
@@ -125,18 +134,21 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
         ElasticsearchAssertions.assertNoFailuresAndResponse(
             // Execute search with text similarity reranking
             client.prepareSearch()
-                .setRankBuilder(new TextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 0.0f))
+                .setRankBuilder(new TextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 0.0f, false))
                 .setQuery(QueryBuilders.matchAllQuery()),
             response -> {
                 // Verify order, rank and score of results
-                SearchHit[] hits = response.getHits().getHits();
-                assertEquals(5, hits.length);
-                // we add + 1 to all expected scores due to the default normalization being applied which shifts positive scores to by 1
-                assertHitHasRankScoreAndText(hits[0], 1, 4.0f + 1f, "4");
-                assertHitHasRankScoreAndText(hits[1], 2, 3.0f + 1f, "3");
-                assertHitHasRankScoreAndText(hits[2], 3, 2.0f + 1f, "2");
-                assertHitHasRankScoreAndText(hits[3], 4, 1.0f + 1f, "1");
-                assertHitHasRankScoreAndText(hits[4], 5, 0.0f + 1f, "0");
+                assertThat(
+                    response.getHits().getHits(),
+                    arrayContaining(
+                        // add 1 to all expected scores due to the default normalization being applied which shifts positive scores by 1
+                        searchHitWith(1, 4.0f + 1f, "4"),
+                        searchHitWith(2, 3.0f + 1f, "3"),
+                        searchHitWith(3, 2.0f + 1f, "2"),
+                        searchHitWith(4, 1.0f + 1f, "1"),
+                        searchHitWith(5, 0.0f + 1f, "0")
+                    )
+                );
             }
         );
     }
@@ -145,15 +157,14 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
         ElasticsearchAssertions.assertNoFailuresAndResponse(
             // Execute search with text similarity reranking
             client.prepareSearch()
-                .setRankBuilder(new TextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 1.5f))
+                .setRankBuilder(new TextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 1.5f, false))
                 .setQuery(QueryBuilders.matchAllQuery()),
             response -> {
                 // Verify order, rank and score of results
-                SearchHit[] hits = response.getHits().getHits();
-                assertEquals(3, hits.length);
-                assertHitHasRankScoreAndText(hits[0], 1, 4.0f + 1f, "4");
-                assertHitHasRankScoreAndText(hits[1], 2, 3.0f + 1f, "3");
-                assertHitHasRankScoreAndText(hits[2], 3, 2.0f + 1f, "2");
+                assertThat(
+                    response.getHits().getHits(),
+                    arrayContaining(searchHitWith(1, 4.0f + 1f, "4"), searchHitWith(2, 3.0f + 1f, "3"), searchHitWith(3, 2.0f + 1f, "2"))
+                );
             }
         );
     }
@@ -169,6 +180,7 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
                         "my-rerank-model",
                         "my query",
                         0.7f,
+                        false,
                         AbstractRerankerIT.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT.name()
                     )
                 )
@@ -178,6 +190,44 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
         );
     }
 
+    public void testRerankInferenceAllowedFailure() {
+        ElasticsearchAssertions.assertNoFailuresAndResponse(
+            // Execute search with text similarity reranking that fails, but it is allowed
+            client.prepareSearch()
+                .setRankBuilder(
+                    new TextSimilarityTestPlugin.ThrowingMockRequestActionBasedRankBuilder(
+                        100,
+                        "text",
+                        "my-rerank-model",
+                        "my query",
+                        null,
+                        true,
+                        AbstractRerankerIT.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT.name()
+                    )
+                )
+                .setQuery(
+                    boolQuery().should(constantScoreQuery(matchQuery("text", "0")).boost(50))
+                        .should(constantScoreQuery(matchQuery("text", "1")).boost(40))
+                        .should(constantScoreQuery(matchQuery("text", "2")).boost(30))
+                        .should(constantScoreQuery(matchQuery("text", "3")).boost(20))
+                        .should(constantScoreQuery(matchQuery("text", "4")).boost(10))
+                ),
+            response -> {
+                // these will all have the scores from the constant score clauses
+                assertThat(
+                    response.getHits().getHits(),
+                    arrayContaining(
+                        searchHitWith(1, 50, "0"),
+                        searchHitWith(2, 40, "1"),
+                        searchHitWith(3, 30, "2"),
+                        searchHitWith(4, 20, "3"),
+                        searchHitWith(5, 10, "4")
+                    )
+                );
+            }
+        );
+    }
+
     public void testRerankTopNConfigurationAndRankWindowSizeMismatch() {
         SearchPhaseExecutionException ex = expectThrows(
             SearchPhaseExecutionException.class,
@@ -212,10 +262,11 @@ public class TextSimilarityRankTests extends ESSingleNodeTestCase {
         assertThat(ex.getDetailedMessage(), containsString("Reranker input document count and returned score count mismatch"));
     }
 
-    private static void assertHitHasRankScoreAndText(SearchHit hit, int expectedRank, float expectedScore, String expectedText) {
-        assertEquals(expectedRank, hit.getRank());
-        assertEquals(expectedScore, hit.getScore(), 0.0f);
-        assertEquals(expectedText, Objects.requireNonNull(hit.getSourceAsMap()).get("text"));
+    private static Matcher<SearchHit> searchHitWith(int expectedRank, float expectedScore, String expectedText) {
+        return allOf(
+            hasRank(expectedRank),
+            hasScore(expectedScore),
+            transformedMatch(hit -> hit.getSourceAsMap().get("text"), equalTo(expectedText))
+        );
     }
-
 }

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java

@@ -30,10 +30,6 @@ import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorCont
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
 import org.elasticsearch.search.rank.rerank.AbstractRerankerIT;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
@@ -51,8 +47,6 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Plugin for text similarity tests. Defines a filter for modifying inference call behavior, as well as a {@code TextSimilarityRankBuilder}
@@ -172,53 +166,18 @@ public class TextSimilarityTestPlugin extends Plugin implements ActionPlugin {
 
     public static class ThrowingMockRequestActionBasedRankBuilder extends TextSimilarityRankBuilder {
 
-        public static final ParseField FIELD_FIELD = new ParseField("field");
-        public static final ParseField INFERENCE_ID = new ParseField("inference_id");
-        public static final ParseField INFERENCE_TEXT = new ParseField("inference_text");
-        public static final ParseField THROWING_TYPE_FIELD = new ParseField("throwing-type");
-
-        static final ConstructingObjectParser<ThrowingMockRequestActionBasedRankBuilder, Void> PARSER = new ConstructingObjectParser<>(
-            "throwing_request_action_based_rank",
-            args -> {
-                int rankWindowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
-                String field = (String) args[1];
-                if (field == null || field.isEmpty()) {
-                    throw new IllegalArgumentException("Field cannot be null or empty");
-                }
-                final String inferenceId = (String) args[2];
-                final String inferenceText = (String) args[3];
-                final float minScore = (float) args[4];
-                String throwingType = (String) args[5];
-                return new ThrowingMockRequestActionBasedRankBuilder(
-                    rankWindowSize,
-                    field,
-                    inferenceId,
-                    inferenceText,
-                    minScore,
-                    throwingType
-                );
-            }
-        );
-
-        static {
-            PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
-            PARSER.declareString(constructorArg(), FIELD_FIELD);
-            PARSER.declareString(constructorArg(), INFERENCE_ID);
-            PARSER.declareString(constructorArg(), INFERENCE_TEXT);
-            PARSER.declareString(constructorArg(), THROWING_TYPE_FIELD);
-        }
-
         protected final AbstractRerankerIT.ThrowingRankBuilderType throwingRankBuilderType;
 
         public ThrowingMockRequestActionBasedRankBuilder(
-            final int rankWindowSize,
-            final String field,
-            final String inferenceId,
-            final String inferenceText,
-            final float minScore,
-            final String throwingType
+            int rankWindowSize,
+            String field,
+            String inferenceId,
+            String inferenceText,
+            Float minScore,
+            boolean failuresAllowed,
+            String throwingType
         ) {
-            super(field, inferenceId, inferenceText, rankWindowSize, minScore);
+            super(field, inferenceId, inferenceText, rankWindowSize, minScore, failuresAllowed);
             this.throwingRankBuilderType = AbstractRerankerIT.ThrowingRankBuilderType.valueOf(throwingType);
         }
 
@@ -233,12 +192,6 @@ public class TextSimilarityTestPlugin extends Plugin implements ActionPlugin {
             out.writeEnum(throwingRankBuilderType);
         }
 
-        @Override
-        public void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-            super.doXContent(builder, params);
-            builder.field(THROWING_TYPE_FIELD.getPreferredName(), throwingRankBuilderType);
-        }
-
         @Override
         public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
             if (this.throwingRankBuilderType == AbstractRerankerIT.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT)
@@ -263,7 +216,8 @@ public class TextSimilarityTestPlugin extends Plugin implements ActionPlugin {
                     client,
                     inferenceId,
                     inferenceText,
-                    minScore
+                    minScore,
+                    failuresAllowed()
                 ) {
                     @Override
                     protected InferenceAction.Request generateRequest(List<String> docFeatures) {

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml

@@ -208,6 +208,36 @@ setup:
               field: inference_text_field
           size: 10
 
+---
+"Text similarity reranking with allowed failure maintains custom sorting":
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    term:
+                      topic: "science"
+                  sort:
+                    - "subtopic"
+              rank_window_size: 10
+              inference_id: failing-rerank-model
+              inference_text: "science"
+              field: text
+              allow_rerank_failures: true
+          size: 10
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+
+  - match: { hits.hits.0._id: "doc_2" }
+  - match: { hits.hits.1._id: "doc_1" }
 
 ---
 "text similarity reranking with explain":

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml

@@ -356,3 +356,114 @@ setup:
   - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" }
   - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" }
   - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*astronomy.*/" }
+
+---
+"rrf retriever with failed text similarity reranker":
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            rrf: {
+              retrievers:
+                [
+                  {
+                    standard: {
+                      query: {
+                        bool: {
+                          should:
+                            [
+                              {
+                                constant_score: {
+                                  filter: {
+                                    term: {
+                                      integer: 1
+                                    }
+                                  },
+                                  boost: 10
+                                }
+                              },
+                              {
+                                constant_score:
+                                  {
+                                    filter:
+                                      {
+                                        term:
+                                          {
+                                            integer: 2
+                                          }
+                                      },
+                                    boost: 1
+                                  }
+                              }
+                            ]
+                        }
+                      }
+                    }
+                  },
+                  {
+                    text_similarity_reranker: {
+                      retriever:
+                        {
+                          standard: {
+                            query: {
+                              bool: {
+                                should:
+                                  [
+                                    {
+                                      constant_score: {
+                                        filter: {
+                                          term: {
+                                            integer: 1
+                                          }
+                                        },
+                                        boost: 3
+                                      }
+                                    },
+                                    {
+                                      constant_score: {
+                                        filter: {
+                                          term: {
+                                            integer: 2
+                                          }
+                                        },
+                                        boost: 2
+                                      }
+                                    },
+                                    {
+                                      constant_score: {
+                                        filter: {
+                                          term: {
+                                            integer: 3
+                                          }
+                                        },
+                                        boost: 1
+                                      }
+                                    }
+                                  ]
+                              }
+                            }
+                          }
+                        },
+                      rank_window_size: 10,
+                      inference_id: failure-rerank-model,
+                      inference_text: "How often does the moon hide the sun?",
+                      field: text,
+                      allow_rerank_failures: true
+                    }
+                  }
+                ],
+              rank_window_size: 10,
+              rank_constant: 1
+            }
+          size: 10
+
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { hits.hits.0._id: "doc_1" }
+  - match: { hits.hits.1._id: "doc_2" }
+  - match: { hits.hits.2._id: "doc_3" }