Remove interim handling of serverless rest protections (#96814)

A special header "X-elastic-internal-origin" was introduced to help facilitate testing of serverless 
scopes introduced in #93607. The use of this header has been superseded and as such this 
special header is no longer needed. This commit removes reference to the header and adjusts 
the tests accordingly. Internal vs. Public scopes are now controlled by operator privileges when 
running in serverless mode. This commit also changes the response from 400 to 404 if the 
requested path is not available. This matches the 404 response code returned by operator 
privs if internal scopes are not reachable.

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -57,6 +57,7 @@ import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 import static org.elasticsearch.rest.RestStatus.METHOD_NOT_ALLOWED;
 import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE;
+import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestController implements HttpServerTransport.Dispatcher {
@@ -70,7 +71,6 @@ public class RestController implements HttpServerTransport.Dispatcher {
     static final Set<String> SAFELISTED_MEDIA_TYPES = Set.of("application/x-www-form-urlencoded", "multipart/form-data", "text/plain");
 
     static final String ELASTIC_PRODUCT_HTTP_HEADER = "X-elastic-product";
-    static final String ELASTIC_INTERNAL_ORIGIN_HTTP_HEADER = "X-elastic-internal-origin";
     static final String ELASTIC_PRODUCT_HTTP_HEADER_VALUE = "Elasticsearch";
     static final Set<String> RESERVED_PATHS = Set.of("/__elb_health__", "/__elb_health__/zk", "/_health", "/_health/zk");
     private static final BytesReference FAVICON_RESPONSE;
@@ -376,14 +376,7 @@ public class RestController implements HttpServerTransport.Dispatcher {
         RestChannel responseChannel = channel;
         if (serverlessEnabled) {
             Scope scope = handler.getServerlessScope();
-            if (Scope.INTERNAL.equals(scope)) {
-                final String internalOrigin = request.header(ELASTIC_INTERNAL_ORIGIN_HTTP_HEADER);
-                boolean internalRequest = internalOrigin != null;
-                if (internalRequest == false) {
-                    handleServerlessRequestToProtectedResource(request.uri(), request.method(), responseChannel);
-                    return;
-                }
-            } else if (Scope.PUBLIC.equals(scope) == false) {
+            if (scope == null) {
                 handleServerlessRequestToProtectedResource(request.uri(), request.method(), responseChannel);
                 return;
             }
@@ -677,7 +670,7 @@ public class RestController implements HttpServerTransport.Dispatcher {
                 );
             }
             builder.endObject();
-            channel.sendResponse(new RestResponse(BAD_REQUEST, builder));
+            channel.sendResponse(new RestResponse(NOT_FOUND, builder));
         }
     }
 

server/src/test/java/org/elasticsearch/rest/RestControllerTests.java

@@ -60,7 +60,6 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.rest.RestController.ELASTIC_INTERNAL_ORIGIN_HTTP_HEADER;
 import static org.elasticsearch.rest.RestController.ELASTIC_PRODUCT_HTTP_HEADER;
 import static org.elasticsearch.rest.RestController.ELASTIC_PRODUCT_HTTP_HEADER_VALUE;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -924,49 +923,23 @@ public class RestControllerTests extends ESTestCase {
     }
 
     /*
-     * Test that when serverless is enabled, a normal user not using the X-elastic-internal-origin header can only access endpoints
-     * annotated with a PUBLIC scope.
+     * Test that when serverless is enabled, a normal user can not access endpoints without a ServerlessScope annotation.
      */
     public void testApiProtectionWithServerlessEnabledAsEndUser() {
         final RestController restController = new RestController(null, client, circuitBreakerService, new UsageService(), tracer, true);
         restController.registerHandler(new PublicRestHandler());
         restController.registerHandler(new InternalRestHandler());
         restController.registerHandler(new HiddenRestHandler());
-        List<String> accessiblePaths = List.of("/public");
-        accessiblePaths.forEach(path -> {
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath(path).build();
-            AssertingChannel channel = new AssertingChannel(request, false, RestStatus.OK);
-            restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-        });
-        List<String> inaccessiblePaths = List.of("/internal", "/hidden");
-        inaccessiblePaths.forEach(path -> {
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath(path).build();
-            AssertingChannel channel = new AssertingChannel(request, false, RestStatus.BAD_REQUEST);
-            restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-        });
-    }
-
-    /*
-     * Test that when serverless is enabled, a system user using the X-elastic-internal-origin header can only access endpoints
-     * annotated with a PUBLIC or INTERNAL scope.
-     */
-    public void testApiProtectionWithServerlessEnabledAsInternalUser() {
-        final RestController restController = new RestController(null, client, circuitBreakerService, new UsageService(), tracer, true);
-        restController.registerHandler(new PublicRestHandler());
-        restController.registerHandler(new InternalRestHandler());
-        restController.registerHandler(new HiddenRestHandler());
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put(ELASTIC_INTERNAL_ORIGIN_HTTP_HEADER, Collections.singletonList("true"));
         List<String> accessiblePaths = List.of("/public", "/internal");
         accessiblePaths.forEach(path -> {
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).withPath(path).build();
+            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath(path).build();
             AssertingChannel channel = new AssertingChannel(request, false, RestStatus.OK);
             restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
         });
         List<String> inaccessiblePaths = List.of("/hidden");
         inaccessiblePaths.forEach(path -> {
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).withPath(path).build();
-            AssertingChannel channel = new AssertingChannel(request, false, RestStatus.BAD_REQUEST);
+            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath(path).build();
+            AssertingChannel channel = new AssertingChannel(request, false, RestStatus.NOT_FOUND);
             restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
         });
     }