Log send failure at debug level if channel closed (#39807)

Currently we log exceptions due to channel close at the debug level in
the normal exception handler. Currently we log all send failures due to
channel close at the warn level. This commit changes that to only log at
warn if the send failure is not due to channel closed. Additionally, it
adds the ssl engine closed as a channel close exception.

server/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java

@@ -58,6 +58,9 @@ public class NetworkExceptionHelper {
             if (e.getMessage().equals("Socket closed")) {
                 return true;
             }
+            if (e.getMessage().equals("SSLEngine closed already")) {
+                return true;
+            }
         }
         return false;
     }

server/src/main/java/org/elasticsearch/transport/OutboundHandler.java

@@ -31,6 +31,7 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.network.CloseableChannel;
+import org.elasticsearch.common.transport.NetworkExceptionHelper;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -156,7 +157,11 @@ final class OutboundHandler {
 
         @Override
         protected void innerOnFailure(Exception e) {
-            logger.warn(() -> new ParameterizedMessage("send message failed [channel: {}]", channel), e);
+            if (NetworkExceptionHelper.isCloseConnectionException(e)) {
+                logger.debug(() -> new ParameterizedMessage("send message failed [channel: {}]", channel), e);
+            } else {
+                logger.warn(() -> new ParameterizedMessage("send message failed [channel: {}]", channel), e);
+            }
             closeAndCallback(() -> listener.onFailure(e));
         }