Fix detection of dead HTTP/2 connections

CHANGELOG.md

@@ -4,6 +4,9 @@
 ### Add
 - Add support for `Sec-CH-DPR` and `Sec-CH-Width` client hints.
 
+### Fix
+- Fix detection of dead HTTP/2 connections.
+
 ### Remove
 - Remove suport for `Viewport-Width` client hint.
 

go.mod

@@ -43,8 +43,8 @@ require (
 	go.opentelemetry.io/otel/trace v1.13.0
 	go.uber.org/automaxprocs v1.5.1
 	golang.org/x/image v0.5.0
-	golang.org/x/net v0.7.0
-	golang.org/x/sys v0.5.0
+	golang.org/x/net v0.9.0
+	golang.org/x/sys v0.7.0
 	google.golang.org/api v0.110.0
 	google.golang.org/grpc v1.53.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.47.0
@@ -136,8 +136,8 @@ require (
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/term v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

go.sum

@@ -564,8 +564,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -639,14 +639,14 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -657,8 +657,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

imagedata/download.go

@@ -3,19 +3,17 @@ package imagedata
 import (
 	"compress/gzip"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/cookiejar"
-	"syscall"
 	"time"
 
 	"github.com/imgproxy/imgproxy/v3/config"
 	"github.com/imgproxy/imgproxy/v3/ierrors"
 	"github.com/imgproxy/imgproxy/v3/security"
 
+	defaultTransport "github.com/imgproxy/imgproxy/v3/transport"
 	azureTransport "github.com/imgproxy/imgproxy/v3/transport/azure"
 	fsTransport "github.com/imgproxy/imgproxy/v3/transport/fs"
 	gcsTransport "github.com/imgproxy/imgproxy/v3/transport/gcs"
@@ -58,32 +56,9 @@ func (e *ErrorNotModified) Error() string {
 }
 
 func initDownloading() error {
-	transport := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-			Control: func(network, address string, c syscall.RawConn) error {
-				return security.VerifySourceNetwork(address)
-			},
-		}).DialContext,
-		MaxIdleConns:          100,
-		MaxIdleConnsPerHost:   config.Concurrency + 1,
-		IdleConnTimeout:       time.Duration(config.ClientKeepAliveTimeout) * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		ForceAttemptHTTP2:     true,
-		DisableCompression:    true,
-	}
-
-	if config.ClientKeepAliveTimeout <= 0 {
-		transport.MaxIdleConnsPerHost = -1
-		transport.DisableKeepAlives = true
-	}
-
-	if config.IgnoreSslVerification {
-		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	transport, err := defaultTransport.New(true)
+	if err != nil {
+		return err
 	}
 
 	registerProtocol := func(scheme string, rt http.RoundTripper) {

transport/gcs/gcs.go

@@ -9,11 +9,15 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
+	"github.com/pkg/errors"
 	"google.golang.org/api/option"
+	raw "google.golang.org/api/storage/v1"
+	htransport "google.golang.org/api/transport/http"
 
 	"github.com/imgproxy/imgproxy/v3/config"
 	"github.com/imgproxy/imgproxy/v3/ctxreader"
 	"github.com/imgproxy/imgproxy/v3/httprange"
+	defaultTransport "github.com/imgproxy/imgproxy/v3/transport"
 	"github.com/imgproxy/imgproxy/v3/transport/notmodified"
 )
 
@@ -24,13 +28,27 @@ type transport struct {
 	client *storage.Client
 }
 
+func buildHTTPClient(opts ...option.ClientOption) (*http.Client, error) {
+	trans, err := defaultTransport.New(false)
+	if err != nil {
+		return nil, err
+	}
+
+	htrans, err := htransport.NewTransport(context.Background(), trans, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating GCS transport")
+	}
+
+	return &http.Client{Transport: htrans}, nil
+}
+
 func New() (http.RoundTripper, error) {
-	var (
-		client *storage.Client
-		err    error
-	)
+	var client *storage.Client
 
-	opts := []option.ClientOption{}
+	opts := []option.ClientOption{
+		option.WithScopes(raw.DevstorageReadOnlyScope),
+		option.WithUserAgent(config.UserAgent),
+	}
 
 	if len(config.GCSKey) > 0 {
 		opts = append(opts, option.WithCredentialsJSON([]byte(config.GCSKey)))
@@ -44,6 +62,12 @@ func New() (http.RoundTripper, error) {
 		opts = append(opts, option.WithoutAuthentication())
 	}
 
+	httpClient, err := buildHTTPClient(opts...)
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, option.WithHTTPClient(httpClient))
+
 	client, err = storage.NewClient(context.Background(), opts...)
 
 	if err != nil {

transport/s3/s3.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 
 	"github.com/imgproxy/imgproxy/v3/config"
+	defaultTransport "github.com/imgproxy/imgproxy/v3/transport"
 )
 
 // transport implements RoundTripper for the 's3' protocol.
@@ -23,6 +24,13 @@ type transport struct {
 func New() (http.RoundTripper, error) {
 	s3Conf := aws.NewConfig()
 
+	trans, err := defaultTransport.New(false)
+	if err != nil {
+		return nil, err
+	}
+
+	s3Conf.HTTPClient = &http.Client{Transport: trans}
+
 	if len(config.S3Region) != 0 {
 		s3Conf.Region = aws.String(config.S3Region)
 	}

transport/swift/swift.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/imgproxy/imgproxy/v3/config"
 	"github.com/imgproxy/imgproxy/v3/ctxreader"
+	defaultTransport "github.com/imgproxy/imgproxy/v3/transport"
 	"github.com/imgproxy/imgproxy/v3/transport/notmodified"
 )
 
@@ -21,6 +22,11 @@ type transport struct {
 }
 
 func New() (http.RoundTripper, error) {
+	trans, err := defaultTransport.New(false)
+	if err != nil {
+		return nil, err
+	}
+
 	c := &swift.Connection{
 		UserName:       config.SwiftUsername,
 		ApiKey:         config.SwiftAPIKey,
@@ -30,11 +36,12 @@ func New() (http.RoundTripper, error) {
 		Tenant:         config.SwiftTenant, // v2 auth only
 		Timeout:        time.Duration(config.SwiftTimeoutSeconds) * time.Second,
 		ConnectTimeout: time.Duration(config.SwiftConnectTimeoutSeconds) * time.Second,
+		Transport:      trans,
 	}
 
 	ctx := context.Background()
 
-	err := c.Authenticate(ctx)
+	err = c.Authenticate(ctx)
 
 	if err != nil {
 		return nil, fmt.Errorf("swift authentication error: %s", err)

transport/transport.go

@@ -0,0 +1,59 @@
+package transport
+
+import (
+	"crypto/tls"
+	"net"
+	"net/http"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/http2"
+
+	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/imgproxy/imgproxy/v3/security"
+)
+
+func New(verifyNetworks bool) (*http.Transport, error) {
+	dialer := &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+		DualStack: true,
+	}
+
+	if verifyNetworks {
+		dialer.Control = func(network, address string, c syscall.RawConn) error {
+			return security.VerifySourceNetwork(address)
+		}
+	}
+
+	transport := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           dialer.DialContext,
+		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   config.Concurrency + 1,
+		IdleConnTimeout:       time.Duration(config.ClientKeepAliveTimeout) * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		ForceAttemptHTTP2:     false,
+		DisableCompression:    true,
+	}
+
+	if config.ClientKeepAliveTimeout <= 0 {
+		transport.MaxIdleConnsPerHost = -1
+		transport.DisableKeepAlives = true
+	}
+
+	if config.IgnoreSslVerification {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	transport2, err := http2.ConfigureTransports(transport)
+	if err != nil {
+		return nil, err
+	}
+
+	transport2.PingTimeout = 5 * time.Second
+	transport2.ReadIdleTimeout = time.Second
+
+	return transport, nil
+}