Load env variables from env files or secrets

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 ### Add
 - Add [multi-region mode](https://docs.imgproxy.net/latest/serving_files_from_s3?id=multi-region-mode) to S3 integration.
+- Add the ability to [load environment variables](https://docs.imgproxy.net/latest/loading_environment_variables) from a file or a cloud secret.
 - Add `IMGPROXY_WORKERS` alias for the `IMGPROXY_CONCURRENCY` config.
 
 ### Change

config/loadenv/aws.go

@@ -0,0 +1,121 @@
+package loadenv
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/DarthSim/godotenv"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/secretsmanager"
+	"github.com/aws/aws-sdk-go/service/ssm"
+)
+
+func loadAWSSecret() error {
+	secretID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_ID")
+	secretVersionID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_ID")
+	secretVersionStage := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE")
+	secretRegion := os.Getenv("IMGPROXY_ENV_AWS_SECRET_REGION")
+
+	if len(secretID) == 0 {
+		return nil
+	}
+
+	sess, err := session.NewSession()
+	if err != nil {
+		return fmt.Errorf("Can't create AWS Secrets Manager session: %s", err)
+	}
+
+	conf := aws.NewConfig()
+
+	if len(secretRegion) != 0 {
+		conf.Region = aws.String(secretRegion)
+	}
+
+	svc := secretsmanager.New(sess, conf)
+
+	input := secretsmanager.GetSecretValueInput{SecretId: aws.String(secretID)}
+	if len(secretVersionID) > 0 {
+		input.VersionId = aws.String(secretVersionID)
+	} else if len(secretVersionStage) > 0 {
+		input.VersionStage = aws.String(secretVersionStage)
+	}
+
+	output, err := svc.GetSecretValue(&input)
+	if err != nil {
+		return fmt.Errorf("Can't retrieve config from AWS Secrets Manager: %s", err)
+	}
+
+	if output.SecretString == nil {
+		return nil
+	}
+
+	envmap, err := godotenv.Unmarshal(*output.SecretString)
+	if err != nil {
+		return fmt.Errorf("Can't parse config from AWS Secrets Manager: %s", err)
+	}
+
+	for k, v := range envmap {
+		if err = os.Setenv(k, v); err != nil {
+			return fmt.Errorf("Can't set %s env variable from AWS Secrets Manager: %s", k, err)
+		}
+	}
+
+	return nil
+}
+
+func loadAWSSystemManagerParams() error {
+	paramsPath := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH")
+	paramsRegion := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION")
+
+	if len(paramsPath) == 0 {
+		return nil
+	}
+
+	sess, err := session.NewSession()
+	if err != nil {
+		return fmt.Errorf("Can't create AWS SSM session: %s", err)
+	}
+
+	conf := aws.NewConfig()
+
+	if len(paramsRegion) != 0 {
+		conf.Region = aws.String(paramsRegion)
+	}
+
+	svc := ssm.New(sess, conf)
+
+	input := ssm.GetParametersByPathInput{
+		Path:           aws.String(paramsPath),
+		WithDecryption: aws.Bool(true),
+	}
+
+	output, err := svc.GetParametersByPath(&input)
+	if err != nil {
+		return fmt.Errorf("Can't retrieve parameters from AWS SSM: %s", err)
+	}
+
+	for _, p := range output.Parameters {
+		if p == nil || p.Name == nil || p.Value == nil {
+			continue
+		}
+
+		if p.DataType == nil || *p.DataType != "text" {
+			continue
+		}
+
+		name := *p.Name
+
+		env := strings.ReplaceAll(
+			strings.TrimPrefix(strings.TrimPrefix(name, paramsPath), "/"),
+			"/", "_",
+		)
+
+		if err = os.Setenv(env, *p.Value); err != nil {
+			return fmt.Errorf("Can't set %s env variable from AWS SSM: %s", env, err)
+		}
+	}
+
+	return nil
+}

config/loadenv/gcp.go

@@ -0,0 +1,82 @@
+package loadenv
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
+	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	"github.com/DarthSim/godotenv"
+	"google.golang.org/api/option"
+)
+
+func loadGCPSecret() error {
+	secretID := os.Getenv("IMGPROXY_ENV_GCP_SECRET_ID")
+	secretVersion := os.Getenv("IMGPROXY_ENV_GCP_SECRET_VERSION_ID")
+	secretProject := os.Getenv("IMGPROXY_ENV_GCP_SECRET_PROJECT_ID")
+	secretKey := os.Getenv("IMGPROXY_ENV_GCP_KEY")
+
+	if len(secretID) == 0 {
+		return nil
+	}
+
+	if len(secretVersion) == 0 {
+		secretVersion = "latest"
+	}
+
+	var (
+		client *secretmanager.Client
+		err    error
+	)
+
+	ctx, ctxcancel := context.WithTimeout(context.Background(), time.Minute)
+	defer ctxcancel()
+
+	opts := []option.ClientOption{}
+
+	if len(secretKey) > 0 {
+		opts = append(opts, option.WithCredentialsJSON([]byte(secretKey)))
+	}
+
+	client, err = secretmanager.NewClient(ctx, opts...)
+
+	if err != nil {
+		return fmt.Errorf("Can't create Google Cloud Secret Manager client: %s", err)
+	}
+
+	req := secretmanagerpb.AccessSecretVersionRequest{
+		Name: fmt.Sprintf("projects/%s/secrets/%s/versions/%s", secretProject, secretID, secretVersion),
+	}
+
+	resp, err := client.AccessSecretVersion(ctx, &req)
+	if err != nil {
+		return fmt.Errorf("Can't get Google Cloud Secret Manager secret: %s", err)
+	}
+
+	payload := resp.GetPayload()
+	if payload == nil {
+		return errors.New("Can't get Google Cloud Secret Manager secret: payload is empty")
+	}
+
+	data := payload.GetData()
+
+	if len(data) == 0 {
+		return nil
+	}
+
+	envmap, err := godotenv.Unmarshal(string(data))
+	if err != nil {
+		return fmt.Errorf("Can't parse config from Google Cloud Secrets Manager: %s", err)
+	}
+
+	for k, v := range envmap {
+		if err = os.Setenv(k, v); err != nil {
+			return fmt.Errorf("Can't set %s env variable from Google Cloud Secrets Manager: %s", k, err)
+		}
+	}
+
+	return nil
+}

config/loadenv/loadenv.go

@@ -0,0 +1,21 @@
+package loadenv
+
+func Load() error {
+	if err := loadAWSSecret(); err != nil {
+		return err
+	}
+
+	if err := loadAWSSystemManagerParams(); err != nil {
+		return err
+	}
+
+	if err := loadGCPSecret(); err != nil {
+		return err
+	}
+
+	if err := loadLocalFile(); err != nil {
+		return err
+	}
+
+	return nil
+}

config/loadenv/local_file.go

@@ -0,0 +1,38 @@
+package loadenv
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/DarthSim/godotenv"
+)
+
+func loadLocalFile() error {
+	path := os.Getenv("IMGPROXY_ENV_LOCAL_FILE_PATH")
+
+	if len(path) == 0 {
+		return nil
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("Can't read loacal environment file: %s", err)
+	}
+
+	if len(data) == 0 {
+		return nil
+	}
+
+	envmap, err := godotenv.Unmarshal(string(data))
+	if err != nil {
+		return fmt.Errorf("Can't parse config from local file: %s", err)
+	}
+
+	for k, v := range envmap {
+		if err = os.Setenv(k, v); err != nil {
+			return fmt.Errorf("Can't set %s env variable from local file: %s", k, err)
+		}
+	}
+
+	return nil
+}

docs/_sidebar.md

@@ -1,7 +1,9 @@
 * [Getting started](GETTING_STARTED.md)
 * [Pro version<img src="/assets/pro.svg">](https://imgproxy.net/#pro)
 * [Installation](installation.md)
-* [Configuration](configuration.md)
+* Configuration
+  * [Configuration](configuration.md)
+  * [Loading environment variables](loading_environment_variables.md)
 * Generating the URL
   * [Generating the URL](generating_the_url.md)
   * [Getting the image info<img title="imgproxy Pro feature" src="/assets/pro.svg">](getting_the_image_info.md)

docs/cloud_watch.md

@@ -42,8 +42,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
 docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
 ```
 
-This is the recommended method when using dockerized imgproxy.
-
 #### Shared credentials file
 
 Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:

docs/loading_environment_variables.md

@@ -0,0 +1,160 @@
+# Loading environment variables
+
+imgproxy can load environment variables from various sources such as:
+
+* [Local file](#local-file)
+* [AWS Secrets Manager](#aws-secrets-manager)
+* [AWS Systems Manager Parameter Store](#aws-systems-manager-parameter-store)
+* [Google Cloud Secret Manager](#google-cloud-secret-manager)
+
+## Local file
+
+You can create an [environment file](#environment-file-syntax) and configure imgproxy to read environment variables from it.
+
+* `IMGPROXY_ENV_LOCAL_FILE_PATH`: the path of the environmebt file to load
+
+## AWS Secrets Manager
+
+You can store the content of an [environment file](#environment-file-syntax) as an AWS Secrets Manager secret and configure imgproxy to read environment variables from it.
+
+* `IMGPROXY_ENV_AWS_SECRET_ID`: the ARN or name of the secret to load
+* `IMGPROXY_ENV_AWS_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load
+* `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE`: _(optional)_ the staging label of the version of the secret to load
+* `IMGPROXY_ENV_AWS_SECRET_REGION`: _(optional)_ the region of the secret to load
+
+**📝 Note:** If both `IMGPROXY_ENV_AWS_SECRET_VERSION_ID` and `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` are set, `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` will be ignored
+
+### Set up AWS Secrets Manager credentials
+
+There are three ways to specify your AWS credentials. The credentials policy should allow performing the `secretsmanager:GetSecretValue` and `secretsmanager:ListSecretVersionIds` actions with the specified secret:
+
+#### IAM Roles
+
+If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret.
+
+* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
+* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html).
+* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html).
+
+#### Environment variables
+
+You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+
+``` bash
+AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
+
+# same for Docker
+docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
+```
+
+#### Shared credentials file
+
+Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:
+
+```ini
+[default]
+aws_access_key_id = %access_key_id
+aws_secret_access_key = %secret_access_key
+```
+
+## AWS Systems Manager Parameter Store
+
+You can store multiple AWS Systems Manager Parameter Store entries and configure imgproxy to load their values to separate environment variables.
+
+* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH`: the [path](#aws-systems-manager-path) of the parameters to load
+* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION`: _(optional)_ the region of the parameters to load
+
+### AWS Systems Manager path
+
+Let's assume that you created the following AWS Systems Manager parameters:
+
+* `/imgproxy/prod/IMGPROXY_KEY`
+* `/imgproxy/prod/IMGPROXY_SALT`
+* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME`
+* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE`
+* `/imgproxy/staging/IMGPROXY_KEY`
+
+If you set `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH` to `/imgproxy/prod`, imgproxy will load these parameters the following way:
+
+* `/imgproxy/prod/IMGPROXY_KEY` value will be loaded to `IMGPROXY_KEY`
+* `/imgproxy/prod/IMGPROXY_SALT` value will be loaded to `IMGPROXY_SALT`
+* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME` value will be loaded to `IMGPROXY_CLOUD_WATCH_SERVICE_NAME`
+* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE` value will be loaded to `IMGPROXY_CLOUD_WATCH_NAMESPACE`
+* `/imgproxy/staging/IMGPROXY_KEY` will be ignored since its path is not `/imgproxy/prod`
+
+### Set up AWS Systems Manager credentials
+
+There are three ways to specify your AWS credentials. The credentials policy should allow performing the `ssm:GetParametersByPath` action with the specified parameters:
+
+#### IAM Roles
+
+If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret.
+
+* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
+* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html).
+* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html).
+
+#### Environment variables
+
+You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+
+``` bash
+AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
+
+# same for Docker
+docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
+```
+
+#### Shared credentials file
+
+Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:
+
+```ini
+[default]
+aws_access_key_id = %access_key_id
+aws_secret_access_key = %secret_access_key
+```
+
+## Google Cloud Secret Manager
+
+You can store the content of an [environment file](#environment-file-syntax) in Google Cloud Secret Manager secret and configure imgproxy to read environment variables from it.
+
+* `IMGPROXY_ENV_GCP_SECRET_ID`: the name of the secret to load
+* `IMGPROXY_ENV_GCP_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load
+* `IMGPROXY_ENV_GCP_SECRET_PROJECT_ID`: the name or ID of the Google Cloud project that contains the secret
+
+### Setup credentials
+
+If you run imgproxy inside Google Cloud infrastructure (Compute Engine, Kubernetes Engine, App Engine, Cloud Functions, etc), and you have granted access to the specified secret to the service account, you probably don't need to do anything here. imgproxy will try to use the credentials provided by Google.
+
+Otherwise, set `IMGPROXY_ENV_GCP_KEY` environment variable to the content of Google Cloud JSON key. Get more info about JSON keys: [https://cloud.google.com/iam/docs/creating-managing-service-account-keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys).
+
+## Environment file syntax
+
+The following syntax rules apply to environment files:
+
+* Blank lines are ignored
+* Lines beginning with `#` are processed as comments and ignored
+* Each line represents a key-value pair. Values can optionally be quoted:
+  * `VAR=VAL` -> `VAL`
+  * `VAR="VAL"` -> `VAL`
+  * `VAR='VAL'` -> `VAL`
+* Unquoted and double-quoted (`"`) values have variable substitution applied:
+  * `VAR=${OTHER_VAR}` -> value of `OTHER_VAR`
+  * `VAR=$OTHER_VAR` -> value of `OTHER_VAR`
+  * `VAR="$OTHER_VAR"` -> value of `OTHER_VAR`
+  * `VAR="${OTHER_VAR}"` -> value of `OTHER_VAR`
+* Single-quoted (`'`) values are used literally:
+  * `VAR='$OTHER_VAR'` -> `$OTHER_VAR`
+  * `VAR='${OTHER_VAR}'` -> `${OTHER_VAR}`
+* Double quotes in double-quoted (`"`) values can be escaped with `\`:
+  * `VAR="{\"hello\": \"json\"}"` -> `{"hello": "json"}`
+* Slash (`\`) in double-quoted values can be escaped with another slash:
+  * `VAR="some\\value"` -> `some\value`
+* A new line can be added to double-quoted values using `\n`:
+  * `VAR="some\nvalue"` ->
+    ```
+    some
+    value
+    ```
+

docs/serving_files_from_s3.md

@@ -39,8 +39,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
 docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
 ```
 
-This is the recommended method when using dockerized imgproxy.
-
 #### Shared credentials file
 
 Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
+	github.com/DarthSim/godotenv v1.3.1
 	github.com/DataDog/datadog-go/v5 v5.3.0
 	github.com/airbrake/gobrake/v5 v5.6.1
 	github.com/aws/aws-sdk-go v1.44.260
@@ -58,6 +59,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.0.1 // indirect
 	cloud.google.com/go/pubsub v1.30.0 // indirect
+	cloud.google.com/go/secretmanager v1.10.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/DataDog/appsec-internal-go v1.0.0 // indirect

go.sum

@@ -37,6 +37,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/pubsub v1.30.0 h1:vCge8m7aUKBJYOgrZp7EsNDf6QMd2CAlXZqWTn3yq6s=
 cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4=
+cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
+cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
@@ -59,6 +61,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272 h1:Gj21neabaU3DEwwVPG6/Vn0GSuDQ1n2m2z1qV33SCI0=
 github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272/go.mod h1:Cnosl0cRZIfKjTMuH49sQog2LeNsU5Hf4WnPIDWIDV0=
+github.com/DarthSim/godotenv v1.3.1 h1:NMWdswlRx2M9uPY4Ux8p/Q/rDs7A97OG89fECiQ/Tz0=
+github.com/DarthSim/godotenv v1.3.1/go.mod h1:B3ySe1HYTUFFR6+TPyHyxPWjUdh48il0Blebg9p1cCc=
 github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c h1:6XK2HjE3YbWRAl4nNpXFwiZ8LP+JZjxihvZw5ZUgyss=
 github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c/go.mod h1:OshtJzwB+6SKoFM4ovJIbsHuwg7PpLGIbpaAOHJwyUU=
 github.com/DataDog/appsec-internal-go v1.0.0 h1:2u5IkF4DBj3KVeQn5Vg2vjPUtt513zxEYglcqnd500U=

main.go

@@ -13,6 +13,7 @@ import (
 	"go.uber.org/automaxprocs/maxprocs"
 
 	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/imgproxy/imgproxy/v3/config/loadenv"
 	"github.com/imgproxy/imgproxy/v3/errorreport"
 	"github.com/imgproxy/imgproxy/v3/gliblog"
 	"github.com/imgproxy/imgproxy/v3/imagedata"
@@ -35,6 +36,10 @@ func initialize() error {
 
 	maxprocs.Set(maxprocs.Logger(log.Debugf))
 
+	if err := loadenv.Load(); err != nil {
+		return err
+	}
+
 	if err := config.Configure(); err != nil {
 		return err
 	}