From e7045033a65b4c507f5bb97887ef645ad1415354 Mon Sep 17 00:00:00 2001 From: Jem Davies <131159520+jem-davies@users.noreply.github.com> Date: Mon, 21 Oct 2024 20:51:54 +0100 Subject: [PATCH] Enable using AWS Secrets from Secret Manager for SQL DSN URLs (#135) Enable using AWS Secrets from Secret Manager for SQL DSN URLs --------- Signed-off-by: Jem Davies --- go.mod | 9 +- go.sum | 18 +- internal/impl/aws/integration_s3_test.go | 1 + internal/impl/gcp/output_bigquery.go | 3 - internal/impl/sql/cache_sql.go | 12 +- internal/impl/sql/conn_fields.go | 97 +++++++- internal/impl/sql/conn_fields_test.go | 96 ++++++++ internal/impl/sql/input_sql_raw.go | 11 +- internal/impl/sql/input_sql_select.go | 11 +- internal/impl/sql/output_sql_deprecated.go | 12 +- internal/impl/sql/output_sql_insert.go | 11 +- internal/impl/sql/output_sql_raw.go | 15 +- internal/impl/sql/processor_sql_deprecated.go | 11 +- internal/impl/sql/processor_sql_insert.go | 9 +- internal/impl/sql/processor_sql_raw.go | 17 +- internal/impl/sql/processor_sql_select.go | 11 +- website/cookbooks/redshift.md | 230 ++++++++++++++++++ website/docs/components/caches/sql.md | 101 ++++++++ website/docs/components/inputs/sql_raw.md | 101 ++++++++ website/docs/components/inputs/sql_select.md | 101 ++++++++ website/docs/components/outputs/sql_insert.md | 101 ++++++++ website/docs/components/outputs/sql_raw.md | 101 ++++++++ .../docs/components/processors/sql_insert.md | 101 ++++++++ website/docs/components/processors/sql_raw.md | 101 ++++++++ .../docs/components/processors/sql_select.md | 101 ++++++++ 25 files changed, 1352 insertions(+), 30 deletions(-) create mode 100644 website/cookbooks/redshift.md diff --git a/go.mod b/go.mod index cc8710335..bfc69d357 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/PaesslerAG/jsonpath v0.1.1 github.com/apache/pulsar-client-go v0.12.0 github.com/aws/aws-lambda-go v1.46.0 - github.com/aws/aws-sdk-go-v2 v1.25.0 + github.com/aws/aws-sdk-go-v2 v1.32.2 github.com/aws/aws-sdk-go-v2/config v1.26.6 github.com/aws/aws-sdk-go-v2/credentials v1.16.16 github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.16 @@ -195,8 +195,8 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.16 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 // indirect @@ -205,9 +205,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect - github.com/aws/smithy-go v1.20.0 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect diff --git a/go.sum b/go.sum index 16be51a05..ffb5e77fd 100644 --- a/go.sum +++ b/go.sum @@ -769,8 +769,8 @@ github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7Rfg github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.42.37/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= -github.com/aws/aws-sdk-go-v2 v1.25.0 h1:sv7+1JVJxOu/dD/sz/csHX7jFqmP001TIY7aytBWDSQ= -github.com/aws/aws-sdk-go-v2 v1.25.0/go.mod h1:G104G1Aho5WqF+SR3mDIobTABQzpYV0WxMsKxlMggOA= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 h1:2UO6/nT1lCZq1LqM67Oa4tdgP1CvL1sLSxvuD+VrOeE= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0/go.mod h1:5zGj2eA85ClyedTDK+Whsu+w9yimnVIZvhvBKrDquM8= github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= @@ -789,10 +789,10 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUt github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 h1:2MUXyGW6dVaQz6aqycpbdLIH1NMcUI6kW6vQ0RabGYg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15/go.mod h1:aHbhbR6WEQgHAiRj41EQ2W47yOYwNtIkWTXmcAtYqj8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 h1:NPs/EqVO+ajwOoq56EfcGKa3L3ruWuazkIw1BqxwOPw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0/go.mod h1:D+duLy2ylgatV+yTlQ8JTuLfDD0BnFvnQRc+o6tbZ4M= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0 h1:ks7KGMVUMoDzcxNWUlEdI+/lokMFD136EL6DWmUOV80= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0/go.mod h1:hL6BWM/d/qz113fVitZjbXR0E+RCTU1+x+1Idyn5NgE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= @@ -826,6 +826,8 @@ github.com/aws/aws-sdk-go-v2/service/lambda v1.50.0/go.mod h1:yEO3Ejj0qBhdIDlRYQ github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1 h1:5XNlsBsEvBZBMO6p82y+sqpWg8j5aBCe+5C2GBFgqBQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 h1:Rrqru2wYkKQCS2IM5/JrgKUQIoNTqA6y/iuxkjzxC6M= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2/go.mod h1:QuCURO98Sqee2AXmqDNxKXYFm2OEDAVAPApMqO0Vqnc= github.com/aws/aws-sdk-go-v2/service/sns v1.27.0 h1:Qa8B9/cgLWNt5zNogF81CuT+Nh+XkzW+hkfO784u1bs= github.com/aws/aws-sdk-go-v2/service/sns v1.27.0/go.mod h1:uaz2BGV8LQxQPlNmuUcqFS9Bf6n+OY3y8cNukcQSTRw= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99eAfm0dxXazGpLceU= @@ -839,8 +841,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BV github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.20.0 h1:6+kZsCXZwKxZS9RfISnPc4EXlHoyAkm2hPuM8X2BrrQ= -github.com/aws/smithy-go v1.20.0/go.mod h1:uo5RKksAl4PzhqaAbjd4rLgFoq5koTsQKYuGe7dklGc= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beanstalkd/go-beanstalk v0.2.0 h1:6UOJugnu47uNB2jJO/lxyDgeD1Yds7owYi1USELqexA= diff --git a/internal/impl/aws/integration_s3_test.go b/internal/impl/aws/integration_s3_test.go index 26b67a88d..d1747770d 100644 --- a/internal/impl/aws/integration_s3_test.go +++ b/internal/impl/aws/integration_s3_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck // Ignore SA1019 package aws import ( diff --git a/internal/impl/gcp/output_bigquery.go b/internal/impl/gcp/output_bigquery.go index 32dcc7ffd..8edf869d0 100644 --- a/internal/impl/gcp/output_bigquery.go +++ b/internal/impl/gcp/output_bigquery.go @@ -276,9 +276,6 @@ func newGCPBigQueryOutput( return g, nil } - // _, isStatic := g.conf.TableID.Static() - // if - var err error if g.fieldDelimiterBytes, err = convertToIso(g.fieldDelimiterBytes); err != nil { return nil, fmt.Errorf("error parsing csv.field_delimiter field: %w", err) diff --git a/internal/impl/sql/cache_sql.go b/internal/impl/sql/cache_sql.go index 8d81d0cdb..d46a9e4c3 100644 --- a/internal/impl/sql/cache_sql.go +++ b/internal/impl/sql/cache_sql.go @@ -12,6 +12,9 @@ import ( "github.com/Jeffail/shutdown" "github.com/warpstreamlabs/bento/public/service" + + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" ) const ( @@ -97,6 +100,8 @@ type sqlCache struct { upsertBuilder squirrel.InsertBuilder deleteBuilder squirrel.DeleteBuilder + awsConf aws.Config + logger *service.Logger shutSig *shutdown.Signaller } @@ -161,7 +166,12 @@ func newSQLCacheFromConfig(conf *service.ParsedConfig, mgr *service.Resources) ( return nil, err } - if s.db, err = sqlOpenWithReworks(context.Background(), s.logger, s.driver, s.dsn, connSettings.initVerifyConn); err != nil { + s.awsConf, err = bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + if s.db, err = sqlOpenWithReworks(context.Background(), s.logger, s.driver, s.dsn, connSettings, s.awsConf); err != nil { return nil, err } connSettings.apply(context.Background(), s.db, s.logger) diff --git a/internal/impl/sql/conn_fields.go b/internal/impl/sql/conn_fields.go index 4bfbb23c3..d638eec08 100644 --- a/internal/impl/sql/conn_fields.go +++ b/internal/impl/sql/conn_fields.go @@ -3,12 +3,17 @@ package sql import ( "context" "database/sql" + "encoding/json" + "errors" "fmt" "net/url" "strings" "sync" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/warpstreamlabs/bento/internal/impl/aws/config" "github.com/warpstreamlabs/bento/public/service" ) @@ -46,7 +51,8 @@ The ` + "[`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos)" + ` dri Example("oracle://foouser:foopass@localhost:1521/service_name") func connFields() []*service.ConfigField { - return []*service.ConfigField{ + + connFields := []*service.ConfigField{ service.NewStringListField("init_files"). Description(` An optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star). @@ -102,7 +108,16 @@ CREATE TABLE IF NOT EXISTS some_table ( Description("An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value <= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)."). Optional(). Advanced(), + service.NewStringField("secret_name"). + Description("An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres."). + Optional(). + Advanced(), } + + connFields = append(connFields, config.SessionFields()...) + + return connFields + } func rawQueryField() *service.ConfigField { @@ -134,6 +149,7 @@ type connSettings struct { initFileStatements [][2]string // (path,statement) initStatement string initVerifyConn bool + secretName string } func (c *connSettings) apply(ctx context.Context, db *sql.DB, log *service.Logger) { @@ -222,6 +238,12 @@ func connSettingsFromParsed( } } + if conf.Contains("secret_name") { + if c.secretName, err = conf.FieldString("secret_name"); err != nil { + return + } + } + return } @@ -257,7 +279,67 @@ func reworkDSN(driver, dsn string) (string, error) { return dsn, nil } -func sqlOpenWithReworks(ctx context.Context, logger *service.Logger, driver, dsn string, shouldPing bool) (*sql.DB, error) { +func getSecretFromAWSSecretManager(secretName string, awsConf aws.Config) (secretString string, err error) { + svc := secretsmanager.NewFromConfig(awsConf) + + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretName), + } + result, err := svc.GetSecretValue(context.TODO(), input) + if err != nil { + return "", err + } + + return *result.SecretString, nil +} + +func BuildAwsDsn(dsn string, driver string, secretName string, awsConf aws.Config, getSecretFunc func(secretName string, awsConf aws.Config) (awsSecret string, err error)) (awsSecretDsn string, err error) { + if secretName != "" && driver == "postgres" { + + parsedDSN, err := url.Parse(dsn) + if err != nil { + return "", fmt.Errorf("error parsing DSN URL: %w", err) + } + + username := parsedDSN.User.Username() + password, _ := parsedDSN.User.Password() + host := parsedDSN.Hostname() + port := parsedDSN.Port() + path := parsedDSN.Path + rawQuery := parsedDSN.RawQuery + + secretString, err := getSecretFunc(secretName, awsConf) + if err != nil { + return "", fmt.Errorf("error retrieving secret: %w", err) + } + + var secrets map[string]interface{} + if err := json.Unmarshal([]byte(secretString), &secrets); err != nil { + return "", fmt.Errorf("error unmarshalling secret: %w", err) + } + + if val, ok := secrets["username"].(string); ok && val != "" { + username = val + } + if val, ok := secrets["password"].(string); ok && val != "" { + password = val + } + + newDSN := fmt.Sprintf("postgres://%s:%s@%s:%s%s", url.QueryEscape(username), url.QueryEscape(password), host, port, path) + if rawQuery != "" { + newDSN = fmt.Sprintf("%s?%s", newDSN, rawQuery) + } + + return newDSN, nil + + } else if secretName != "" && driver != "postgres" { + return "", errors.New("secret_name with DSN info currently only works for postgres DSNs") + } + + return dsn, nil +} + +func sqlOpenWithReworks(ctx context.Context, logger *service.Logger, driver, dsn string, connSettings *connSettings, awsConf aws.Config) (*sql.DB, error) { updatedDSN, err := reworkDSN(driver, dsn) if err != nil { return nil, err @@ -267,12 +349,21 @@ func sqlOpenWithReworks(ctx context.Context, logger *service.Logger, driver, dsn logger.Warnf("Detected old-style Clickhouse Data Source Name: '%v', replacing with new style: '%v'", dsn, updatedDSN) } + updatedDSN, err = BuildAwsDsn(dsn, driver, connSettings.secretName, awsConf, getSecretFromAWSSecretManager) + if err != nil { + return nil, err + } + + if updatedDSN != dsn { + logger.Infof("Updated dsn with info from AWS Secret '%v'", connSettings.secretName) + } + db, err := sql.Open(driver, updatedDSN) if err != nil { return nil, err } - if shouldPing { + if connSettings.initVerifyConn { if err := db.PingContext(ctx); err != nil { _ = db.Close() return nil, fmt.Errorf("could not establish connection to database: %w", err) diff --git a/internal/impl/sql/conn_fields_test.go b/internal/impl/sql/conn_fields_test.go index 4253bc3ad..04680844f 100644 --- a/internal/impl/sql/conn_fields_test.go +++ b/internal/impl/sql/conn_fields_test.go @@ -2,15 +2,19 @@ package sql_test import ( "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + bento_sql "github.com/warpstreamlabs/bento/internal/impl/sql" "github.com/warpstreamlabs/bento/public/service" _ "github.com/warpstreamlabs/bento/public/components/pure" @@ -184,3 +188,95 @@ sql_select: `{"bar":"third bar","baz":"third baz","foo":"third"}`, }, msgs) } + +func mockGetSecretFromAWS(secretName string, awsConf aws.Config) (secretString string, err error) { + var secret map[string]interface{} + if secretName == "validFullSecret" { + secret = map[string]interface{}{ + "username": "testUser", + "password": "testPassword", + "host": "testHost", + "port": 5432, + "dbName": "testDB", + } + } else if secretName == "validUserPassSecret" { + secret = map[string]interface{}{ + "username": "testUser", + "password": "testPassword", + } + } else if secretName == "SecretDoesNotExist" { + return "", errors.New("ResourceNotFoundException: Secrets Manager can't find the specified secret.") + } + secretBytes, _ := json.Marshal(secret) + return string(secretBytes), nil +} + +func TestBuildAwsDsn(t *testing.T) { + awsConf := aws.Config{} + + tests := []struct { + name string + dsn string + driver string + secretName string + expectedDSN string + expectedError bool + errorValue string + }{ + { + name: "validFullSecretTest", + dsn: "postgres://user:password@host:5432/dbname?param1=value1¶m2=value2", + driver: "postgres", + secretName: "validFullSecret", + expectedDSN: "postgres://testUser:testPassword@host:5432/dbname?param1=value1¶m2=value2", + expectedError: false, + }, + { + name: "validUserPassSecretTest", + dsn: "postgres://user:password@host:5432/dbname?param1=value1¶m2=value2", + driver: "postgres", + secretName: "validUserPassSecret", + expectedDSN: "postgres://testUser:testPassword@host:5432/dbname?param1=value1¶m2=value2", + expectedError: false, + }, + { + name: "SecretNotFoundTest", + dsn: "postgres://user:password@host:5432/dbname?param1=value1¶m2=value2", + driver: "postgres", + secretName: "SecretDoesNotExist", + expectedDSN: "postgres://testUser:testPassword@host:5432/dbname?param1=value1¶m2=value2", + expectedError: true, + errorValue: "error retrieving secret: ResourceNotFoundException: Secrets Manager can't find the specified secret.", + }, + { + name: "DriverNotPostgresTest", + dsn: "mysql://root@localhost/username", + driver: "mysql", + secretName: "validFullSecret", + expectedDSN: "", + expectedError: true, + errorValue: "secret_name with DSN info currently only works for postgres DSNs", + }, + { + name: "NoSecretName", + dsn: "postgres://user:password@host:5432/dbname?param1=value1¶m2=value2", + driver: "postgres", + secretName: "", + expectedDSN: "postgres://user:password@host:5432/dbname?param1=value1¶m2=value2", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + awsSecretDsn, err := bento_sql.BuildAwsDsn(tt.dsn, tt.driver, tt.secretName, awsConf, mockGetSecretFromAWS) + if tt.expectedError { + assert.Error(t, err) + assert.Equal(t, tt.errorValue, err.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedDSN, awsSecretDsn) + } + }) + } +} diff --git a/internal/impl/sql/input_sql_raw.go b/internal/impl/sql/input_sql_raw.go index ac75dd924..176f18826 100644 --- a/internal/impl/sql/input_sql_raw.go +++ b/internal/impl/sql/input_sql_raw.go @@ -8,6 +8,9 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" + "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -82,6 +85,7 @@ type sqlRawInput struct { argsMapping *bloblang.Executor connSettings *connSettings + awsConf aws.Config logger *service.Logger shutSig *shutdown.Signaller @@ -120,6 +124,11 @@ func newSQLRawInputFromConfig(conf *service.ParsedConfig, mgr *service.Resources if s.connSettings, err = connSettingsFromParsed(conf, mgr); err != nil { return nil, err } + s.awsConf, err = bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + return s, nil } @@ -132,7 +141,7 @@ func (s *sqlRawInput) Connect(ctx context.Context) (err error) { } var db *sql.DB - if db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings.initVerifyConn); err != nil { + if db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings, s.awsConf); err != nil { return err } defer func() { diff --git a/internal/impl/sql/input_sql_select.go b/internal/impl/sql/input_sql_select.go index bb6eb2ec0..f777372ef 100644 --- a/internal/impl/sql/input_sql_select.go +++ b/internal/impl/sql/input_sql_select.go @@ -10,6 +10,9 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" + "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -103,6 +106,7 @@ type sqlSelectInput struct { argsMapping *bloblang.Executor connSettings *connSettings + awsConf aws.Config logger *service.Logger shutSig *shutdown.Signaller @@ -172,6 +176,11 @@ func newSQLSelectInputFromConfig(conf *service.ParsedConfig, mgr *service.Resour if s.connSettings, err = connSettingsFromParsed(conf, mgr); err != nil { return nil, err } + + s.awsConf, err = bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } return s, nil } @@ -184,7 +193,7 @@ func (s *sqlSelectInput) Connect(ctx context.Context) (err error) { } var db *sql.DB - if db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings.initVerifyConn); err != nil { + if db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings, s.awsConf); err != nil { return } defer func() { diff --git a/internal/impl/sql/output_sql_deprecated.go b/internal/impl/sql/output_sql_deprecated.go index 8847ecc9f..878918245 100644 --- a/internal/impl/sql/output_sql_deprecated.go +++ b/internal/impl/sql/output_sql_deprecated.go @@ -1,8 +1,12 @@ package sql import ( + "context" + "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" + + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" ) func sqlDeprecatedOutputConfig() *service.ConfigSpec { @@ -77,5 +81,11 @@ func newSQLDeprecatedOutputFromConfig(conf *service.ParsedConfig, mgr *service.R if err != nil { return nil, err } - return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, nil, argsMapping, connSettings), nil + + awsConf, err := bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, nil, argsMapping, connSettings, awsConf), nil } diff --git a/internal/impl/sql/output_sql_insert.go b/internal/impl/sql/output_sql_insert.go index dc10a0ee0..757709beb 100644 --- a/internal/impl/sql/output_sql_insert.go +++ b/internal/impl/sql/output_sql_insert.go @@ -10,6 +10,8 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -103,6 +105,7 @@ type sqlInsertOutput struct { argsMapping *bloblang.Executor connSettings *connSettings + awsConf aws.Config logger *service.Logger shutSig *shutdown.Signaller @@ -180,6 +183,12 @@ func newSQLInsertOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resou if s.connSettings, err = connSettingsFromParsed(conf, mgr); err != nil { return nil, err } + + s.awsConf, err = bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + return s, nil } @@ -192,7 +201,7 @@ func (s *sqlInsertOutput) Connect(ctx context.Context) error { } var err error - if s.db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings.initVerifyConn); err != nil { + if s.db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings, s.awsConf); err != nil { return err } diff --git a/internal/impl/sql/output_sql_raw.go b/internal/impl/sql/output_sql_raw.go index 934249c4b..1a46ba2b1 100644 --- a/internal/impl/sql/output_sql_raw.go +++ b/internal/impl/sql/output_sql_raw.go @@ -8,6 +8,8 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -93,6 +95,7 @@ type sqlRawOutput struct { argsMapping *bloblang.Executor connSettings *connSettings + awsConf aws.Config logger *service.Logger shutSig *shutdown.Signaller @@ -134,7 +137,13 @@ func newSQLRawOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resource if err != nil { return nil, err } - return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, argsMapping, connSettings), nil + + awsConf, err := bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, argsMapping, connSettings, awsConf), nil } func newSQLRawOutput( @@ -144,6 +153,7 @@ func newSQLRawOutput( queryDyn *service.InterpolatedString, argsMapping *bloblang.Executor, connSettings *connSettings, + awsConf aws.Config, ) *sqlRawOutput { return &sqlRawOutput{ logger: logger, @@ -154,6 +164,7 @@ func newSQLRawOutput( queryDyn: queryDyn, argsMapping: argsMapping, connSettings: connSettings, + awsConf: awsConf, } } @@ -166,7 +177,7 @@ func (s *sqlRawOutput) Connect(ctx context.Context) error { } var err error - if s.db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings.initVerifyConn); err != nil { + if s.db, err = sqlOpenWithReworks(ctx, s.logger, s.driver, s.dsn, s.connSettings, s.awsConf); err != nil { return err } diff --git a/internal/impl/sql/processor_sql_deprecated.go b/internal/impl/sql/processor_sql_deprecated.go index b64db28f4..9a9f6a71e 100644 --- a/internal/impl/sql/processor_sql_deprecated.go +++ b/internal/impl/sql/processor_sql_deprecated.go @@ -1,6 +1,9 @@ package sql import ( + "context" + + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -92,5 +95,11 @@ func NewSQLDeprecatedProcessorFromConfig(conf *service.ParsedConfig, mgr *servic if err != nil { return nil, err } - return newSQLRawProcessor(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, onlyExec, argsMapping, connSettings) + + awsConf, err := bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + return newSQLRawProcessor(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, onlyExec, argsMapping, connSettings, awsConf) } diff --git a/internal/impl/sql/processor_sql_insert.go b/internal/impl/sql/processor_sql_insert.go index ece94b1b9..6261a95ef 100644 --- a/internal/impl/sql/processor_sql_insert.go +++ b/internal/impl/sql/processor_sql_insert.go @@ -10,6 +10,8 @@ import ( "github.com/Jeffail/shutdown" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" + "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -171,7 +173,12 @@ func NewSQLInsertProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Re return nil, err } - if s.db, err = sqlOpenWithReworks(context.Background(), mgr.Logger(), driverStr, dsnStr, connSettings.initVerifyConn); err != nil { + awsConf, err := bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + if s.db, err = sqlOpenWithReworks(context.Background(), mgr.Logger(), driverStr, dsnStr, connSettings, awsConf); err != nil { return nil, err } diff --git a/internal/impl/sql/processor_sql_raw.go b/internal/impl/sql/processor_sql_raw.go index ef30a9402..2b1022a02 100644 --- a/internal/impl/sql/processor_sql_raw.go +++ b/internal/impl/sql/processor_sql_raw.go @@ -8,6 +8,9 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" + "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -99,6 +102,8 @@ type sqlRawProcessor struct { argsMapping *bloblang.Executor + awsConf aws.Config + logger *service.Logger shutSig *shutdown.Signaller } @@ -145,7 +150,13 @@ func NewSQLRawProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Resou if err != nil { return nil, err } - return newSQLRawProcessor(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, onlyExec, argsMapping, connSettings) + + awsConf, err := bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + return newSQLRawProcessor(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, onlyExec, argsMapping, connSettings, awsConf) } func newSQLRawProcessor( @@ -156,6 +167,7 @@ func newSQLRawProcessor( onlyExec bool, argsMapping *bloblang.Executor, connSettings *connSettings, + awsConf aws.Config, ) (*sqlRawProcessor, error) { s := &sqlRawProcessor{ logger: logger, @@ -164,10 +176,11 @@ func newSQLRawProcessor( queryDyn: queryDyn, onlyExec: onlyExec, argsMapping: argsMapping, + awsConf: awsConf, } var err error - if s.db, err = sqlOpenWithReworks(context.Background(), logger, driverStr, dsnStr, connSettings.initVerifyConn); err != nil { + if s.db, err = sqlOpenWithReworks(context.Background(), logger, driverStr, dsnStr, connSettings, awsConf); err != nil { return nil, err } connSettings.apply(context.Background(), s.db, s.logger) diff --git a/internal/impl/sql/processor_sql_select.go b/internal/impl/sql/processor_sql_select.go index c06363f14..2f43b07ab 100644 --- a/internal/impl/sql/processor_sql_select.go +++ b/internal/impl/sql/processor_sql_select.go @@ -10,6 +10,8 @@ import ( "github.com/Jeffail/shutdown" + "github.com/aws/aws-sdk-go-v2/aws" + bento_aws "github.com/warpstreamlabs/bento/internal/impl/aws" "github.com/warpstreamlabs/bento/public/bloblang" "github.com/warpstreamlabs/bento/public/service" ) @@ -100,6 +102,8 @@ type sqlSelectProcessor struct { where string argsMapping *bloblang.Executor + awsConf aws.Config + logger *service.Logger shutSig *shutdown.Signaller } @@ -171,7 +175,12 @@ func NewSQLSelectProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Re return nil, err } - if s.db, err = sqlOpenWithReworks(context.Background(), mgr.Logger(), driverStr, dsnStr, connSettings.initVerifyConn); err != nil { + s.awsConf, err = bento_aws.GetSession(context.Background(), conf) + if err != nil { + return nil, err + } + + if s.db, err = sqlOpenWithReworks(context.Background(), mgr.Logger(), driverStr, dsnStr, connSettings, s.awsConf); err != nil { return nil, err } connSettings.apply(context.Background(), s.db, s.logger) diff --git a/website/cookbooks/redshift.md b/website/cookbooks/redshift.md new file mode 100644 index 000000000..54b4e4fd7 --- /dev/null +++ b/website/cookbooks/redshift.md @@ -0,0 +1,230 @@ +--- +slug: redshift +title: Redshift +description: Hook up Bento with a Redshift Cluster +--- + +This cookbook demonstrates how to connect Bento to an Amazon Redshift cluster. + +There is an example a terraform configuration at the bottom of this page. + +_______________________ + +So you are going to need: + - A Redshift Cluster + - A Secret in SecretManager, with username + password keys as the secret value + - Some AWS Access Keys Bento can use to get the Secret + + +``` +redshift_endpoint = "my-redshift-cluster.[...].eu-west-1.redshift.amazonaws.com:5439" +redshift_master_password_secret_name = "redshift-password" +``` + +_______________________ + +## Bento Config + +You will need to create an AWS Access Key that can retrieve the aforementioned secret, in the below example we have used `credentials` in the config, but you can configure this a number of ways, to find out what options you have you can [read the guide][credentials]. + +```yaml +output: + sql_insert: + driver: postgres + dsn: postgresql://username_from_secret:password_from_secret@"$REDSHIFT_ENDPOINT"/dev + table: test + columns: [age, name] + args_mapping: | + root = [ + this.age, + this.name, + ] + init_statement: | + CREATE TABLE test (name varchar(255), age int); + secret_name: "$REDSHIFT_SECRET_NAME" + region: eu-west-1 + credentials: + id: "$AWS_ACCESS_KEY_ID" + secret: "$AWS_SECRET_ACCESS_KEY" +``` + + - `driver`: Redshift is compatible with PostgreSQL, so we set the driver field to `postgres` + - `dsn`: We configure this to connect to Redshift, but the username + password we have them set to `username_from_secret` & `password_from_secret`, Bento will fetch the secret and update the DSN to use them. Bento will expect that the secret will have the keys 'username' & 'password'. Below is a standalone terraform module that will create a redshift cluster and store the username + password in a secret. Note: The username + password will be overwritten after we have the value from Secret Manager, but we still need to provide a DSN with a valid "shape" so it can be parsed correctly. + - `secret_name`: Here we give the secret_name of the secret Bento will use to update the DSN. + - `credentials`: This is one way to give the Bento Config access to the Secret in AWS. There are other ways explained [here][credentials]. + +The other fields are can be looked up on the [sql_insert][credentials] page. The above config makes use of [environment variable interpolation][env_var_interpolation]. + +The secret should be stored in the AWS secret manager with the name you specified in the configuration above. The value of the secret should be a JSON documentation with a username/password: + +```json +{ + "username": "some_username", + "password": "some_password" +} +``` + +Below is a sample standalone terraform module that creates a redshift cluster, username/password, and stores the username/password in AWS Secret manager in the right format for Bento to read it. + +_______________________ +## terraform +```terraform +provider "aws" { + region = "eu-west-1" # Change to your desired AWS region +} + +resource "random_password" "password" { + length = 16 + special = true + override_special = "!$%&*()-_=+[]{}<>:?" +} + +resource "aws_vpc" "redshift_vpc" { + cidr_block = "10.0.0.0/16" + instance_tenancy = "default" + tags = { + Name = "redshift-vpc" + } +} + +resource "aws_internet_gateway" "redshift_vpc_gw" { + vpc_id = aws_vpc.redshift_vpc.id + depends_on = [aws_vpc.redshift_vpc] +} + +resource "aws_security_group" "redshift_sg" { + vpc_id = aws_vpc.redshift_vpc.id + + ingress { + from_port = 5439 + to_port = 5439 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # Change to your specific IP range + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "redshift-sg" + } + + depends_on = [aws_vpc.redshift_vpc] +} + +resource "aws_route_table" "redshift_route_table" { + vpc_id = aws_vpc.redshift_vpc.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.redshift_vpc_gw.id + } + + tags = { + Name = "redshift-route-table" + } +} + +resource "aws_subnet" "redshift_subnet_1" { + vpc_id = aws_vpc.redshift_vpc.id + cidr_block = "10.0.1.0/24" + availability_zone = "eu-west-1a" + map_public_ip_on_launch = "true" + + tags = { + Name = "redshift-subnet-1" + } + + depends_on = [aws_vpc.redshift_vpc] +} + +resource "aws_subnet" "redshift_subnet_2" { + vpc_id = aws_vpc.redshift_vpc.id + cidr_block = "10.0.2.0/24" + availability_zone = "eu-west-1b" + map_public_ip_on_launch = "true" + + tags = { + Name = "redshift-subnet-2" + } + + depends_on = [aws_vpc.redshift_vpc] +} + +resource "aws_route_table_association" "redshift_subnet_1_association" { + subnet_id = aws_subnet.redshift_subnet_1.id + route_table_id = aws_route_table.redshift_route_table.id +} + +resource "aws_route_table_association" "redshift_subnet_2_association" { + subnet_id = aws_subnet.redshift_subnet_2.id + route_table_id = aws_route_table.redshift_route_table.id +} + +resource "aws_redshift_subnet_group" "redshift_subnet_group" { + name = "redshift-subnet-group" + subnet_ids = [aws_subnet.redshift_subnet_1.id, aws_subnet.redshift_subnet_2.id] + + tags = { + Name = "redshift-subnet-group" + } + +} + +resource "aws_secretsmanager_secret" "redshift_master_password" { + name = "redshift-password" +} + +resource "aws_secretsmanager_secret_version" "redshift_master_password_version" { + secret_id = aws_secretsmanager_secret.redshift_master_password.id + secret_string = jsonencode({ + username = "admin" + password = random_password.password.result + }) +} + +resource "aws_redshift_cluster" "my_redshift" { + cluster_identifier = "my-redshift-cluster" + database_name = "dev" + + # Use the Secrets Manager secret to get the username and password + master_username = jsondecode(aws_secretsmanager_secret_version.redshift_master_password_version.secret_string)["username"] + master_password = jsondecode(aws_secretsmanager_secret_version.redshift_master_password_version.secret_string)["password"] + + node_type = "dc2.large" + cluster_type = "single-node" + number_of_nodes = 1 + cluster_subnet_group_name = aws_redshift_subnet_group.redshift_subnet_group.id + port = 5439 + publicly_accessible = true + vpc_security_group_ids = [aws_security_group.redshift_sg.id] + skip_final_snapshot = true + + tags = { + Name = "MyRedshiftCluster" + } + + depends_on = [ + aws_vpc.redshift_vpc, + aws_security_group.redshift_sg, + aws_redshift_subnet_group.redshift_subnet_group, + ] +} + +output "redshift_endpoint" { + value = aws_redshift_cluster.my_redshift.endpoint +} + +output "redshift_master_password_secret_name" { + value = aws_secretsmanager_secret.redshift_master_password.name +} +``` + +[terraform]: /docs/cookbooks/redshift#terraform +[credentials]: /docs/guides/cloud/aws +[sql_insert]: /docs/components/outputs/sql_insert +[env_var_interpolation]: /docs/configuration/interpolation \ No newline at end of file diff --git a/website/docs/components/caches/sql.md b/website/docs/components/caches/sql.md index 6701580c4..f2d3e02e7 100644 --- a/website/docs/components/caches/sql.md +++ b/website/docs/components/caches/sql.md @@ -68,6 +68,17 @@ sql: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -289,4 +300,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + diff --git a/website/docs/components/inputs/sql_raw.md b/website/docs/components/inputs/sql_raw.md index 556afa1cd..042196c74 100644 --- a/website/docs/components/inputs/sql_raw.md +++ b/website/docs/components/inputs/sql_raw.md @@ -68,6 +68,17 @@ input: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -287,4 +298,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + diff --git a/website/docs/components/inputs/sql_select.md b/website/docs/components/inputs/sql_select.md index 673001310..8b7e94078 100644 --- a/website/docs/components/inputs/sql_select.md +++ b/website/docs/components/inputs/sql_select.md @@ -74,6 +74,17 @@ input: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -328,4 +339,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + diff --git a/website/docs/components/outputs/sql_insert.md b/website/docs/components/outputs/sql_insert.md index c1672a8af..bda65e05f 100644 --- a/website/docs/components/outputs/sql_insert.md +++ b/website/docs/components/outputs/sql_insert.md @@ -74,6 +74,17 @@ output: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" batching: count: 0 byte_size: 0 @@ -323,6 +334,96 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + ### `batching` Allows you to configure a [batching policy](/docs/configuration/batching). diff --git a/website/docs/components/outputs/sql_raw.md b/website/docs/components/outputs/sql_raw.md index fea1b974d..06bfa0777 100644 --- a/website/docs/components/outputs/sql_raw.md +++ b/website/docs/components/outputs/sql_raw.md @@ -71,6 +71,17 @@ output: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" batching: count: 0 byte_size: 0 @@ -304,6 +315,96 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + ### `batching` Allows you to configure a [batching policy](/docs/configuration/batching). diff --git a/website/docs/components/processors/sql_insert.md b/website/docs/components/processors/sql_insert.md index b5a93048e..60ce40a2a 100644 --- a/website/docs/components/processors/sql_insert.md +++ b/website/docs/components/processors/sql_insert.md @@ -65,6 +65,17 @@ sql_insert: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -303,4 +314,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + diff --git a/website/docs/components/processors/sql_raw.md b/website/docs/components/processors/sql_raw.md index 89e764e8f..b645a0b4b 100644 --- a/website/docs/components/processors/sql_raw.md +++ b/website/docs/components/processors/sql_raw.md @@ -64,6 +64,17 @@ sql_raw: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -310,4 +321,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` + diff --git a/website/docs/components/processors/sql_select.md b/website/docs/components/processors/sql_select.md index dfb34655e..3d2469c89 100644 --- a/website/docs/components/processors/sql_select.md +++ b/website/docs/components/processors/sql_select.md @@ -67,6 +67,17 @@ sql_select: conn_max_life_time: "" # No default (optional) conn_max_idle: 2 conn_max_open: 0 # No default (optional) + secret_name: "" # No default (optional) + region: "" + endpoint: "" + credentials: + profile: "" + id: "" + secret: "" + token: "" + from_ec2_role: false + role: "" + role_external_id: "" ``` @@ -319,4 +330,94 @@ An optional maximum number of open connections to the database. If conn_max_idle Type: `int` +### `secret_name` + +An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to postgres. + + +Type: `string` + +### `region` + +The AWS region to target. + + +Type: `string` +Default: `""` + +### `endpoint` + +Allows you to specify a custom endpoint for the AWS API. + + +Type: `string` +Default: `""` + +### `credentials` + +Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws). + + +Type: `object` + +### `credentials.profile` + +A profile from `~/.aws/credentials` to use. + + +Type: `string` +Default: `""` + +### `credentials.id` + +The ID of credentials to use. + + +Type: `string` +Default: `""` + +### `credentials.secret` + +The secret for the credentials being used. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `credentials.token` + +The token for the credentials being used, required when using short term credentials. + + +Type: `string` +Default: `""` + +### `credentials.from_ec2_role` + +Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). + + +Type: `bool` +Default: `false` +Requires version 1.0.0 or newer + +### `credentials.role` + +A role ARN to assume. + + +Type: `string` +Default: `""` + +### `credentials.role_external_id` + +An external ID to provide when assuming a role. + + +Type: `string` +Default: `""` +