From fbbb2e2fda10bfe1e665f308930e4417ce7088fe Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Tue, 18 Jan 2022 08:13:13 +0100 Subject: [PATCH] config: Add prefix support and test coverage --- backend/config/config.go | 59 +++++--- backend/config/config_test.go | 278 ++++++++++++++++++++++++++++++++++ backend/go.mod | 1 - 3 files changed, 314 insertions(+), 24 deletions(-) create mode 100644 backend/config/config_test.go diff --git a/backend/config/config.go b/backend/config/config.go index 43e43ae..ded49fa 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -20,10 +20,12 @@ const ( type FileStore int const ( - FileSystemStore = iota + FileSystemStore FileStore = iota S3Store ) +const DefaultBindAddr = "localhost:8888" + type Config struct { Environment Environment BindAddr string @@ -41,54 +43,64 @@ type Config struct { FFmpegWorkerPoolSize int } +// TODO: update deployment and add prefix. +var Prefix = "" + +func envPrefix(k string) string { return Prefix + k } +func getenvPrefix(k string) string { return os.Getenv(Prefix + k) } + func NewFromEnv() (Config, error) { - envString := os.Getenv("ENV") + envVarName := envPrefix("ENV") + envString := os.Getenv(envVarName) var env Environment switch envString { case "production": env = EnvProduction - case "development": + case "development", "": env = EnvDevelopment - case "": - return Config{}, errors.New("ENV not set") default: - return Config{}, fmt.Errorf("invalid ENV value: %s", envString) + return Config{}, fmt.Errorf("invalid %s value: %s", envVarName, envString) } - bindAddr := os.Getenv("BIND_ADDR") + bindAddr := getenvPrefix("BIND_ADDR") if bindAddr == "" { - bindAddr = "localhost:8888" + bindAddr = DefaultBindAddr } - tlsCertFile := os.Getenv("TLS_CERT_FILE") - tlsKeyFile := os.Getenv("TLS_KEY_FILE") + tlsCertFileName := envPrefix("TLS_CERT_FILE") + tlsKeyFileName := envPrefix("TLS_KEY_FILE") + tlsCertFile := os.Getenv(tlsCertFileName) + tlsKeyFile := os.Getenv(tlsKeyFileName) if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") { - return Config{}, errors.New("both TLS_CERT_FILE and TLS_KEY_FILE must be set") + return Config{}, fmt.Errorf("both %s and %s must be set", tlsCertFileName, tlsKeyFileName) } - databaseURL := os.Getenv("DATABASE_URL") + databaseURLName := envPrefix("DATABASE_URL") + databaseURL := os.Getenv(databaseURLName) if databaseURL == "" { - return Config{}, errors.New("DATABASE_URL not set") + return Config{}, fmt.Errorf("%s not set", databaseURLName) } - fileStoreString := os.Getenv("FILE_STORE") + fileStoreName := envPrefix("FILE_STORE") + fileStoreString := os.Getenv(fileStoreName) var fileStore FileStore - switch os.Getenv("FILE_STORE") { + switch getenvPrefix("FILE_STORE") { case "s3": fileStore = S3Store case "filesystem", "": fileStore = FileSystemStore default: - return Config{}, fmt.Errorf("invalid FILE_STORE value: %s", fileStoreString) + return Config{}, fmt.Errorf("invalid %s value: %s", fileStoreName, fileStoreString) } - fileStoreHTTPBaseURLString := os.Getenv("FILE_STORE_HTTP_BASE_URL") + fileStoreHTTPBaseURLName := envPrefix("FILE_STORE_HTTP_BASE_URL") + fileStoreHTTPBaseURLString := os.Getenv(fileStoreHTTPBaseURLName) if !strings.HasSuffix(fileStoreHTTPBaseURLString, "/") { fileStoreHTTPBaseURLString += "/" } fileStoreHTTPBaseURL, err := url.Parse(fileStoreHTTPBaseURLString) if err != nil { - return Config{}, fmt.Errorf("invalid FILE_STORE_HTTP_BASE_URL: %v", err) + return Config{}, fmt.Errorf("invalid %s: %v", fileStoreHTTPBaseURLName, fileStoreHTTPBaseURLString) } var awsAccessKeyID, awsSecretAccessKey, awsRegion, s3Bucket, fileStoreHTTPRoot string @@ -108,22 +120,23 @@ func NewFromEnv() (Config, error) { return Config{}, errors.New("AWS_REGION not set") } - s3Bucket = os.Getenv("S3_BUCKET") + s3Bucket = getenvPrefix("S3_BUCKET") if s3Bucket == "" { return Config{}, errors.New("S3_BUCKET not set") } } else { - if fileStoreHTTPRoot = os.Getenv("FILE_STORE_HTTP_ROOT"); fileStoreHTTPRoot == "" { + if fileStoreHTTPRoot = getenvPrefix("FILE_STORE_HTTP_ROOT"); fileStoreHTTPRoot == "" { return Config{}, errors.New("FILE_STORE_HTTP_ROOT not set") } } - assetsHTTPRoot := os.Getenv("ASSETS_HTTP_ROOT") + assetsHTTPRoot := getenvPrefix("ASSETS_HTTP_ROOT") ffmpegWorkerPoolSize := runtime.NumCPU() - if s := os.Getenv("FFMPEG_WORKER_POOL_SIZE"); s != "" { + ffmpegWorkerPoolSizeName := envPrefix("FFMPEG_WORKER_POOL_SIZE") + if s := os.Getenv(ffmpegWorkerPoolSizeName); s != "" { if n, err := strconv.Atoi(s); err != nil { - return Config{}, fmt.Errorf("invalid FFMPEG_WORKER_POOL_SIZE value: %s", s) + return Config{}, fmt.Errorf("invalid %s value: %s", ffmpegWorkerPoolSizeName, s) } else { ffmpegWorkerPoolSize = n } diff --git a/backend/config/config_test.go b/backend/config/config_test.go new file mode 100644 index 0000000..c0dbd5e --- /dev/null +++ b/backend/config/config_test.go @@ -0,0 +1,278 @@ +package config_test + +import ( + "net/url" + "os" + "runtime" + "strings" + "testing" + + "git.netflux.io/rob/clipper/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + config.Prefix = "CLIPPER_" +} + +type env map[string]string + +// clearenv clears prefixed keys from the environment. Currently it does not +// clear AWS_* environment variables. +func clearenv() { + for _, kv := range os.Environ() { + split := strings.SplitN(kv, "=", 2) + k := split[0] + if !strings.HasPrefix(k, config.Prefix) { + continue + } + os.Unsetenv(k) + } +} + +// setupenv sets up a valid environment, including AWS_* configuration +// variables. +func setupenv() { + e := env{ + "CLIPPER_ENV": "development", + "CLIPPER_DATABASE_URL": "postgresql://localhost:5432/db", + "CLIPPER_FILE_STORE_HTTP_ROOT": "/data", + "CLIPPER_S3_BUCKET": "bucket", + "AWS_ACCESS_KEY_ID": "key", + "AWS_SECRET_ACCESS_KEY": "secret", + "AWS_REGION": "eu-west-1", + } + for k, v := range e { + os.Setenv(k, v) + } +} + +func mustParseURL(t *testing.T, u string) *url.URL { + pu, err := url.Parse(u) + require.NoError(t, err) + return pu +} + +func TestNewFromEnv(t *testing.T) { + t.Run("ENV", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Setenv("CLIPPER_ENV", "foo") + c, err := config.NewFromEnv() + assert.EqualError(t, err, "invalid CLIPPER_ENV value: foo") + + os.Setenv("CLIPPER_ENV", "") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.EnvDevelopment, c.Environment) + + os.Setenv("CLIPPER_ENV", "development") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.EnvDevelopment, c.Environment) + + os.Setenv("CLIPPER_ENV", "production") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.EnvProduction, c.Environment) + }) + + t.Run("BIND_ADDR", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Setenv("CLIPPER_BIND_ADDR", "") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.DefaultBindAddr, c.BindAddr) + + os.Setenv("CLIPPER_BIND_ADDR", "example.com:1234") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "example.com:1234", c.BindAddr) + }) + + t.Run("TLS_CERT_FILE and TLS_KEY_FILE", func(t *testing.T) { + defer clearenv() + setupenv() + + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "", c.TLSCertFile) + assert.Equal(t, "", c.TLSKeyFile) + + const expErr = "both CLIPPER_TLS_CERT_FILE and CLIPPER_TLS_KEY_FILE must be set" + os.Setenv("CLIPPER_TLS_CERT_FILE", "foo") + os.Setenv("CLIPPER_TLS_KEY_FILE", "") + c, err = config.NewFromEnv() + assert.EqualError(t, err, expErr) + + os.Setenv("CLIPPER_TLS_CERT_FILE", "") + os.Setenv("CLIPPER_TLS_KEY_FILE", "bar") + c, err = config.NewFromEnv() + assert.EqualError(t, err, expErr) + + os.Setenv("CLIPPER_TLS_CERT_FILE", "foo") + os.Setenv("CLIPPER_TLS_KEY_FILE", "bar") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "foo", c.TLSCertFile) + assert.Equal(t, "bar", c.TLSKeyFile) + }) + + t.Run("DATABASE_URL", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Unsetenv("CLIPPER_DATABASE_URL") + _, err := config.NewFromEnv() + assert.EqualError(t, err, "CLIPPER_DATABASE_URL not set") + + os.Setenv("CLIPPER_DATABASE_URL", "foo") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "foo", c.DatabaseURL) + }) + + t.Run("FILE_STORE", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Unsetenv("CLIPPER_FILE_STORE") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.FileSystemStore, c.FileStore) + + os.Setenv("CLIPPER_FILE_STORE", "foo") + c, err = config.NewFromEnv() + assert.EqualError(t, err, "invalid CLIPPER_FILE_STORE value: foo") + + os.Setenv("CLIPPER_FILE_STORE", "filesystem") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.FileSystemStore, c.FileStore) + + os.Setenv("CLIPPER_FILE_STORE", "s3") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, config.S3Store, c.FileStore) + }) + + t.Run("FILE_STORE_HTTP_ROOT", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Unsetenv("CLIPPER_FILE_STORE_HTTP_ROOT") + _, err := config.NewFromEnv() + require.EqualError(t, err, "FILE_STORE_HTTP_ROOT not set") + + os.Setenv("CLIPPER_FILE_STORE_HTTP_ROOT", "/foo") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "/foo", c.FileStoreHTTPRoot) + }) + + t.Run("FILE_STORE_HTTP_BASE_URL", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Setenv("CLIPPER_FILE_STORE_HTTP_BASE_URL", "%%") + _, err := config.NewFromEnv() + require.EqualError(t, err, "invalid CLIPPER_FILE_STORE_HTTP_BASE_URL: %%/") + + os.Unsetenv("CLIPPER_FILE_STORE_HTTP_BASE_URL") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, mustParseURL(t, "/"), c.FileStoreHTTPBaseURL) + + os.Setenv("CLIPPER_FILE_STORE_HTTP_BASE_URL", "/foo") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, mustParseURL(t, "/foo/"), c.FileStoreHTTPBaseURL) + + os.Setenv("CLIPPER_FILE_STORE_HTTP_BASE_URL", "/foo/") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, mustParseURL(t, "/foo/"), c.FileStoreHTTPBaseURL) + + os.Setenv("CLIPPER_FILE_STORE_HTTP_BASE_URL", "https://www.example.com/foo") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, mustParseURL(t, "https://www.example.com/foo/"), c.FileStoreHTTPBaseURL) + }) + + t.Run("ASSETS_HTTP_ROOT", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Unsetenv("CLIPPER_ASSETS_HTTP_ROOT") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "", c.AssetsHTTPRoot) + + os.Setenv("CLIPPER_ASSETS_HTTP_ROOT", "/bar") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "/bar", c.AssetsHTTPRoot) + }) + + t.Run("FFMPEG_WORKER_POOL_SIZE", func(t *testing.T) { + defer clearenv() + setupenv() + + os.Setenv("CLIPPER_FFMPEG_WORKER_POOL_SIZE", "nope") + c, err := config.NewFromEnv() + assert.EqualError(t, err, "invalid CLIPPER_FFMPEG_WORKER_POOL_SIZE value: nope") + + os.Unsetenv("CLIPPER_FFMPEG_WORKER_POOL_SIZE") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, runtime.NumCPU(), c.FFmpegWorkerPoolSize) + + os.Setenv("CLIPPER_FFMPEG_WORKER_POOL_SIZE", "10") + c, err = config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, 10, c.FFmpegWorkerPoolSize) + }) + + t.Run("AWS configuration", func(t *testing.T) { + defer clearenv() + setupenv() + os.Setenv("CLIPPER_FILE_STORE", "s3") + + os.Unsetenv("AWS_ACCESS_KEY_ID") + _, err := config.NewFromEnv() + assert.EqualError(t, err, "AWS_ACCESS_KEY_ID not set") + + os.Setenv("AWS_ACCESS_KEY_ID", "key") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") + _, err = config.NewFromEnv() + assert.EqualError(t, err, "AWS_SECRET_ACCESS_KEY not set") + + os.Setenv("AWS_ACCESS_KEY_ID", "key") + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Unsetenv("AWS_REGION") + _, err = config.NewFromEnv() + assert.EqualError(t, err, "AWS_REGION not set") + + os.Setenv("AWS_ACCESS_KEY_ID", "key") + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_REGION", "eu-west-1") + os.Unsetenv("CLIPPER_S3_BUCKET") + _, err = config.NewFromEnv() + assert.EqualError(t, err, "S3_BUCKET not set") + + os.Setenv("AWS_ACCESS_KEY_ID", "key") + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_REGION", "eu-west-1") + os.Setenv("CLIPPER_S3_BUCKET", "bucket") + c, err := config.NewFromEnv() + require.NoError(t, err) + assert.Equal(t, "key", c.AWSAccessKeyID) + assert.Equal(t, "secret", c.AWSSecretAccessKey) + assert.Equal(t, "eu-west-1", c.AWSRegion) + assert.Equal(t, "bucket", c.S3Bucket) + }) +} diff --git a/backend/go.mod b/backend/go.mod index e611061..4461392 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -8,7 +8,6 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.6.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.22.0 github.com/aws/smithy-go v1.9.0 - github.com/gofrs/uuid v4.0.0+incompatible github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/schema v1.2.0