From b643ea28245342b6ef038b700e1c543ea68835ef Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Mon, 22 Nov 2021 19:26:51 +0100 Subject: [PATCH] Add config package --- backend/.env.example | 4 +++ backend/cmd/clipper/main.go | 25 ++++++++----- backend/config/config.go | 72 +++++++++++++++++++++++++++++++++++++ backend/media/service.go | 17 ++++----- backend/media/uploader.go | 1 + backend/server/server.go | 30 ++++++++-------- 6 files changed, 117 insertions(+), 32 deletions(-) create mode 100644 backend/config/config.go diff --git a/backend/.env.example b/backend/.env.example index d738c50..752a9f9 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,4 +1,8 @@ +ENV=development # or production + AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +AWS_REGION= +S3_BUCKET= DATABASE_URL= diff --git a/backend/cmd/clipper/main.go b/backend/cmd/clipper/main.go index 682f61a..13a7834 100644 --- a/backend/cmd/clipper/main.go +++ b/backend/cmd/clipper/main.go @@ -4,13 +4,14 @@ import ( "context" "database/sql" "log" - "os" "time" + "git.netflux.io/rob/clipper/config" "git.netflux.io/rob/clipper/generated/store" "git.netflux.io/rob/clipper/media" "git.netflux.io/rob/clipper/server" - "github.com/aws/aws-sdk-go-v2/config" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/kkdai/youtube/v2" _ "github.com/lib/pq" @@ -24,20 +25,26 @@ const ( func main() { ctx := context.Background() - // Create a store - databaseURL := os.Getenv("DATABASE_URL") - if databaseURL == "" { - log.Fatal("DATABASE_URL not set") + config, err := config.NewFromEnv() + if err != nil { + log.Fatalf("error loading config: %v", err) } - db, err := sql.Open("postgres", databaseURL) + // Create a store + db, err := sql.Open("postgres", config.DatabaseURL) if err != nil { log.Fatal(err) } store := store.New(db) // Create an Amazon S3 service s3Client - cfg, err := config.LoadDefaultConfig(ctx) + cfg, err := awsconfig.LoadDefaultConfig( + ctx, + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""), + ), + awsconfig.WithRegion(config.AWSRegion), + ) if err != nil { log.Fatal(err) } @@ -50,7 +57,7 @@ func main() { var youtubeClient youtube.Client serverOptions := server.Options{ - Environment: server.Development, + Config: config, BindAddr: DefaultHTTPBindAddr, Timeout: DefaultTimeout, Store: store, diff --git a/backend/config/config.go b/backend/config/config.go new file mode 100644 index 0000000..78b6560 --- /dev/null +++ b/backend/config/config.go @@ -0,0 +1,72 @@ +package config + +import ( + "errors" + "fmt" + "os" +) + +type Environment int + +const ( + EnvDevelopment Environment = iota + EnvProduction +) + +type Config struct { + Environment Environment + DatabaseURL string + AWSAccessKeyID string + AWSSecretAccessKey string + AWSRegion string + S3Bucket string +} + +func NewFromEnv() (Config, error) { + envString := os.Getenv("ENV") + var env Environment + switch envString { + case "production": + env = EnvProduction + case "development": + env = EnvDevelopment + case "": + return Config{}, errors.New("ENV not set") + default: + return Config{}, fmt.Errorf("invalid ENVIRONMENT value: %s", envString) + } + + databaseURL := os.Getenv("DATABASE_URL") + if databaseURL == "" { + return Config{}, errors.New("DATABASE_URL not set") + } + + awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID") + if awsAccessKeyID == "" { + return Config{}, errors.New("AWS_ACCESS_KEY_ID not set") + } + + awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + if awsSecretAccessKey == "" { + return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set") + } + + awsRegion := os.Getenv("AWS_REGION") + if awsRegion == "" { + return Config{}, errors.New("AWS_REGION not set") + } + + s3Bucket := os.Getenv("S3_BUCKET") + if s3Bucket == "" { + return Config{}, errors.New("S3_BUCKET not set") + } + + return Config{ + Environment: env, + DatabaseURL: databaseURL, + AWSAccessKeyID: awsAccessKeyID, + AWSSecretAccessKey: awsSecretAccessKey, + AWSRegion: awsRegion, + S3Bucket: s3Bucket, + }, nil +} diff --git a/backend/media/service.go b/backend/media/service.go index d7f1073..4a33352 100644 --- a/backend/media/service.go +++ b/backend/media/service.go @@ -13,6 +13,7 @@ import ( "strconv" "time" + "git.netflux.io/rob/clipper/config" "git.netflux.io/rob/clipper/generated/store" "github.com/aws/aws-sdk-go-v2/aws" signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" @@ -23,8 +24,6 @@ import ( ) const ( - s3Bucket = "clipper-development" - getVideoExpiresIn = time.Hour * 1 ) @@ -97,14 +96,16 @@ type MediaSetService struct { store Store youtube YoutubeClient s3 S3API + config config.Config logger *zap.SugaredLogger } -func NewMediaSetService(store Store, youtubeClient YoutubeClient, s3API S3API, logger *zap.Logger) *MediaSetService { +func NewMediaSetService(store Store, youtubeClient YoutubeClient, s3API S3API, config config.Config, logger *zap.Logger) *MediaSetService { return &MediaSetService{ store: store, youtube: youtubeClient, s3: s3API, + config: config, logger: logger.Sugar(), } } @@ -269,7 +270,7 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP s3Key := fmt.Sprintf("media_sets/%s/video.mp4", mediaSet.ID) if mediaSet.VideoS3UploadedAt.Valid { - input := s3.GetObjectInput{Bucket: aws.String(s3Bucket), Key: aws.String(s3Key)} + input := s3.GetObjectInput{Bucket: aws.String(s.config.S3Bucket), Key: aws.String(s3Key)} request, signErr := s.s3.PresignGetObject(ctx, &input, s3.WithPresignExpires(getVideoExpiresIn)) if signErr != nil { return nil, fmt.Errorf("error generating presigned URL: %v", signErr) @@ -299,7 +300,7 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP stream, format.ContentLength, mediaSet.ID, - s3Bucket, + s.config.S3Bucket, s3Key, format.MimeType, ) @@ -436,7 +437,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor getAudioProgressReader: getAudioProgressReader, ffmpegReader: ffmpegReader, uploader: uploader, - s3Bucket: s3Bucket, + s3Bucket: s.config.S3Bucket, s3Key: s3Key, store: s.store, channels: format.AudioChannels, @@ -707,7 +708,7 @@ func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet uploader := newMultipartUploader(s.s3) const mimeType = "application/jpeg" - _, err = uploader.Upload(ctx, bytes.NewReader(imageData), s3Bucket, s3Key, mimeType) + _, err = uploader.Upload(ctx, bytes.NewReader(imageData), s.config.S3Bucket, s3Key, mimeType) if err != nil { return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err) } @@ -715,7 +716,7 @@ func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet storeParams := store.SetVideoThumbnailUploadedParams{ ID: mediaSet.ID, VideoThumbnailMimeType: sqlString(mimeType), - VideoThumbnailS3Bucket: sqlString(s3Bucket), + VideoThumbnailS3Bucket: sqlString(s.config.S3Bucket), VideoThumbnailS3Key: sqlString(s3Key), VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)), VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)), diff --git a/backend/media/uploader.go b/backend/media/uploader.go index 3cc3c15..6bede6b 100644 --- a/backend/media/uploader.go +++ b/backend/media/uploader.go @@ -201,6 +201,7 @@ outer: } if _, err = u.s3.CompleteMultipartUpload(ctx, &completeInput); err != nil { + log.Printf("parts: %+v", completedParts) return 0, fmt.Errorf("error completing upload: %v", err) } diff --git a/backend/server/server.go b/backend/server/server.go index 946edcd..b9cea9c 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "git.netflux.io/rob/clipper/config" pbmediaset "git.netflux.io/rob/clipper/generated/pb/media_set" "git.netflux.io/rob/clipper/media" "github.com/google/uuid" @@ -58,15 +59,8 @@ func newResponseError(err error) *ResponseError { return &ResponseError{err: err, s: defaultResponseMessage} } -type Environment int - -const ( - Development Environment = iota - Production -) - type Options struct { - Environment Environment + Config config.Config BindAddr string Timeout time.Duration Store media.Store @@ -232,15 +226,21 @@ func (c *mediaSetServiceController) GetVideoThumbnail(ctx context.Context, reque } func Start(options Options) error { - logger, err := buildLogger(options.Environment) + logger, err := buildLogger(options.Config) if err != nil { return fmt.Errorf("error building logger: %v", err) } defer logger.Sync() - fetchMediaSetService := media.NewMediaSetService(options.Store, options.YoutubeClient, options.S3API, logger) + fetchMediaSetService := media.NewMediaSetService( + options.Store, + options.YoutubeClient, + options.S3API, + options.Config, + logger, + ) - grpcServer := buildGRPCServer(options, logger) + grpcServer := buildGRPCServer(options.Config, logger) mediaSetController := &mediaSetServiceController{mediaSetService: fetchMediaSetService, logger: logger.Sugar().Named("controller")} pbmediaset.RegisterMediaSetServiceServer(grpcServer, mediaSetController) @@ -257,21 +257,21 @@ func Start(options Options) error { return httpServer.ListenAndServe() } -func buildLogger(env Environment) (*zap.Logger, error) { - if env == Production { +func buildLogger(c config.Config) (*zap.Logger, error) { + if c.Environment == config.EnvProduction { return zap.NewProduction() } return zap.NewDevelopment() } -func buildGRPCServer(options Options, logger *zap.Logger) *grpc.Server { +func buildGRPCServer(c config.Config, logger *zap.Logger) *grpc.Server { unaryInterceptors := []grpc.UnaryServerInterceptor{ grpczap.UnaryServerInterceptor(logger), } streamInterceptors := []grpc.StreamServerInterceptor{ grpczap.StreamServerInterceptor(logger), } - if options.Environment == Production { + if c.Environment == config.EnvProduction { panicOpts := []grpcrecovery.Option{ grpcrecovery.WithRecoveryHandler(func(p interface{}) error { return newResponseError(fmt.Errorf("%v", p))