From 4168cb150c173dffa9c1a2c47dc820793dc2753d Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Wed, 8 Dec 2021 20:58:13 +0100 Subject: [PATCH] Add FileSystemStore file store --- .gitignore | 1 + backend/.env.example | 21 +++++--- backend/cmd/clipper/main.go | 61 ++++++++++++++--------- backend/config/config.go | 99 ++++++++++++++++++++++++------------- backend/filestore/fs.go | 88 +++++++++++++++++++++++++++++++++ backend/server/server.go | 7 +++ 6 files changed, 213 insertions(+), 64 deletions(-) create mode 100644 backend/filestore/fs.go diff --git a/.gitignore b/.gitignore index c50247e..556f5cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /backend/.env /backend/debug/ +/backend/data diff --git a/backend/.env.example b/backend/.env.example index a3f0898..e29a8dc 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -2,15 +2,24 @@ ENV=development # or production BIND_ADDR=localhost:8888 -# AWS credentials, currently required. -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_REGION= -S3_BUCKET= - # PostgreSQL connection string. DATABASE_URL= # Optional. If set, files in this location will be served over HTTP at /. # Mostly useful for deployment. ASSETS_HTTP_BASE_PATH= + +# Set the store type - either s3 or filesystem. Defaults to filesystem. The S3 +# store is recommended for production usage. +# +# NOTE: Enabling the file system store will disable serving assets over HTTP. +FILE_STORE=filesystem + +# The base path for the file system store. +FILE_STORE_HTTP_BASE_PATH=data/ + +# AWS credentials, required for the S3 store. +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= +S3_BUCKET= diff --git a/backend/cmd/clipper/main.go b/backend/cmd/clipper/main.go index e5cb688..4a34133 100644 --- a/backend/cmd/clipper/main.go +++ b/backend/cmd/clipper/main.go @@ -8,6 +8,7 @@ import ( "git.netflux.io/rob/clipper/config" "git.netflux.io/rob/clipper/filestore" "git.netflux.io/rob/clipper/generated/store" + "git.netflux.io/rob/clipper/media" "git.netflux.io/rob/clipper/server" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" @@ -40,20 +41,6 @@ func main() { // Create a Youtube client var youtubeClient youtube.Client - // Create an Amazon S3 service s3Client - cfg, err := awsconfig.LoadDefaultConfig( - ctx, - awsconfig.WithCredentialsProvider( - credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""), - ), - awsconfig.WithRegion(config.AWSRegion), - ) - if err != nil { - log.Fatal(err) - } - s3Client := s3.NewFromConfig(cfg) - s3PresignClient := s3.NewPresignClient(s3Client) - // Create a logger logger, err := buildLogger(config) if err != nil { @@ -62,15 +49,10 @@ func main() { defer logger.Sync() // Create a file store - fileStore := filestore.NewS3FileStore( - filestore.S3API{ - S3Client: s3Client, - S3PresignClient: s3PresignClient, - }, - config.S3Bucket, - defaultURLExpiry, - logger.Sugar().Named("filestore"), - ) + fileStore, err := buildFileStore(ctx, config, logger) + if err != nil { + log.Fatal(err) + } log.Fatal(server.Start(server.Options{ Config: config, @@ -88,3 +70,36 @@ func buildLogger(c config.Config) (*zap.Logger, error) { } return zap.NewProduction() } + +func buildFileStore(ctx context.Context, c config.Config, logger *zap.Logger) (media.FileStore, error) { + if c.FileStore == config.S3Store { + logger.Info("Initializing S3 store") + // Create an Amazon S3 service s3Client + cfg, err := awsconfig.LoadDefaultConfig( + ctx, + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.AWSAccessKeyID, c.AWSSecretAccessKey, ""), + ), + awsconfig.WithRegion(c.AWSRegion), + ) + if err != nil { + log.Fatal(err) + } + s3Client := s3.NewFromConfig(cfg) + s3PresignClient := s3.NewPresignClient(s3Client) + + // Create a file store + return filestore.NewS3FileStore( + filestore.S3API{ + S3Client: s3Client, + S3PresignClient: s3PresignClient, + }, + c.S3Bucket, + defaultURLExpiry, + logger.Sugar().Named("filestore"), + ), nil + } + + logger.Info("Initializing file sysytem store") + return filestore.NewFileSystemStore(c.FileStoreHTTPBasePath, "/") +} diff --git a/backend/config/config.go b/backend/config/config.go index e1f2f56..3297b3a 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -13,17 +13,26 @@ const ( EnvProduction ) +type FileStore int + +const ( + FileSystemStore = iota + S3Store +) + type Config struct { - Environment Environment - BindAddr string - TLSCertFile string - TLSKeyFile string - DatabaseURL string - AWSAccessKeyID string - AWSSecretAccessKey string - AWSRegion string - S3Bucket string - AssetsHTTPBasePath string + Environment Environment + BindAddr string + TLSCertFile string + TLSKeyFile string + DatabaseURL string + FileStore FileStore + FileStoreHTTPBasePath string + AWSAccessKeyID string + AWSSecretAccessKey string + AWSRegion string + S3Bucket string + AssetsHTTPBasePath string } func NewFromEnv() (Config, error) { @@ -48,7 +57,7 @@ func NewFromEnv() (Config, error) { tlsCertFile := os.Getenv("TLS_CERT_FILE") tlsKeyFile := os.Getenv("TLS_KEY_FILE") if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") { - return Config{}, errors.New("Both TLS_CERT_FILE and TLS_KEY_FILE must be set") + return Config{}, errors.New("both TLS_CERT_FILE and TLS_KEY_FILE must be set") } databaseURL := os.Getenv("DATABASE_URL") @@ -56,38 +65,58 @@ func NewFromEnv() (Config, error) { return Config{}, errors.New("DATABASE_URL not set") } - awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID") - if awsAccessKeyID == "" { - return Config{}, errors.New("AWS_ACCESS_KEY_ID not set") + fileStoreString := os.Getenv("FILE_STORE") + var fileStore FileStore + switch os.Getenv("FILE_STORE") { + case "s3": + fileStore = S3Store + case "filesystem", "": + fileStore = FileSystemStore + default: + return Config{}, fmt.Errorf("invalid FILE_STORE value: %s", fileStoreString) } - awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY") - if awsSecretAccessKey == "" { - return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set") - } + var awsAccessKeyID, awsSecretAccessKey, awsRegion, s3Bucket, fileStoreHTTPBasePath string + if fileStore == S3Store { + awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + if awsAccessKeyID == "" { + return Config{}, errors.New("AWS_ACCESS_KEY_ID not set") + } - awsRegion := os.Getenv("AWS_REGION") - if awsRegion == "" { - return Config{}, errors.New("AWS_REGION not set") - } + awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + if awsSecretAccessKey == "" { + return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set") + } - s3Bucket := os.Getenv("S3_BUCKET") - if s3Bucket == "" { - return Config{}, errors.New("S3_BUCKET not set") + awsRegion = os.Getenv("AWS_REGION") + if awsRegion == "" { + return Config{}, errors.New("AWS_REGION not set") + } + + s3Bucket = os.Getenv("S3_BUCKET") + if s3Bucket == "" { + return Config{}, errors.New("S3_BUCKET not set") + } + } else { + if fileStoreHTTPBasePath = os.Getenv("FILE_STORE_HTTP_BASE_PATH"); fileStoreHTTPBasePath == "" { + return Config{}, errors.New("FILE_STORE_HTTP_BASE_PATH not set") + } } assetsHTTPBasePath := os.Getenv("ASSETS_HTTP_BASE_PATH") return Config{ - Environment: env, - BindAddr: bindAddr, - TLSCertFile: tlsCertFile, - TLSKeyFile: tlsKeyFile, - DatabaseURL: databaseURL, - AWSAccessKeyID: awsAccessKeyID, - AWSSecretAccessKey: awsSecretAccessKey, - AWSRegion: awsRegion, - S3Bucket: s3Bucket, - AssetsHTTPBasePath: assetsHTTPBasePath, + Environment: env, + BindAddr: bindAddr, + TLSCertFile: tlsCertFile, + TLSKeyFile: tlsKeyFile, + DatabaseURL: databaseURL, + FileStore: fileStore, + AWSAccessKeyID: awsAccessKeyID, + AWSSecretAccessKey: awsSecretAccessKey, + AWSRegion: awsRegion, + S3Bucket: s3Bucket, + AssetsHTTPBasePath: assetsHTTPBasePath, + FileStoreHTTPBasePath: fileStoreHTTPBasePath, }, nil } diff --git a/backend/filestore/fs.go b/backend/filestore/fs.go new file mode 100644 index 0000000..af9a66a --- /dev/null +++ b/backend/filestore/fs.go @@ -0,0 +1,88 @@ +package filestore + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "path/filepath" +) + +// FileSystemStore is a file store that stores files on the local filesystem. +// It is currently intended for usage in a development environment. +type FileSystemStore struct { + rootPath string + baseURL *url.URL +} + +// NewFileSystemStore creates a new FileSystemStore. It accepts a root path, +// which is the storage location on the local file system for stored objects, +// and a baseURL which is a URL which should be configured to serve the stored +// files over HTTP. +func NewFileSystemStore(rootPath string, baseURL string) (*FileSystemStore, error) { + url, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("error parsing URL: %v", err) + } + return &FileSystemStore{rootPath: rootPath, baseURL: url}, nil +} + +// GetObject retrieves an object from the local filesystem. +func (s *FileSystemStore) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { + path := filepath.Join(s.rootPath, key) + fptr, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %v", err) + } + return fptr, nil +} + +type readCloser struct { + io.Reader + io.Closer +} + +// GetObjectWithRange retrieves an object from the local filesystem with the given byte range. +func (s *FileSystemStore) GetObjectWithRange(ctx context.Context, key string, start, end int64) (io.ReadCloser, error) { + path := filepath.Join(s.rootPath, key) + fptr, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %v", err) + } + _, err = fptr.Seek(start, os.SEEK_SET) + if err != nil { + return nil, fmt.Errorf("error seeking in file: %v", err) + } + + return readCloser{ + Reader: io.LimitReader(fptr, end-start), + Closer: fptr, + }, nil +} + +// GetURL returns an HTTP URL for the provided file path. +func (s *FileSystemStore) GetURL(ctx context.Context, key string) (string, error) { + url := url.URL{} + url.Host = s.baseURL.Host + url.Path = fmt.Sprintf("%s%s", s.baseURL.Path, key) + return url.String(), nil +} + +// PutObject writes an object to the local filesystem. +func (s *FileSystemStore) PutObject(ctx context.Context, key string, r io.Reader, _ string) (int64, error) { + path := filepath.Join(s.rootPath, key) + if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { + return 0, fmt.Errorf("error creating directories: %v", err) + } + fptr, err := os.Create(path) + if err != nil { + return 0, fmt.Errorf("error opening file: %v", err) + } + + n, err := io.Copy(fptr, r) + if err != nil { + return n, fmt.Errorf("error writing file: %v", err) + } + return n, nil +} diff --git a/backend/server/server.go b/backend/server/server.go index 86c122c..3f630f8 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -248,10 +248,17 @@ func Start(options Options) error { log := options.Logger.Sugar() fileHandler := http.NotFoundHandler() + + // Enabling the file system store disables serving assets over HTTP. + // TODO: fix this. if options.Config.AssetsHTTPBasePath != "" { log.With("basePath", options.Config.AssetsHTTPBasePath).Info("Configured to serve assets over HTTP") fileHandler = http.FileServer(http.Dir(options.Config.AssetsHTTPBasePath)) } + if options.Config.FileStoreHTTPBasePath != "" { + log.With("basePath", options.Config.FileStoreHTTPBasePath).Info("Configured to serve file store over HTTP") + fileHandler = http.FileServer(http.Dir(options.Config.FileStoreHTTPBasePath)) + } httpServer := http.Server{ Addr: options.Config.BindAddr,