rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Add FileSystemStore file store

This commit is contained in:
Rob Watson 2021-12-08 20:58:13 +01:00
parent f2d7af0860
commit 4168cb150c
6 changed files with 213 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/backend/.env
/backend/debug/
/backend/data

21
backend/.env.example
View File

@ -2,15 +2,24 @@ ENV=development # or production
BIND_ADDR=localhost:8888
# AWS credentials, currently required.
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
S3_BUCKET=
# PostgreSQL connection string.
DATABASE_URL=
# Optional. If set, files in this location will be served over HTTP at /.
# Mostly useful for deployment.
ASSETS_HTTP_BASE_PATH=
# Set the store type - either s3 or filesystem. Defaults to filesystem. The S3
# store is recommended for production usage.
#
# NOTE: Enabling the file system store will disable serving assets over HTTP.
FILE_STORE=filesystem
# The base path for the file system store.
FILE_STORE_HTTP_BASE_PATH=data/
# AWS credentials, required for the S3 store.
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
S3_BUCKET=

61
backend/cmd/clipper/main.go
View File

@ -8,6 +8,7 @@ import (
"git.netflux.io/rob/clipper/config"
"git.netflux.io/rob/clipper/filestore"
"git.netflux.io/rob/clipper/generated/store"
"git.netflux.io/rob/clipper/media"
"git.netflux.io/rob/clipper/server"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
@ -40,20 +41,6 @@ func main() {
// Create a Youtube client
var youtubeClient youtube.Client
// Create an Amazon S3 service s3Client
cfg, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""),
),
awsconfig.WithRegion(config.AWSRegion),
)
if err != nil {
log.Fatal(err)
}
s3Client := s3.NewFromConfig(cfg)
s3PresignClient := s3.NewPresignClient(s3Client)
// Create a logger
logger, err := buildLogger(config)
if err != nil {
@ -62,15 +49,10 @@ func main() {
defer logger.Sync()
// Create a file store
fileStore := filestore.NewS3FileStore(
filestore.S3API{
S3Client: s3Client,
S3PresignClient: s3PresignClient,
},
config.S3Bucket,
defaultURLExpiry,
logger.Sugar().Named("filestore"),
)
fileStore, err := buildFileStore(ctx, config, logger)
if err != nil {
log.Fatal(err)
}
log.Fatal(server.Start(server.Options{
Config: config,
@ -88,3 +70,36 @@ func buildLogger(c config.Config) (*zap.Logger, error) {
}
return zap.NewProduction()
}
func buildFileStore(ctx context.Context, c config.Config, logger *zap.Logger) (media.FileStore, error) {
if c.FileStore == config.S3Store {
logger.Info("Initializing S3 store")
// Create an Amazon S3 service s3Client
cfg, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(c.AWSAccessKeyID, c.AWSSecretAccessKey, ""),
),
awsconfig.WithRegion(c.AWSRegion),
)
if err != nil {
log.Fatal(err)
}
s3Client := s3.NewFromConfig(cfg)
s3PresignClient := s3.NewPresignClient(s3Client)
// Create a file store
return filestore.NewS3FileStore(
filestore.S3API{
S3Client: s3Client,
S3PresignClient: s3PresignClient,
},
c.S3Bucket,
defaultURLExpiry,
logger.Sugar().Named("filestore"),
), nil
}
logger.Info("Initializing file sysytem store")
return filestore.NewFileSystemStore(c.FileStoreHTTPBasePath, "/")
}

99
backend/config/config.go
View File

@ -13,17 +13,26 @@ const (
EnvProduction
)
type FileStore int
const (
FileSystemStore = iota
S3Store
)
type Config struct {
Environment Environment
BindAddr string
TLSCertFile string
TLSKeyFile string
DatabaseURL string
AWSAccessKeyID string
AWSSecretAccessKey string
AWSRegion string
S3Bucket string
AssetsHTTPBasePath string
Environment Environment
BindAddr string
TLSCertFile string
TLSKeyFile string
DatabaseURL string
FileStore FileStore
FileStoreHTTPBasePath string
AWSAccessKeyID string
AWSSecretAccessKey string
AWSRegion string
S3Bucket string
AssetsHTTPBasePath string
}
func NewFromEnv() (Config, error) {
@ -48,7 +57,7 @@ func NewFromEnv() (Config, error) {
tlsCertFile := os.Getenv("TLS_CERT_FILE")
tlsKeyFile := os.Getenv("TLS_KEY_FILE")
if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") {
return Config{}, errors.New("Both TLS_CERT_FILE and TLS_KEY_FILE must be set")
return Config{}, errors.New("both TLS_CERT_FILE and TLS_KEY_FILE must be set")
}
databaseURL := os.Getenv("DATABASE_URL")
@ -56,38 +65,58 @@ func NewFromEnv() (Config, error) {
return Config{}, errors.New("DATABASE_URL not set")
}
awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
if awsAccessKeyID == "" {
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
fileStoreString := os.Getenv("FILE_STORE")
var fileStore FileStore
switch os.Getenv("FILE_STORE") {
case "s3":
fileStore = S3Store
case "filesystem", "":
fileStore = FileSystemStore
default:
return Config{}, fmt.Errorf("invalid FILE_STORE value: %s", fileStoreString)
}
awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if awsSecretAccessKey == "" {
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
}
var awsAccessKeyID, awsSecretAccessKey, awsRegion, s3Bucket, fileStoreHTTPBasePath string
if fileStore == S3Store {
awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
if awsAccessKeyID == "" {
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
}
awsRegion := os.Getenv("AWS_REGION")
if awsRegion == "" {
return Config{}, errors.New("AWS_REGION not set")
}
awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
if awsSecretAccessKey == "" {
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
}
s3Bucket := os.Getenv("S3_BUCKET")
if s3Bucket == "" {
return Config{}, errors.New("S3_BUCKET not set")
awsRegion = os.Getenv("AWS_REGION")
if awsRegion == "" {
return Config{}, errors.New("AWS_REGION not set")
}
s3Bucket = os.Getenv("S3_BUCKET")
if s3Bucket == "" {
return Config{}, errors.New("S3_BUCKET not set")
}
} else {
if fileStoreHTTPBasePath = os.Getenv("FILE_STORE_HTTP_BASE_PATH"); fileStoreHTTPBasePath == "" {
return Config{}, errors.New("FILE_STORE_HTTP_BASE_PATH not set")
}
}
assetsHTTPBasePath := os.Getenv("ASSETS_HTTP_BASE_PATH")
return Config{
Environment: env,
BindAddr: bindAddr,
TLSCertFile: tlsCertFile,
TLSKeyFile: tlsKeyFile,
DatabaseURL: databaseURL,
AWSAccessKeyID: awsAccessKeyID,
AWSSecretAccessKey: awsSecretAccessKey,
AWSRegion: awsRegion,
S3Bucket: s3Bucket,
AssetsHTTPBasePath: assetsHTTPBasePath,
Environment: env,
BindAddr: bindAddr,
TLSCertFile: tlsCertFile,
TLSKeyFile: tlsKeyFile,
DatabaseURL: databaseURL,
FileStore: fileStore,
AWSAccessKeyID: awsAccessKeyID,
AWSSecretAccessKey: awsSecretAccessKey,
AWSRegion: awsRegion,
S3Bucket: s3Bucket,
AssetsHTTPBasePath: assetsHTTPBasePath,
FileStoreHTTPBasePath: fileStoreHTTPBasePath,
}, nil
}

88
backend/filestore/fs.go Normal file
View File

@ -0,0 +1,88 @@
package filestore
import (
"context"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
)
// FileSystemStore is a file store that stores files on the local filesystem.
// It is currently intended for usage in a development environment.
type FileSystemStore struct {
rootPath string
baseURL *url.URL
}
// NewFileSystemStore creates a new FileSystemStore. It accepts a root path,
// which is the storage location on the local file system for stored objects,
// and a baseURL which is a URL which should be configured to serve the stored
// files over HTTP.
func NewFileSystemStore(rootPath string, baseURL string) (*FileSystemStore, error) {
url, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("error parsing URL: %v", err)
}
return &FileSystemStore{rootPath: rootPath, baseURL: url}, nil
}
// GetObject retrieves an object from the local filesystem.
func (s *FileSystemStore) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
path := filepath.Join(s.rootPath, key)
fptr, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("error opening file: %v", err)
}
return fptr, nil
}
type readCloser struct {
io.Reader
io.Closer
}
// GetObjectWithRange retrieves an object from the local filesystem with the given byte range.
func (s *FileSystemStore) GetObjectWithRange(ctx context.Context, key string, start, end int64) (io.ReadCloser, error) {
path := filepath.Join(s.rootPath, key)
fptr, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("error opening file: %v", err)
}
_, err = fptr.Seek(start, os.SEEK_SET)
if err != nil {
return nil, fmt.Errorf("error seeking in file: %v", err)
}
return readCloser{
Reader: io.LimitReader(fptr, end-start),
Closer: fptr,
}, nil
}
// GetURL returns an HTTP URL for the provided file path.
func (s *FileSystemStore) GetURL(ctx context.Context, key string) (string, error) {
url := url.URL{}
url.Host = s.baseURL.Host
url.Path = fmt.Sprintf("%s%s", s.baseURL.Path, key)
return url.String(), nil
}
// PutObject writes an object to the local filesystem.
func (s *FileSystemStore) PutObject(ctx context.Context, key string, r io.Reader, _ string) (int64, error) {
path := filepath.Join(s.rootPath, key)
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return 0, fmt.Errorf("error creating directories: %v", err)
}
fptr, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("error opening file: %v", err)
}
n, err := io.Copy(fptr, r)
if err != nil {
return n, fmt.Errorf("error writing file: %v", err)
}
return n, nil
}

7
backend/server/server.go
View File

@ -248,10 +248,17 @@ func Start(options Options) error {
log := options.Logger.Sugar()
fileHandler := http.NotFoundHandler()
// Enabling the file system store disables serving assets over HTTP.
// TODO: fix this.
if options.Config.AssetsHTTPBasePath != "" {
log.With("basePath", options.Config.AssetsHTTPBasePath).Info("Configured to serve assets over HTTP")
fileHandler = http.FileServer(http.Dir(options.Config.AssetsHTTPBasePath))
}
if options.Config.FileStoreHTTPBasePath != "" {
log.With("basePath", options.Config.FileStoreHTTPBasePath).Info("Configured to serve file store over HTTP")
fileHandler = http.FileServer(http.Dir(options.Config.FileStoreHTTPBasePath))
}
httpServer := http.Server{
Addr: options.Config.BindAddr,