rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Add FileSystemStore file store

This commit is contained in:
Rob Watson 2021-12-08 20:58:13 +01:00
parent f2d7af0860
commit 4168cb150c
6 changed files with 213 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/backend/.env
/backend/debug/
/backend/data

21
backend/.env.example
View File

@ -2,15 +2,24 @@ ENV=development # or production
BIND_ADDR=localhost:8888
# AWS credentials, currently required.
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
S3_BUCKET=
# PostgreSQL connection string.
DATABASE_URL=
# Optional. If set, files in this location will be served over HTTP at /.
# Mostly useful for deployment.
ASSETS_HTTP_BASE_PATH=
# Set the store type - either s3 or filesystem. Defaults to filesystem. The S3
# store is recommended for production usage.
#
# NOTE: Enabling the file system store will disable serving assets over HTTP.
FILE_STORE=filesystem
# The base path for the file system store.
FILE_STORE_HTTP_BASE_PATH=data/
# AWS credentials, required for the S3 store.
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
S3_BUCKET=

61
backend/cmd/clipper/main.go
View File

@ -8,6 +8,7 @@ import (
"git.netflux.io/rob/clipper/config"
"git.netflux.io/rob/clipper/filestore"
"git.netflux.io/rob/clipper/generated/store"
"git.netflux.io/rob/clipper/media"
"git.netflux.io/rob/clipper/server"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
@ -40,20 +41,6 @@ func main() {
// Create a Youtube client
var youtubeClient youtube.Client
// Create an Amazon S3 service s3Client
cfg, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""),
),
awsconfig.WithRegion(config.AWSRegion),
)
if err != nil {
log.Fatal(err)
}
s3Client := s3.NewFromConfig(cfg)
s3PresignClient := s3.NewPresignClient(s3Client)
// Create a logger
logger, err := buildLogger(config)
if err != nil {
@ -62,15 +49,10 @@ func main() {
defer logger.Sync()
// Create a file store
fileStore := filestore.NewS3FileStore(
filestore.S3API{
S3Client: s3Client,
S3PresignClient: s3PresignClient,
},
config.S3Bucket,
defaultURLExpiry,
logger.Sugar().Named("filestore"),
)
fileStore, err := buildFileStore(ctx, config, logger)
if err != nil {
log.Fatal(err)
}
log.Fatal(server.Start(server.Options{
Config: config,
@ -88,3 +70,36 @@ func buildLogger(c config.Config) (*zap.Logger, error) {
}
return zap.NewProduction()
}
func buildFileStore(ctx context.Context, c config.Config, logger *zap.Logger) (media.FileStore, error) {
if c.FileStore == config.S3Store {
logger.Info("Initializing S3 store")
// Create an Amazon S3 service s3Client
cfg, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(c.AWSAccessKeyID, c.AWSSecretAccessKey, ""),
),
awsconfig.WithRegion(c.AWSRegion),
)
if err != nil {
log.Fatal(err)
}
s3Client := s3.NewFromConfig(cfg)
s3PresignClient := s3.NewPresignClient(s3Client)
// Create a file store
return filestore.NewS3FileStore(
filestore.S3API{
S3Client: s3Client,
S3PresignClient: s3PresignClient,
},
c.S3Bucket,
defaultURLExpiry,
logger.Sugar().Named("filestore"),
), nil
}
logger.Info("Initializing file sysytem store")
return filestore.NewFileSystemStore(c.FileStoreHTTPBasePath, "/")
}

39
backend/config/config.go
View File

@ -13,12 +13,21 @@ const (
EnvProduction
)
type FileStore int
const (
FileSystemStore = iota
S3Store
)
type Config struct {
Environment Environment
BindAddr string
TLSCertFile string
TLSKeyFile string
DatabaseURL string
FileStore FileStore
FileStoreHTTPBasePath string
AWSAccessKeyID string
AWSSecretAccessKey string
AWSRegion string
@ -48,7 +57,7 @@ func NewFromEnv() (Config, error) {
tlsCertFile := os.Getenv("TLS_CERT_FILE")
tlsKeyFile := os.Getenv("TLS_KEY_FILE")
if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") {
return Config{}, errors.New("Both TLS_CERT_FILE and TLS_KEY_FILE must be set")
return Config{}, errors.New("both TLS_CERT_FILE and TLS_KEY_FILE must be set")
}
databaseURL := os.Getenv("DATABASE_URL")
@ -56,25 +65,43 @@ func NewFromEnv() (Config, error) {
return Config{}, errors.New("DATABASE_URL not set")
}
awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
fileStoreString := os.Getenv("FILE_STORE")
var fileStore FileStore
switch os.Getenv("FILE_STORE") {
case "s3":
fileStore = S3Store
case "filesystem", "":
fileStore = FileSystemStore
default:
return Config{}, fmt.Errorf("invalid FILE_STORE value: %s", fileStoreString)
}
var awsAccessKeyID, awsSecretAccessKey, awsRegion, s3Bucket, fileStoreHTTPBasePath string
if fileStore == S3Store {
awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
if awsAccessKeyID == "" {
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
}
awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
if awsSecretAccessKey == "" {
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
}
awsRegion := os.Getenv("AWS_REGION")
awsRegion = os.Getenv("AWS_REGION")
if awsRegion == "" {
return Config{}, errors.New("AWS_REGION not set")
}
s3Bucket := os.Getenv("S3_BUCKET")
s3Bucket = os.Getenv("S3_BUCKET")
if s3Bucket == "" {
return Config{}, errors.New("S3_BUCKET not set")
}
} else {
if fileStoreHTTPBasePath = os.Getenv("FILE_STORE_HTTP_BASE_PATH"); fileStoreHTTPBasePath == "" {
return Config{}, errors.New("FILE_STORE_HTTP_BASE_PATH not set")
}
}
assetsHTTPBasePath := os.Getenv("ASSETS_HTTP_BASE_PATH")
@ -84,10 +111,12 @@ func NewFromEnv() (Config, error) {
TLSCertFile: tlsCertFile,
TLSKeyFile: tlsKeyFile,
DatabaseURL: databaseURL,
FileStore: fileStore,
AWSAccessKeyID: awsAccessKeyID,
AWSSecretAccessKey: awsSecretAccessKey,
AWSRegion: awsRegion,
S3Bucket: s3Bucket,
AssetsHTTPBasePath: assetsHTTPBasePath,
FileStoreHTTPBasePath: fileStoreHTTPBasePath,
}, nil
}

88
backend/filestore/fs.go Normal file
View File

@ -0,0 +1,88 @@
package filestore
import (
"context"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
)
// FileSystemStore is a file store that stores files on the local filesystem.
// It is currently intended for usage in a development environment.
type FileSystemStore struct {
rootPath string
baseURL *url.URL
}
// NewFileSystemStore creates a new FileSystemStore. It accepts a root path,
// which is the storage location on the local file system for stored objects,
// and a baseURL which is a URL which should be configured to serve the stored
// files over HTTP.
func NewFileSystemStore(rootPath string, baseURL string) (*FileSystemStore, error) {
url, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("error parsing URL: %v", err)
}
return &FileSystemStore{rootPath: rootPath, baseURL: url}, nil
}
// GetObject retrieves an object from the local filesystem.
func (s *FileSystemStore) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
path := filepath.Join(s.rootPath, key)
fptr, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("error opening file: %v", err)
}
return fptr, nil
}
type readCloser struct {
io.Reader
io.Closer
}
// GetObjectWithRange retrieves an object from the local filesystem with the given byte range.
func (s *FileSystemStore) GetObjectWithRange(ctx context.Context, key string, start, end int64) (io.ReadCloser, error) {
path := filepath.Join(s.rootPath, key)
fptr, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("error opening file: %v", err)
}
_, err = fptr.Seek(start, os.SEEK_SET)
if err != nil {
return nil, fmt.Errorf("error seeking in file: %v", err)
}
return readCloser{
Reader: io.LimitReader(fptr, end-start),
Closer: fptr,
}, nil
}
// GetURL returns an HTTP URL for the provided file path.
func (s *FileSystemStore) GetURL(ctx context.Context, key string) (string, error) {
url := url.URL{}
url.Host = s.baseURL.Host
url.Path = fmt.Sprintf("%s%s", s.baseURL.Path, key)
return url.String(), nil
}
// PutObject writes an object to the local filesystem.
func (s *FileSystemStore) PutObject(ctx context.Context, key string, r io.Reader, _ string) (int64, error) {
path := filepath.Join(s.rootPath, key)
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return 0, fmt.Errorf("error creating directories: %v", err)
}
fptr, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("error opening file: %v", err)
}
n, err := io.Copy(fptr, r)
if err != nil {
return n, fmt.Errorf("error writing file: %v", err)
}
return n, nil
}

7
backend/server/server.go
View File

@ -248,10 +248,17 @@ func Start(options Options) error {
log := options.Logger.Sugar()
fileHandler := http.NotFoundHandler()
// Enabling the file system store disables serving assets over HTTP.
// TODO: fix this.
if options.Config.AssetsHTTPBasePath != "" {
log.With("basePath", options.Config.AssetsHTTPBasePath).Info("Configured to serve assets over HTTP")
fileHandler = http.FileServer(http.Dir(options.Config.AssetsHTTPBasePath))
}
if options.Config.FileStoreHTTPBasePath != "" {
log.With("basePath", options.Config.FileStoreHTTPBasePath).Info("Configured to serve file store over HTTP")
fileHandler = http.FileServer(http.Dir(options.Config.FileStoreHTTPBasePath))
}
httpServer := http.Server{
Addr: options.Config.BindAddr,