Add FileSystemStore file store
This commit is contained in:
parent
f2d7af0860
commit
4168cb150c
|
@ -1,2 +1,3 @@
|
||||||
/backend/.env
|
/backend/.env
|
||||||
/backend/debug/
|
/backend/debug/
|
||||||
|
/backend/data
|
||||||
|
|
|
@ -2,15 +2,24 @@ ENV=development # or production
|
||||||
|
|
||||||
BIND_ADDR=localhost:8888
|
BIND_ADDR=localhost:8888
|
||||||
|
|
||||||
# AWS credentials, currently required.
|
|
||||||
AWS_ACCESS_KEY_ID=
|
|
||||||
AWS_SECRET_ACCESS_KEY=
|
|
||||||
AWS_REGION=
|
|
||||||
S3_BUCKET=
|
|
||||||
|
|
||||||
# PostgreSQL connection string.
|
# PostgreSQL connection string.
|
||||||
DATABASE_URL=
|
DATABASE_URL=
|
||||||
|
|
||||||
# Optional. If set, files in this location will be served over HTTP at /.
|
# Optional. If set, files in this location will be served over HTTP at /.
|
||||||
# Mostly useful for deployment.
|
# Mostly useful for deployment.
|
||||||
ASSETS_HTTP_BASE_PATH=
|
ASSETS_HTTP_BASE_PATH=
|
||||||
|
|
||||||
|
# Set the store type - either s3 or filesystem. Defaults to filesystem. The S3
|
||||||
|
# store is recommended for production usage.
|
||||||
|
#
|
||||||
|
# NOTE: Enabling the file system store will disable serving assets over HTTP.
|
||||||
|
FILE_STORE=filesystem
|
||||||
|
|
||||||
|
# The base path for the file system store.
|
||||||
|
FILE_STORE_HTTP_BASE_PATH=data/
|
||||||
|
|
||||||
|
# AWS credentials, required for the S3 store.
|
||||||
|
AWS_ACCESS_KEY_ID=
|
||||||
|
AWS_SECRET_ACCESS_KEY=
|
||||||
|
AWS_REGION=
|
||||||
|
S3_BUCKET=
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"git.netflux.io/rob/clipper/config"
|
"git.netflux.io/rob/clipper/config"
|
||||||
"git.netflux.io/rob/clipper/filestore"
|
"git.netflux.io/rob/clipper/filestore"
|
||||||
"git.netflux.io/rob/clipper/generated/store"
|
"git.netflux.io/rob/clipper/generated/store"
|
||||||
|
"git.netflux.io/rob/clipper/media"
|
||||||
"git.netflux.io/rob/clipper/server"
|
"git.netflux.io/rob/clipper/server"
|
||||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
|
@ -40,20 +41,6 @@ func main() {
|
||||||
// Create a Youtube client
|
// Create a Youtube client
|
||||||
var youtubeClient youtube.Client
|
var youtubeClient youtube.Client
|
||||||
|
|
||||||
// Create an Amazon S3 service s3Client
|
|
||||||
cfg, err := awsconfig.LoadDefaultConfig(
|
|
||||||
ctx,
|
|
||||||
awsconfig.WithCredentialsProvider(
|
|
||||||
credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""),
|
|
||||||
),
|
|
||||||
awsconfig.WithRegion(config.AWSRegion),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
s3Client := s3.NewFromConfig(cfg)
|
|
||||||
s3PresignClient := s3.NewPresignClient(s3Client)
|
|
||||||
|
|
||||||
// Create a logger
|
// Create a logger
|
||||||
logger, err := buildLogger(config)
|
logger, err := buildLogger(config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -62,15 +49,10 @@ func main() {
|
||||||
defer logger.Sync()
|
defer logger.Sync()
|
||||||
|
|
||||||
// Create a file store
|
// Create a file store
|
||||||
fileStore := filestore.NewS3FileStore(
|
fileStore, err := buildFileStore(ctx, config, logger)
|
||||||
filestore.S3API{
|
if err != nil {
|
||||||
S3Client: s3Client,
|
log.Fatal(err)
|
||||||
S3PresignClient: s3PresignClient,
|
}
|
||||||
},
|
|
||||||
config.S3Bucket,
|
|
||||||
defaultURLExpiry,
|
|
||||||
logger.Sugar().Named("filestore"),
|
|
||||||
)
|
|
||||||
|
|
||||||
log.Fatal(server.Start(server.Options{
|
log.Fatal(server.Start(server.Options{
|
||||||
Config: config,
|
Config: config,
|
||||||
|
@ -88,3 +70,36 @@ func buildLogger(c config.Config) (*zap.Logger, error) {
|
||||||
}
|
}
|
||||||
return zap.NewProduction()
|
return zap.NewProduction()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildFileStore(ctx context.Context, c config.Config, logger *zap.Logger) (media.FileStore, error) {
|
||||||
|
if c.FileStore == config.S3Store {
|
||||||
|
logger.Info("Initializing S3 store")
|
||||||
|
// Create an Amazon S3 service s3Client
|
||||||
|
cfg, err := awsconfig.LoadDefaultConfig(
|
||||||
|
ctx,
|
||||||
|
awsconfig.WithCredentialsProvider(
|
||||||
|
credentials.NewStaticCredentialsProvider(c.AWSAccessKeyID, c.AWSSecretAccessKey, ""),
|
||||||
|
),
|
||||||
|
awsconfig.WithRegion(c.AWSRegion),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
s3Client := s3.NewFromConfig(cfg)
|
||||||
|
s3PresignClient := s3.NewPresignClient(s3Client)
|
||||||
|
|
||||||
|
// Create a file store
|
||||||
|
return filestore.NewS3FileStore(
|
||||||
|
filestore.S3API{
|
||||||
|
S3Client: s3Client,
|
||||||
|
S3PresignClient: s3PresignClient,
|
||||||
|
},
|
||||||
|
c.S3Bucket,
|
||||||
|
defaultURLExpiry,
|
||||||
|
logger.Sugar().Named("filestore"),
|
||||||
|
), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("Initializing file sysytem store")
|
||||||
|
return filestore.NewFileSystemStore(c.FileStoreHTTPBasePath, "/")
|
||||||
|
}
|
||||||
|
|
|
@ -13,12 +13,21 @@ const (
|
||||||
EnvProduction
|
EnvProduction
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type FileStore int
|
||||||
|
|
||||||
|
const (
|
||||||
|
FileSystemStore = iota
|
||||||
|
S3Store
|
||||||
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Environment Environment
|
Environment Environment
|
||||||
BindAddr string
|
BindAddr string
|
||||||
TLSCertFile string
|
TLSCertFile string
|
||||||
TLSKeyFile string
|
TLSKeyFile string
|
||||||
DatabaseURL string
|
DatabaseURL string
|
||||||
|
FileStore FileStore
|
||||||
|
FileStoreHTTPBasePath string
|
||||||
AWSAccessKeyID string
|
AWSAccessKeyID string
|
||||||
AWSSecretAccessKey string
|
AWSSecretAccessKey string
|
||||||
AWSRegion string
|
AWSRegion string
|
||||||
|
@ -48,7 +57,7 @@ func NewFromEnv() (Config, error) {
|
||||||
tlsCertFile := os.Getenv("TLS_CERT_FILE")
|
tlsCertFile := os.Getenv("TLS_CERT_FILE")
|
||||||
tlsKeyFile := os.Getenv("TLS_KEY_FILE")
|
tlsKeyFile := os.Getenv("TLS_KEY_FILE")
|
||||||
if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") {
|
if (tlsCertFile == "" && tlsKeyFile != "") || (tlsCertFile != "" && tlsKeyFile == "") {
|
||||||
return Config{}, errors.New("Both TLS_CERT_FILE and TLS_KEY_FILE must be set")
|
return Config{}, errors.New("both TLS_CERT_FILE and TLS_KEY_FILE must be set")
|
||||||
}
|
}
|
||||||
|
|
||||||
databaseURL := os.Getenv("DATABASE_URL")
|
databaseURL := os.Getenv("DATABASE_URL")
|
||||||
|
@ -56,25 +65,43 @@ func NewFromEnv() (Config, error) {
|
||||||
return Config{}, errors.New("DATABASE_URL not set")
|
return Config{}, errors.New("DATABASE_URL not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
|
fileStoreString := os.Getenv("FILE_STORE")
|
||||||
|
var fileStore FileStore
|
||||||
|
switch os.Getenv("FILE_STORE") {
|
||||||
|
case "s3":
|
||||||
|
fileStore = S3Store
|
||||||
|
case "filesystem", "":
|
||||||
|
fileStore = FileSystemStore
|
||||||
|
default:
|
||||||
|
return Config{}, fmt.Errorf("invalid FILE_STORE value: %s", fileStoreString)
|
||||||
|
}
|
||||||
|
|
||||||
|
var awsAccessKeyID, awsSecretAccessKey, awsRegion, s3Bucket, fileStoreHTTPBasePath string
|
||||||
|
if fileStore == S3Store {
|
||||||
|
awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
|
||||||
if awsAccessKeyID == "" {
|
if awsAccessKeyID == "" {
|
||||||
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
|
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||||
if awsSecretAccessKey == "" {
|
if awsSecretAccessKey == "" {
|
||||||
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
|
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
awsRegion := os.Getenv("AWS_REGION")
|
awsRegion = os.Getenv("AWS_REGION")
|
||||||
if awsRegion == "" {
|
if awsRegion == "" {
|
||||||
return Config{}, errors.New("AWS_REGION not set")
|
return Config{}, errors.New("AWS_REGION not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
s3Bucket := os.Getenv("S3_BUCKET")
|
s3Bucket = os.Getenv("S3_BUCKET")
|
||||||
if s3Bucket == "" {
|
if s3Bucket == "" {
|
||||||
return Config{}, errors.New("S3_BUCKET not set")
|
return Config{}, errors.New("S3_BUCKET not set")
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
if fileStoreHTTPBasePath = os.Getenv("FILE_STORE_HTTP_BASE_PATH"); fileStoreHTTPBasePath == "" {
|
||||||
|
return Config{}, errors.New("FILE_STORE_HTTP_BASE_PATH not set")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
assetsHTTPBasePath := os.Getenv("ASSETS_HTTP_BASE_PATH")
|
assetsHTTPBasePath := os.Getenv("ASSETS_HTTP_BASE_PATH")
|
||||||
|
|
||||||
|
@ -84,10 +111,12 @@ func NewFromEnv() (Config, error) {
|
||||||
TLSCertFile: tlsCertFile,
|
TLSCertFile: tlsCertFile,
|
||||||
TLSKeyFile: tlsKeyFile,
|
TLSKeyFile: tlsKeyFile,
|
||||||
DatabaseURL: databaseURL,
|
DatabaseURL: databaseURL,
|
||||||
|
FileStore: fileStore,
|
||||||
AWSAccessKeyID: awsAccessKeyID,
|
AWSAccessKeyID: awsAccessKeyID,
|
||||||
AWSSecretAccessKey: awsSecretAccessKey,
|
AWSSecretAccessKey: awsSecretAccessKey,
|
||||||
AWSRegion: awsRegion,
|
AWSRegion: awsRegion,
|
||||||
S3Bucket: s3Bucket,
|
S3Bucket: s3Bucket,
|
||||||
AssetsHTTPBasePath: assetsHTTPBasePath,
|
AssetsHTTPBasePath: assetsHTTPBasePath,
|
||||||
|
FileStoreHTTPBasePath: fileStoreHTTPBasePath,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,88 @@
|
||||||
|
package filestore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FileSystemStore is a file store that stores files on the local filesystem.
|
||||||
|
// It is currently intended for usage in a development environment.
|
||||||
|
type FileSystemStore struct {
|
||||||
|
rootPath string
|
||||||
|
baseURL *url.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFileSystemStore creates a new FileSystemStore. It accepts a root path,
|
||||||
|
// which is the storage location on the local file system for stored objects,
|
||||||
|
// and a baseURL which is a URL which should be configured to serve the stored
|
||||||
|
// files over HTTP.
|
||||||
|
func NewFileSystemStore(rootPath string, baseURL string) (*FileSystemStore, error) {
|
||||||
|
url, err := url.Parse(baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error parsing URL: %v", err)
|
||||||
|
}
|
||||||
|
return &FileSystemStore{rootPath: rootPath, baseURL: url}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetObject retrieves an object from the local filesystem.
|
||||||
|
func (s *FileSystemStore) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
|
path := filepath.Join(s.rootPath, key)
|
||||||
|
fptr, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error opening file: %v", err)
|
||||||
|
}
|
||||||
|
return fptr, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type readCloser struct {
|
||||||
|
io.Reader
|
||||||
|
io.Closer
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetObjectWithRange retrieves an object from the local filesystem with the given byte range.
|
||||||
|
func (s *FileSystemStore) GetObjectWithRange(ctx context.Context, key string, start, end int64) (io.ReadCloser, error) {
|
||||||
|
path := filepath.Join(s.rootPath, key)
|
||||||
|
fptr, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error opening file: %v", err)
|
||||||
|
}
|
||||||
|
_, err = fptr.Seek(start, os.SEEK_SET)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error seeking in file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return readCloser{
|
||||||
|
Reader: io.LimitReader(fptr, end-start),
|
||||||
|
Closer: fptr,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetURL returns an HTTP URL for the provided file path.
|
||||||
|
func (s *FileSystemStore) GetURL(ctx context.Context, key string) (string, error) {
|
||||||
|
url := url.URL{}
|
||||||
|
url.Host = s.baseURL.Host
|
||||||
|
url.Path = fmt.Sprintf("%s%s", s.baseURL.Path, key)
|
||||||
|
return url.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutObject writes an object to the local filesystem.
|
||||||
|
func (s *FileSystemStore) PutObject(ctx context.Context, key string, r io.Reader, _ string) (int64, error) {
|
||||||
|
path := filepath.Join(s.rootPath, key)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
|
||||||
|
return 0, fmt.Errorf("error creating directories: %v", err)
|
||||||
|
}
|
||||||
|
fptr, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("error opening file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := io.Copy(fptr, r)
|
||||||
|
if err != nil {
|
||||||
|
return n, fmt.Errorf("error writing file: %v", err)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
|
@ -248,10 +248,17 @@ func Start(options Options) error {
|
||||||
|
|
||||||
log := options.Logger.Sugar()
|
log := options.Logger.Sugar()
|
||||||
fileHandler := http.NotFoundHandler()
|
fileHandler := http.NotFoundHandler()
|
||||||
|
|
||||||
|
// Enabling the file system store disables serving assets over HTTP.
|
||||||
|
// TODO: fix this.
|
||||||
if options.Config.AssetsHTTPBasePath != "" {
|
if options.Config.AssetsHTTPBasePath != "" {
|
||||||
log.With("basePath", options.Config.AssetsHTTPBasePath).Info("Configured to serve assets over HTTP")
|
log.With("basePath", options.Config.AssetsHTTPBasePath).Info("Configured to serve assets over HTTP")
|
||||||
fileHandler = http.FileServer(http.Dir(options.Config.AssetsHTTPBasePath))
|
fileHandler = http.FileServer(http.Dir(options.Config.AssetsHTTPBasePath))
|
||||||
}
|
}
|
||||||
|
if options.Config.FileStoreHTTPBasePath != "" {
|
||||||
|
log.With("basePath", options.Config.FileStoreHTTPBasePath).Info("Configured to serve file store over HTTP")
|
||||||
|
fileHandler = http.FileServer(http.Dir(options.Config.FileStoreHTTPBasePath))
|
||||||
|
}
|
||||||
|
|
||||||
httpServer := http.Server{
|
httpServer := http.Server{
|
||||||
Addr: options.Config.BindAddr,
|
Addr: options.Config.BindAddr,
|
||||||
|
|
Loading…
Reference in New Issue