rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Add config package

This commit is contained in:
Rob Watson 2021-11-22 19:26:51 +01:00
parent e7c76d0c6b
commit b643ea2824
6 changed files with 117 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/.env.example
View File

@ -1,4 +1,8 @@
ENV=development # or production
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
S3_BUCKET=
DATABASE_URL=

25
backend/cmd/clipper/main.go
View File

@ -4,13 +4,14 @@ import (
"context"
"database/sql"
"log"
"os"
"time"
"git.netflux.io/rob/clipper/config"
"git.netflux.io/rob/clipper/generated/store"
"git.netflux.io/rob/clipper/media"
"git.netflux.io/rob/clipper/server"
"github.com/aws/aws-sdk-go-v2/config"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/kkdai/youtube/v2"
_ "github.com/lib/pq"
@ -24,20 +25,26 @@ const (
func main() {
ctx := context.Background()
// Create a store
databaseURL := os.Getenv("DATABASE_URL")
if databaseURL == "" {
log.Fatal("DATABASE_URL not set")
config, err := config.NewFromEnv()
if err != nil {
log.Fatalf("error loading config: %v", err)
}
db, err := sql.Open("postgres", databaseURL)
// Create a store
db, err := sql.Open("postgres", config.DatabaseURL)
if err != nil {
log.Fatal(err)
}
store := store.New(db)
// Create an Amazon S3 service s3Client
cfg, err := config.LoadDefaultConfig(ctx)
cfg, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(config.AWSAccessKeyID, config.AWSSecretAccessKey, ""),
),
awsconfig.WithRegion(config.AWSRegion),
)
if err != nil {
log.Fatal(err)
}
@ -50,7 +57,7 @@ func main() {
var youtubeClient youtube.Client
serverOptions := server.Options{
Environment: server.Development,
Config: config,
BindAddr: DefaultHTTPBindAddr,
Timeout: DefaultTimeout,
Store: store,

72
backend/config/config.go Normal file
View File

@ -0,0 +1,72 @@
package config
import (
"errors"
"fmt"
"os"
)
type Environment int
const (
EnvDevelopment Environment = iota
EnvProduction
)
type Config struct {
Environment Environment
DatabaseURL string
AWSAccessKeyID string
AWSSecretAccessKey string
AWSRegion string
S3Bucket string
}
func NewFromEnv() (Config, error) {
envString := os.Getenv("ENV")
var env Environment
switch envString {
case "production":
env = EnvProduction
case "development":
env = EnvDevelopment
case "":
return Config{}, errors.New("ENV not set")
default:
return Config{}, fmt.Errorf("invalid ENVIRONMENT value: %s", envString)
}
databaseURL := os.Getenv("DATABASE_URL")
if databaseURL == "" {
return Config{}, errors.New("DATABASE_URL not set")
}
awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
if awsAccessKeyID == "" {
return Config{}, errors.New("AWS_ACCESS_KEY_ID not set")
}
awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if awsSecretAccessKey == "" {
return Config{}, errors.New("AWS_SECRET_ACCESS_KEY not set")
}
awsRegion := os.Getenv("AWS_REGION")
if awsRegion == "" {
return Config{}, errors.New("AWS_REGION not set")
}
s3Bucket := os.Getenv("S3_BUCKET")
if s3Bucket == "" {
return Config{}, errors.New("S3_BUCKET not set")
}
return Config{
Environment: env,
DatabaseURL: databaseURL,
AWSAccessKeyID: awsAccessKeyID,
AWSSecretAccessKey: awsSecretAccessKey,
AWSRegion: awsRegion,
S3Bucket: s3Bucket,
}, nil
}

17
backend/media/service.go
View File

@ -13,6 +13,7 @@ import (
"strconv"
"time"
"git.netflux.io/rob/clipper/config"
"git.netflux.io/rob/clipper/generated/store"
"github.com/aws/aws-sdk-go-v2/aws"
signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
@ -23,8 +24,6 @@ import (
)
const (
s3Bucket = "clipper-development"
getVideoExpiresIn = time.Hour * 1
)
@ -97,14 +96,16 @@ type MediaSetService struct {
store Store
youtube YoutubeClient
s3 S3API
config config.Config
logger *zap.SugaredLogger
}
func NewMediaSetService(store Store, youtubeClient YoutubeClient, s3API S3API, logger *zap.Logger) *MediaSetService {
func NewMediaSetService(store Store, youtubeClient YoutubeClient, s3API S3API, config config.Config, logger *zap.Logger) *MediaSetService {
return &MediaSetService{
store: store,
youtube: youtubeClient,
s3: s3API,
config: config,
logger: logger.Sugar(),
}
}
@ -269,7 +270,7 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP
s3Key := fmt.Sprintf("media_sets/%s/video.mp4", mediaSet.ID)
if mediaSet.VideoS3UploadedAt.Valid {
input := s3.GetObjectInput{Bucket: aws.String(s3Bucket), Key: aws.String(s3Key)}
input := s3.GetObjectInput{Bucket: aws.String(s.config.S3Bucket), Key: aws.String(s3Key)}
request, signErr := s.s3.PresignGetObject(ctx, &input, s3.WithPresignExpires(getVideoExpiresIn))
if signErr != nil {
return nil, fmt.Errorf("error generating presigned URL: %v", signErr)
@ -299,7 +300,7 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP
stream,
format.ContentLength,
mediaSet.ID,
s3Bucket,
s.config.S3Bucket,
s3Key,
format.MimeType,
)
@ -436,7 +437,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor
getAudioProgressReader: getAudioProgressReader,
ffmpegReader: ffmpegReader,
uploader: uploader,
s3Bucket: s3Bucket,
s3Bucket: s.config.S3Bucket,
s3Key: s3Key,
store: s.store,
channels: format.AudioChannels,
@ -707,7 +708,7 @@ func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet
uploader := newMultipartUploader(s.s3)
const mimeType = "application/jpeg"
_, err = uploader.Upload(ctx, bytes.NewReader(imageData), s3Bucket, s3Key, mimeType)
_, err = uploader.Upload(ctx, bytes.NewReader(imageData), s.config.S3Bucket, s3Key, mimeType)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err)
}
@ -715,7 +716,7 @@ func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet
storeParams := store.SetVideoThumbnailUploadedParams{
ID: mediaSet.ID,
VideoThumbnailMimeType: sqlString(mimeType),
VideoThumbnailS3Bucket: sqlString(s3Bucket),
VideoThumbnailS3Bucket: sqlString(s.config.S3Bucket),
VideoThumbnailS3Key: sqlString(s3Key),
VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)),
VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)),

1
backend/media/uploader.go
View File

@ -201,6 +201,7 @@ outer:
}
if _, err = u.s3.CompleteMultipartUpload(ctx, &completeInput); err != nil {
log.Printf("parts: %+v", completedParts)
return 0, fmt.Errorf("error completing upload: %v", err)
}

30
backend/server/server.go
View File

@ -7,6 +7,7 @@ import (
"net/http"
"time"
"git.netflux.io/rob/clipper/config"
pbmediaset "git.netflux.io/rob/clipper/generated/pb/media_set"
"git.netflux.io/rob/clipper/media"
"github.com/google/uuid"
@ -58,15 +59,8 @@ func newResponseError(err error) *ResponseError {
return &ResponseError{err: err, s: defaultResponseMessage}
}
type Environment int
const (
Development Environment = iota
Production
)
type Options struct {
Environment Environment
Config config.Config
BindAddr string
Timeout time.Duration
Store media.Store
@ -232,15 +226,21 @@ func (c *mediaSetServiceController) GetVideoThumbnail(ctx context.Context, reque
}
func Start(options Options) error {
logger, err := buildLogger(options.Environment)
logger, err := buildLogger(options.Config)
if err != nil {
return fmt.Errorf("error building logger: %v", err)
}
defer logger.Sync()
fetchMediaSetService := media.NewMediaSetService(options.Store, options.YoutubeClient, options.S3API, logger)
fetchMediaSetService := media.NewMediaSetService(
options.Store,
options.YoutubeClient,
options.S3API,
options.Config,
logger,
)
grpcServer := buildGRPCServer(options, logger)
grpcServer := buildGRPCServer(options.Config, logger)
mediaSetController := &mediaSetServiceController{mediaSetService: fetchMediaSetService, logger: logger.Sugar().Named("controller")}
pbmediaset.RegisterMediaSetServiceServer(grpcServer, mediaSetController)
@ -257,21 +257,21 @@ func Start(options Options) error {
return httpServer.ListenAndServe()
}
func buildLogger(env Environment) (*zap.Logger, error) {
if env == Production {
func buildLogger(c config.Config) (*zap.Logger, error) {
if c.Environment == config.EnvProduction {
return zap.NewProduction()
}
return zap.NewDevelopment()
}
func buildGRPCServer(options Options, logger *zap.Logger) *grpc.Server {
func buildGRPCServer(c config.Config, logger *zap.Logger) *grpc.Server {
unaryInterceptors := []grpc.UnaryServerInterceptor{
grpczap.UnaryServerInterceptor(logger),
}
streamInterceptors := []grpc.StreamServerInterceptor{
grpczap.StreamServerInterceptor(logger),
}
if options.Environment == Production {
if c.Environment == config.EnvProduction {
panicOpts := []grpcrecovery.Option{
grpcrecovery.WithRecoveryHandler(func(p interface{}) error {
return newResponseError(fmt.Errorf("%v", p))