rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Add types.go, small refactor and tidy

This commit is contained in:
Rob Watson 2021-11-29 16:06:43 +01:00
parent 1552fc19a1
commit b3bc63621a
3 changed files with 124 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
backend/media/media_set.go
View File

@ -1,55 +0,0 @@
package media
import (
"fmt"
"time"
"github.com/google/uuid"
)
const SizeOfInt16 = 2
type Audio struct {
ContentLength int64
Channels int
// ApproxFrames is used during initial processing when a precise frame count
// cannot be determined. Prefer Frames in all other cases.
ApproxFrames int64
Frames int64
SampleRate int
YoutubeItag int
MimeType string
}
type Video struct {
ContentLength int64
Duration time.Duration
// not sure if this are needed any more?
ThumbnailWidth int
ThumbnailHeight int
YoutubeItag int
MimeType string
}
// MediaSet represents the media and metadata associated with a single media
// resource (for example, a YouTube video).
type MediaSet struct {
Audio Audio `json:"audio"`
Video Video `json:"video"`
ID uuid.UUID `json:"id"`
YoutubeID string `json:"youtube_id"`
exists bool
}
// New builds a new MediaSet with the given ID.
func NewMediaSet(youtubeID string) *MediaSet {
return &MediaSet{YoutubeID: youtubeID}
}
// TODO: pass io.Readers/Writers instead of strings.
func (m *MediaSet) RawAudioPath() string { return fmt.Sprintf("cache/%s.raw", m.YoutubeID) }
func (m *MediaSet) EncodedAudioPath() string { return fmt.Sprintf("cache/%s.m4a", m.YoutubeID) }
func (m *MediaSet) VideoPath() string { return fmt.Sprintf("cache/%s.mp4", m.YoutubeID) }
func (m *MediaSet) ThumbnailPath() string { return fmt.Sprintf("cache/%s.jpg", m.YoutubeID) }
func (m *MediaSet) MetadataPath() string { return fmt.Sprintf("cache/%s.json", m.YoutubeID) }

109
backend/media/service.go
View File

@ -15,7 +15,6 @@ import (
"git.netflux.io/rob/clipper/config" "git.netflux.io/rob/clipper/config"
"git.netflux.io/rob/clipper/generated/store" "git.netflux.io/rob/clipper/generated/store"
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
@ -40,69 +39,6 @@ const (
thumbnailHeight = 100 // " thumbnailHeight = 100 // "
) )
// progressReader is a reader that prints progress logs as it reads.
type progressReader struct {
io.Reader
label string
total, exp int64
logger *zap.SugaredLogger
}
func newProgressReader(reader io.Reader, label string, exp int64, logger *zap.SugaredLogger) *progressReader {
return &progressReader{
Reader: reader,
exp: exp,
logger: logger.Named(fmt.Sprintf("ProgressReader %s", label)),
}
}
func (r *progressReader) Read(p []byte) (int, error) {
n, err := r.Reader.Read(p)
r.total += int64(n)
r.logger.Debugf("Read %d of %d (%.02f%%) bytes from the provided reader", r.total, r.exp, (float32(r.total)/float32(r.exp))*100.0)
return n, err
}
// Store wraps a database store.
type Store interface {
GetMediaSet(context.Context, uuid.UUID) (store.MediaSet, error)
GetMediaSetByYoutubeID(context.Context, string) (store.MediaSet, error)
CreateMediaSet(context.Context, store.CreateMediaSetParams) (store.MediaSet, error)
SetRawAudioUploaded(context.Context, store.SetRawAudioUploadedParams) (store.MediaSet, error)
SetEncodedAudioUploaded(context.Context, store.SetEncodedAudioUploadedParams) (store.MediaSet, error)
SetVideoUploaded(context.Context, store.SetVideoUploadedParams) (store.MediaSet, error)
SetVideoThumbnailUploaded(context.Context, store.SetVideoThumbnailUploadedParams) (store.MediaSet, error)
}
// S3API provides an API to AWS S3.
type S3API struct {
S3Client
S3PresignClient
}
// S3Client wraps the AWS S3 service client.
type S3Client interface {
GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
UploadPart(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
}
// S3PresignClient wraps the AWS S3 Presign client.
type S3PresignClient interface {
PresignGetObject(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*signerv4.PresignedHTTPRequest, error)
}
// YoutubeClient wraps the youtube.Client client.
type YoutubeClient interface {
GetVideoContext(context.Context, string) (*youtubev2.Video, error)
GetStreamContext(context.Context, *youtubev2.Video, *youtubev2.Format) (io.ReadCloser, int64, error)
}
// MediaSetService exposes logical flows handling MediaSets. // MediaSetService exposes logical flows handling MediaSets.
type MediaSetService struct { type MediaSetService struct {
store Store store Store
@ -123,7 +59,9 @@ func NewMediaSetService(store Store, youtubeClient YoutubeClient, s3API S3API, c
} }
// Get fetches the metadata for a given MediaSet source. If it does not exist // Get fetches the metadata for a given MediaSet source. If it does not exist
// in the local DB, it will attempt to create it. // in the local DB, it will attempt to create it. After the resource has been
// created, other endpoints (e.g. GetAudio) can be called to fetch media from
// Youtube and store it in S3.
func (s *MediaSetService) Get(ctx context.Context, youtubeID string) (*MediaSet, error) { func (s *MediaSetService) Get(ctx context.Context, youtubeID string) (*MediaSet, error) {
var ( var (
mediaSet *MediaSet mediaSet *MediaSet
@ -132,13 +70,13 @@ func (s *MediaSetService) Get(ctx context.Context, youtubeID string) (*MediaSet,
mediaSet, err = s.findMediaSet(ctx, youtubeID) mediaSet, err = s.findMediaSet(ctx, youtubeID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error getting existing media set: %v", err) return nil, fmt.Errorf("error finding existing media set: %v", err)
} }
if mediaSet == nil { if mediaSet == nil {
mediaSet, err = s.createMediaSet(ctx, youtubeID) mediaSet, err = s.createMediaSet(ctx, youtubeID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error getting new media set: %v", err) return nil, fmt.Errorf("error creating new media set: %v", err)
} }
} }
@ -281,11 +219,11 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP
return nil, fmt.Errorf("error getting media set: %v", err) return nil, fmt.Errorf("error getting media set: %v", err)
} }
// TODO: use mediaSet func to fetch s3Key
s3Key := fmt.Sprintf("media_sets/%s/video.mp4", mediaSet.ID)
if mediaSet.VideoS3UploadedAt.Valid { if mediaSet.VideoS3UploadedAt.Valid {
input := s3.GetObjectInput{Bucket: aws.String(s.config.S3Bucket), Key: aws.String(s3Key)} input := s3.GetObjectInput{
Bucket: aws.String(s.config.S3Bucket),
Key: aws.String(mediaSet.VideoS3Key.String),
}
request, signErr := s.s3.PresignGetObject(ctx, &input, s3.WithPresignExpires(getVideoExpiresIn)) request, signErr := s.s3.PresignGetObject(ctx, &input, s3.WithPresignExpires(getVideoExpiresIn))
if signErr != nil { if signErr != nil {
return nil, fmt.Errorf("error generating presigned URL: %v", signErr) return nil, fmt.Errorf("error generating presigned URL: %v", signErr)
@ -309,6 +247,9 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP
return nil, fmt.Errorf("error fetching stream: %v", err) return nil, fmt.Errorf("error fetching stream: %v", err)
} }
// TODO: use mediaSet func to fetch s3Key
s3Key := fmt.Sprintf("media_sets/%s/video.mp4", mediaSet.ID)
videoGetter := newVideoGetter(s.s3, s.store, s.logger) videoGetter := newVideoGetter(s.s3, s.store, s.logger)
return videoGetter.GetVideo( return videoGetter.GetVideo(
ctx, ctx,
@ -654,3 +595,29 @@ func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet
return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil
} }
// progressReader is a reader that prints progress logs as it reads.
type progressReader struct {
io.Reader
label string
total, exp int64
logger *zap.SugaredLogger
}
func newProgressReader(reader io.Reader, label string, exp int64, logger *zap.SugaredLogger) *progressReader {
return &progressReader{
Reader: reader,
exp: exp,
logger: logger.Named(fmt.Sprintf("ProgressReader %s", label)),
}
}
func (r *progressReader) Read(p []byte) (int, error) {
n, err := r.Reader.Read(p)
r.total += int64(n)
r.logger.Debugf("Read %d of %d (%.02f%%) bytes from the provided reader", r.total, r.exp, (float32(r.total)/float32(r.exp))*100.0)
return n, err
}

86
backend/media/types.go Normal file
View File

@ -0,0 +1,86 @@
package media
import (
"context"
"io"
"time"
"git.netflux.io/rob/clipper/generated/store"
signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
youtubev2 "github.com/kkdai/youtube/v2"
)
// An int16 has two bytes.
const SizeOfInt16 = 2
// MediaSet represents the media and metadata associated with a single media
// resource (for example, a YouTube video).
type MediaSet struct {
Audio Audio
Video Video
ID uuid.UUID
YoutubeID string
}
// Audio contains the metadata for the audio part of the media set.
type Audio struct {
ContentLength int64
Channels int
// ApproxFrames is used during initial processing when a precise frame count
// cannot be determined. Prefer Frames in all other cases.
ApproxFrames int64
Frames int64
SampleRate int
YoutubeItag int
MimeType string
}
// Video contains the metadata for the video part of the media set.
type Video struct {
ContentLength int64
Duration time.Duration
// not sure if this are needed any more?
ThumbnailWidth int
ThumbnailHeight int
YoutubeItag int
MimeType string
}
// Store wraps a database store.
type Store interface {
GetMediaSet(context.Context, uuid.UUID) (store.MediaSet, error)
GetMediaSetByYoutubeID(context.Context, string) (store.MediaSet, error)
CreateMediaSet(context.Context, store.CreateMediaSetParams) (store.MediaSet, error)
SetRawAudioUploaded(context.Context, store.SetRawAudioUploadedParams) (store.MediaSet, error)
SetEncodedAudioUploaded(context.Context, store.SetEncodedAudioUploadedParams) (store.MediaSet, error)
SetVideoUploaded(context.Context, store.SetVideoUploadedParams) (store.MediaSet, error)
SetVideoThumbnailUploaded(context.Context, store.SetVideoThumbnailUploadedParams) (store.MediaSet, error)
}
// S3API provides an API to AWS S3.
type S3API struct {
S3Client
S3PresignClient
}
// S3Client wraps the AWS S3 service client.
type S3Client interface {
GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
UploadPart(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
}
// S3PresignClient wraps the AWS S3 Presign client.
type S3PresignClient interface {
PresignGetObject(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*signerv4.PresignedHTTPRequest, error)
}
// YoutubeClient wraps the youtube.Client client.
type YoutubeClient interface {
GetVideoContext(context.Context, string) (*youtubev2.Video, error)
GetStreamContext(context.Context, *youtubev2.Video, *youtubev2.Format) (io.ReadCloser, int64, error)
}