diff --git a/backend/media/service.go b/backend/media/service.go index 9e28ac0..d7f1073 100644 --- a/backend/media/service.go +++ b/backend/media/service.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "log" + "net/http" "strconv" "time" @@ -57,11 +58,12 @@ func (pw *progressReader) Read(p []byte) (int, error) { // Store wraps a database store. type Store interface { - GetMediaSet(ctx context.Context, id uuid.UUID) (store.MediaSet, error) - GetMediaSetByYoutubeID(ctx context.Context, youtubeID string) (store.MediaSet, error) - CreateMediaSet(ctx context.Context, arg store.CreateMediaSetParams) (store.MediaSet, error) - SetAudioUploaded(ctx context.Context, arg store.SetAudioUploadedParams) (store.MediaSet, error) - SetVideoUploaded(ctx context.Context, arg store.SetVideoUploadedParams) (store.MediaSet, error) + GetMediaSet(context.Context, uuid.UUID) (store.MediaSet, error) + GetMediaSetByYoutubeID(context.Context, string) (store.MediaSet, error) + CreateMediaSet(context.Context, store.CreateMediaSetParams) (store.MediaSet, error) + SetAudioUploaded(context.Context, store.SetAudioUploadedParams) (store.MediaSet, error) + SetVideoUploaded(context.Context, store.SetVideoUploadedParams) (store.MediaSet, error) + SetVideoThumbnailUploaded(context.Context, store.SetVideoThumbnailUploadedParams) (store.MediaSet, error) } // S3API provides an API to AWS S3. @@ -592,6 +594,10 @@ func sqlInt64(i int64) sql.NullInt64 { return sql.NullInt64{Int64: i, Valid: true} } +func sqlInt32(i int32) sql.NullInt32 { + return sql.NullInt32{Int32: i, Valid: true} +} + // ModuloBufReader reads from a reader in block sizes that are exactly modulo // modSize, with any remainder buffered until the next read. type ModuloBufReader struct { @@ -628,3 +634,95 @@ func (r *ModuloBufReader) Read(p []byte) (int, error) { return nr - rem, err } + +type VideoThumbnail struct { + Data []byte + Width, Height int +} + +func (s *MediaSetService) GetVideoThumbnail(ctx context.Context, id uuid.UUID) (VideoThumbnail, error) { + mediaSet, err := s.store.GetMediaSet(ctx, id) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error getting media set: %v", err) + } + + if mediaSet.VideoThumbnailS3UploadedAt.Valid { + return s.getThumbnailFromS3(ctx, mediaSet) + } + + return s.getThumbnailFromYoutube(ctx, mediaSet) +} + +func (s *MediaSetService) getThumbnailFromS3(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { + input := s3.GetObjectInput{ + Bucket: aws.String(mediaSet.VideoThumbnailS3Bucket.String), + Key: aws.String(mediaSet.VideoThumbnailS3Key.String), + } + output, err := s.s3.GetObject(ctx, &input) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail from s3: %v", err) + } + defer output.Body.Close() + + imageData, err := io.ReadAll(output.Body) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error reading thumbnail from s3: %v", err) + } + + return VideoThumbnail{ + Width: int(mediaSet.VideoThumbnailWidth.Int32), + Height: int(mediaSet.VideoThumbnailHeight.Int32), + Data: imageData, + }, nil +} + +func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { + video, err := s.youtube.GetVideoContext(ctx, mediaSet.YoutubeID) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching video: %v", err) + } + + if len(video.Formats) == 0 { + return VideoThumbnail{}, errors.New("no format available") + } + + thumbnails := video.Thumbnails + SortYoutubeThumbnails(thumbnails) + thumbnail := thumbnails[0] + + resp, err := http.Get(thumbnail.URL) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail: %v", err) + } + defer resp.Body.Close() + + imageData, err := io.ReadAll(resp.Body) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error reading thumbnail: %v", err) + } + + // TODO: use mediaSet func to fetch s3Key + s3Key := fmt.Sprintf("media_sets/%s/thumbnail.jpg", mediaSet.ID) + + uploader := newMultipartUploader(s.s3) + const mimeType = "application/jpeg" + + _, err = uploader.Upload(ctx, bytes.NewReader(imageData), s3Bucket, s3Key, mimeType) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err) + } + + storeParams := store.SetVideoThumbnailUploadedParams{ + ID: mediaSet.ID, + VideoThumbnailMimeType: sqlString(mimeType), + VideoThumbnailS3Bucket: sqlString(s3Bucket), + VideoThumbnailS3Key: sqlString(s3Key), + VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)), + VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)), + } + if _, err := s.store.SetVideoThumbnailUploaded(ctx, storeParams); err != nil { + return VideoThumbnail{}, fmt.Errorf("error updating media set: %v", err) + } + + return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil +} diff --git a/backend/media/youtube.go b/backend/media/youtube.go index 0d13849..8d23307 100644 --- a/backend/media/youtube.go +++ b/backend/media/youtube.go @@ -51,3 +51,16 @@ func FilterYoutubeVideo(inFormats youtubev2.FormatList) youtubev2.FormatList { }) return formats } + +// SortYoutubeThumbnails sorts the provided thumbnails ordered in descending preferred order. +func SortYoutubeThumbnails(thumbnails youtubev2.Thumbnails) { + sort.SliceStable(thumbnails, func(i, j int) bool { + // TODO: get rid of these magic 177s. + isMinSizeI := thumbnails[i].Width >= 177 + isMinSizeJ := thumbnails[j].Width >= 177 + if isMinSizeI && isMinSizeJ { + return thumbnails[i].Width < thumbnails[j].Width + } + return isMinSizeI + }) +} diff --git a/backend/media/youtube_test.go b/backend/media/youtube_test.go index a4406d6..cab039b 100644 --- a/backend/media/youtube_test.go +++ b/backend/media/youtube_test.go @@ -86,3 +86,31 @@ func TestFilterVideo(t *testing.T) { assert.Equal(t, formats[2], sortedFormats[1]) assert.Equal(t, formats[0], sortedFormats[2]) } + +func TestSortThumbnails(t *testing.T) { + thumbnails := []youtubev2.Thumbnail{ + { + Width: 250, + Height: 100, + }, + { + Width: 1920, + Height: 1080, + }, + { + Width: 50, + Height: 33, + }, + { + Width: 75, + Height: 51, + }, + } + + media.SortYoutubeThumbnails(thumbnails) + + assert.Equal(t, uint(250), thumbnails[0].Width) + assert.Equal(t, uint(1920), thumbnails[1].Width) + assert.Equal(t, uint(50), thumbnails[2].Width) + assert.Equal(t, uint(75), thumbnails[3].Width) +} diff --git a/backend/server/server.go b/backend/server/server.go index e0269a4..946edcd 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -211,6 +211,26 @@ func (c *mediaSetServiceController) GetVideo(request *pbmediaset.GetVideoRequest return nil } +func (c *mediaSetServiceController) GetVideoThumbnail(ctx context.Context, request *pbmediaset.GetVideoThumbnailRequest) (*pbmediaset.GetVideoThumbnailResponse, error) { + id, err := uuid.Parse(request.GetId()) + if err != nil { + return nil, newResponseError(err) + } + + thumbnail, err := c.mediaSetService.GetVideoThumbnail(ctx, id) + if err != nil { + return nil, newResponseError(err) + } + + response := pbmediaset.GetVideoThumbnailResponse{ + Image: thumbnail.Data, + Width: int32(thumbnail.Width), + Height: int32(thumbnail.Height), + } + + return &response, nil +} + func Start(options Options) error { logger, err := buildLogger(options.Environment) if err != nil { diff --git a/backend/sql/queries.sql b/backend/sql/queries.sql index 3475cbc..b8c2ed5 100644 --- a/backend/sql/queries.sql +++ b/backend/sql/queries.sql @@ -20,3 +20,9 @@ UPDATE media_sets SET video_s3_bucket = $2, video_s3_key = $3, video_s3_uploaded_at = NOW(), updated_at = NOW() WHERE id = $1 RETURNING *; + +-- name: SetVideoThumbnailUploaded :one +UPDATE media_sets +SET video_thumbnail_width = $2, video_thumbnail_height = $3, video_thumbnail_mime_type = $4, video_thumbnail_s3_bucket = $5, video_thumbnail_s3_key = $6, video_thumbnail_s3_uploaded_at = NOW(), updated_at = NOW() +WHERE id = $1 +RETURNING *; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1b4b127..aa47fb9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -181,6 +181,7 @@ function App(): JSX.Element { /> = ({ + mediaSet, position, duration, height, @@ -17,6 +21,53 @@ export const VideoPreview: React.FC = ({ // effects + // load thumbnail, to display when the component is loaded for the first + // time. This is needed because of browser autoplay limitations. + useEffect(() => { + (async function () { + if (mediaSet == null) { + return; + } + + const canvas = videoCanvasRef.current; + if (canvas == null) { + console.error('no canvas ref available'); + return; + } + + const ctx = canvas.getContext('2d'); + if (ctx == null) { + console.error('no 2d context available'); + return; + } + + // Set aspect ratio. + canvas.width = canvas.height * (canvas.clientWidth / canvas.clientHeight); + + console.log('getting video thumbnail...'); + const rpc = newRPC(); + const service = new MediaSetServiceClientImpl(rpc); + + const thumbnail = await service.GetVideoThumbnail({ id: mediaSet.id }); + + console.log('got thumbnail', thumbnail); + + const url = URL.createObjectURL( + new Blob([thumbnail.image], { type: 'image/jpeg' }) + ); + const img = new Image(thumbnail.width, thumbnail.height); + + img.src = url; + console.log('img', img); + img.onerror = console.error; + img.onload = () => { + ctx.drawImage(img, 0, 0, 177, 100); + }; + + console.log('set src to', url); + })(); + }, [mediaSet]); + // render canvas useEffect(() => { // TODO: not sure if requestAnimationFrame is recommended here. diff --git a/proto/media_set.proto b/proto/media_set.proto index 1b9acd8..cf97cf4 100644 --- a/proto/media_set.proto +++ b/proto/media_set.proto @@ -56,9 +56,20 @@ message GetVideoProgress { string url = 2; } +message GetVideoThumbnailRequest { + string id = 1; +} + +message GetVideoThumbnailResponse { + bytes image = 1; + int32 width = 2; + int32 height = 3; +} + service MediaSetService { rpc Get(GetRequest) returns (MediaSet) {} rpc GetAudio(GetAudioRequest) returns (stream GetAudioProgress) {} rpc GetAudioSegment(GetAudioSegmentRequest) returns (GetAudioSegmentResponse) {} rpc GetVideo(GetVideoRequest) returns (stream GetVideoProgress) {} + rpc GetVideoThumbnail(GetVideoThumbnailRequest) returns (GetVideoThumbnailResponse) {} }