Add video thumbnail support

backend/media/service.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"strconv"
 	"time"
 
@@ -57,11 +58,12 @@ func (pw *progressReader) Read(p []byte) (int, error) {
 
 // Store wraps a database store.
 type Store interface {
-	GetMediaSet(ctx context.Context, id uuid.UUID) (store.MediaSet, error)
+	GetMediaSet(context.Context, uuid.UUID) (store.MediaSet, error)
-	GetMediaSetByYoutubeID(ctx context.Context, youtubeID string) (store.MediaSet, error)
+	GetMediaSetByYoutubeID(context.Context, string) (store.MediaSet, error)
-	CreateMediaSet(ctx context.Context, arg store.CreateMediaSetParams) (store.MediaSet, error)
+	CreateMediaSet(context.Context, store.CreateMediaSetParams) (store.MediaSet, error)
-	SetAudioUploaded(ctx context.Context, arg store.SetAudioUploadedParams) (store.MediaSet, error)
+	SetAudioUploaded(context.Context, store.SetAudioUploadedParams) (store.MediaSet, error)
-	SetVideoUploaded(ctx context.Context, arg store.SetVideoUploadedParams) (store.MediaSet, error)
+	SetVideoUploaded(context.Context, store.SetVideoUploadedParams) (store.MediaSet, error)
+	SetVideoThumbnailUploaded(context.Context, store.SetVideoThumbnailUploadedParams) (store.MediaSet, error)
 }
 
 // S3API provides an API to AWS S3.
@@ -592,6 +594,10 @@ func sqlInt64(i int64) sql.NullInt64 {
 	return sql.NullInt64{Int64: i, Valid: true}
 }
 
+func sqlInt32(i int32) sql.NullInt32 {
+	return sql.NullInt32{Int32: i, Valid: true}
+}
+
 // ModuloBufReader reads from a reader in block sizes that are exactly modulo
 // modSize, with any remainder buffered until the next read.
 type ModuloBufReader struct {
@@ -628,3 +634,95 @@ func (r *ModuloBufReader) Read(p []byte) (int, error) {
 
 	return nr - rem, err
 }
+
+type VideoThumbnail struct {
+	Data          []byte
+	Width, Height int
+}
+
+func (s *MediaSetService) GetVideoThumbnail(ctx context.Context, id uuid.UUID) (VideoThumbnail, error) {
+	mediaSet, err := s.store.GetMediaSet(ctx, id)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error getting media set: %v", err)
+	}
+
+	if mediaSet.VideoThumbnailS3UploadedAt.Valid {
+		return s.getThumbnailFromS3(ctx, mediaSet)
+	}
+
+	return s.getThumbnailFromYoutube(ctx, mediaSet)
+}
+
+func (s *MediaSetService) getThumbnailFromS3(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) {
+	input := s3.GetObjectInput{
+		Bucket: aws.String(mediaSet.VideoThumbnailS3Bucket.String),
+		Key:    aws.String(mediaSet.VideoThumbnailS3Key.String),
+	}
+	output, err := s.s3.GetObject(ctx, &input)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail from s3: %v", err)
+	}
+	defer output.Body.Close()
+
+	imageData, err := io.ReadAll(output.Body)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error reading thumbnail from s3: %v", err)
+	}
+
+	return VideoThumbnail{
+		Width:  int(mediaSet.VideoThumbnailWidth.Int32),
+		Height: int(mediaSet.VideoThumbnailHeight.Int32),
+		Data:   imageData,
+	}, nil
+}
+
+func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) {
+	video, err := s.youtube.GetVideoContext(ctx, mediaSet.YoutubeID)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error fetching video: %v", err)
+	}
+
+	if len(video.Formats) == 0 {
+		return VideoThumbnail{}, errors.New("no format available")
+	}
+
+	thumbnails := video.Thumbnails
+	SortYoutubeThumbnails(thumbnails)
+	thumbnail := thumbnails[0]
+
+	resp, err := http.Get(thumbnail.URL)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail: %v", err)
+	}
+	defer resp.Body.Close()
+
+	imageData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error reading thumbnail: %v", err)
+	}
+
+	// TODO: use mediaSet func to fetch s3Key
+	s3Key := fmt.Sprintf("media_sets/%s/thumbnail.jpg", mediaSet.ID)
+
+	uploader := newMultipartUploader(s.s3)
+	const mimeType = "application/jpeg"
+
+	_, err = uploader.Upload(ctx, bytes.NewReader(imageData), s3Bucket, s3Key, mimeType)
+	if err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err)
+	}
+
+	storeParams := store.SetVideoThumbnailUploadedParams{
+		ID:                     mediaSet.ID,
+		VideoThumbnailMimeType: sqlString(mimeType),
+		VideoThumbnailS3Bucket: sqlString(s3Bucket),
+		VideoThumbnailS3Key:    sqlString(s3Key),
+		VideoThumbnailWidth:    sqlInt32(int32(thumbnail.Width)),
+		VideoThumbnailHeight:   sqlInt32(int32(thumbnail.Height)),
+	}
+	if _, err := s.store.SetVideoThumbnailUploaded(ctx, storeParams); err != nil {
+		return VideoThumbnail{}, fmt.Errorf("error updating media set: %v", err)
+	}
+
+	return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil
+}

backend/media/youtube.go

@@ -51,3 +51,16 @@ func FilterYoutubeVideo(inFormats youtubev2.FormatList) youtubev2.FormatList {
 	})
 	return formats
 }
+
+// SortYoutubeThumbnails sorts the provided thumbnails ordered in descending preferred order.
+func SortYoutubeThumbnails(thumbnails youtubev2.Thumbnails) {
+	sort.SliceStable(thumbnails, func(i, j int) bool {
+		// TODO: get rid of these magic 177s.
+		isMinSizeI := thumbnails[i].Width >= 177
+		isMinSizeJ := thumbnails[j].Width >= 177
+		if isMinSizeI && isMinSizeJ {
+			return thumbnails[i].Width < thumbnails[j].Width
+		}
+		return isMinSizeI
+	})
+}

backend/media/youtube_test.go

@@ -86,3 +86,31 @@ func TestFilterVideo(t *testing.T) {
 	assert.Equal(t, formats[2], sortedFormats[1])
 	assert.Equal(t, formats[0], sortedFormats[2])
 }
+
+func TestSortThumbnails(t *testing.T) {
+	thumbnails := []youtubev2.Thumbnail{
+		{
+			Width:  250,
+			Height: 100,
+		},
+		{
+			Width:  1920,
+			Height: 1080,
+		},
+		{
+			Width:  50,
+			Height: 33,
+		},
+		{
+			Width:  75,
+			Height: 51,
+		},
+	}
+
+	media.SortYoutubeThumbnails(thumbnails)
+
+	assert.Equal(t, uint(250), thumbnails[0].Width)
+	assert.Equal(t, uint(1920), thumbnails[1].Width)
+	assert.Equal(t, uint(50), thumbnails[2].Width)
+	assert.Equal(t, uint(75), thumbnails[3].Width)
+}

backend/server/server.go

@@ -211,6 +211,26 @@ func (c *mediaSetServiceController) GetVideo(request *pbmediaset.GetVideoRequest
 	return nil
 }
 
+func (c *mediaSetServiceController) GetVideoThumbnail(ctx context.Context, request *pbmediaset.GetVideoThumbnailRequest) (*pbmediaset.GetVideoThumbnailResponse, error) {
+	id, err := uuid.Parse(request.GetId())
+	if err != nil {
+		return nil, newResponseError(err)
+	}
+
+	thumbnail, err := c.mediaSetService.GetVideoThumbnail(ctx, id)
+	if err != nil {
+		return nil, newResponseError(err)
+	}
+
+	response := pbmediaset.GetVideoThumbnailResponse{
+		Image:  thumbnail.Data,
+		Width:  int32(thumbnail.Width),
+		Height: int32(thumbnail.Height),
+	}
+
+	return &response, nil
+}
+
 func Start(options Options) error {
 	logger, err := buildLogger(options.Environment)
 	if err != nil {

backend/sql/queries.sql

@@ -20,3 +20,9 @@ UPDATE media_sets
   SET video_s3_bucket = $2, video_s3_key = $3, video_s3_uploaded_at = NOW(), updated_at = NOW()
   WHERE id = $1
   RETURNING *;
+
+-- name: SetVideoThumbnailUploaded :one
+UPDATE media_sets
+SET video_thumbnail_width = $2, video_thumbnail_height = $3, video_thumbnail_mime_type = $4, video_thumbnail_s3_bucket = $5, video_thumbnail_s3_key = $6, video_thumbnail_s3_uploaded_at = NOW(), updated_at = NOW()
+WHERE id = $1
+RETURNING *;

frontend/src/App.tsx

@@ -181,6 +181,7 @@ function App(): JSX.Element {
           />
 
           <VideoPreview
+            mediaSet={mediaSet}
             video={video}
             position={position}
             duration={millisFromDuration(mediaSet.videoDuration)}

frontend/src/VideoPreview.tsx

@@ -1,6 +1,9 @@
+import { MediaSet, MediaSetServiceClientImpl } from './generated/media_set';
+import { newRPC } from './App';
 import { useEffect, useRef } from 'react';
 
 interface Props {
+  mediaSet: MediaSet;
   position: number;
   duration: number;
   height: number;
@@ -8,6 +11,7 @@ interface Props {
 }
 
 export const VideoPreview: React.FC<Props> = ({
+  mediaSet,
   position,
   duration,
   height,
@@ -17,6 +21,53 @@ export const VideoPreview: React.FC<Props> = ({
 
   // effects
 
+  // load thumbnail, to display when the component is loaded for the first
+  // time. This is needed because of browser autoplay limitations.
+  useEffect(() => {
+    (async function () {
+      if (mediaSet == null) {
+        return;
+      }
+
+      const canvas = videoCanvasRef.current;
+      if (canvas == null) {
+        console.error('no canvas ref available');
+        return;
+      }
+
+      const ctx = canvas.getContext('2d');
+      if (ctx == null) {
+        console.error('no 2d context available');
+        return;
+      }
+
+      // Set aspect ratio.
+      canvas.width = canvas.height * (canvas.clientWidth / canvas.clientHeight);
+
+      console.log('getting video thumbnail...');
+      const rpc = newRPC();
+      const service = new MediaSetServiceClientImpl(rpc);
+
+      const thumbnail = await service.GetVideoThumbnail({ id: mediaSet.id });
+
+      console.log('got thumbnail', thumbnail);
+
+      const url = URL.createObjectURL(
+        new Blob([thumbnail.image], { type: 'image/jpeg' })
+      );
+      const img = new Image(thumbnail.width, thumbnail.height);
+
+      img.src = url;
+      console.log('img', img);
+      img.onerror = console.error;
+      img.onload = () => {
+        ctx.drawImage(img, 0, 0, 177, 100);
+      };
+
+      console.log('set src to', url);
+    })();
+  }, [mediaSet]);
+
   // render canvas
   useEffect(() => {
     // TODO: not sure if requestAnimationFrame is recommended here.

proto/media_set.proto

@@ -56,9 +56,20 @@ message GetVideoProgress {
   string url = 2;
 }
 
+message GetVideoThumbnailRequest {
+  string id = 1;
+}
+
+message GetVideoThumbnailResponse {
+  bytes image = 1;
+  int32 width = 2;
+  int32 height = 3;
+}
+
 service MediaSetService {
   rpc Get(GetRequest) returns (MediaSet) {}
   rpc GetAudio(GetAudioRequest) returns (stream GetAudioProgress) {}
   rpc GetAudioSegment(GetAudioSegmentRequest) returns (GetAudioSegmentResponse) {}
   rpc GetVideo(GetVideoRequest) returns (stream GetVideoProgress) {}
+  rpc GetVideoThumbnail(GetVideoThumbnailRequest) returns (GetVideoThumbnailResponse) {}
 }