rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Add video thumbnail support

This commit is contained in:
Rob Watson 2021-11-21 20:43:40 +01:00
parent 2a0f2e22e0
commit 2f7aae1d6e
8 changed files with 233 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
backend/media/service.go
View File

@ -9,6 +9,7 @@ import (
"fmt"
"io"
"log"
"net/http"
"strconv"
"time"
@ -57,11 +58,12 @@ func (pw *progressReader) Read(p []byte) (int, error) {
// Store wraps a database store.
type Store interface {
GetMediaSet(ctx context.Context, id uuid.UUID) (store.MediaSet, error)
GetMediaSetByYoutubeID(ctx context.Context, youtubeID string) (store.MediaSet, error)
CreateMediaSet(ctx context.Context, arg store.CreateMediaSetParams) (store.MediaSet, error)
SetAudioUploaded(ctx context.Context, arg store.SetAudioUploadedParams) (store.MediaSet, error)
SetVideoUploaded(ctx context.Context, arg store.SetVideoUploadedParams) (store.MediaSet, error)
GetMediaSet(context.Context, uuid.UUID) (store.MediaSet, error)
GetMediaSetByYoutubeID(context.Context, string) (store.MediaSet, error)
CreateMediaSet(context.Context, store.CreateMediaSetParams) (store.MediaSet, error)
SetAudioUploaded(context.Context, store.SetAudioUploadedParams) (store.MediaSet, error)
SetVideoUploaded(context.Context, store.SetVideoUploadedParams) (store.MediaSet, error)
SetVideoThumbnailUploaded(context.Context, store.SetVideoThumbnailUploadedParams) (store.MediaSet, error)
}
// S3API provides an API to AWS S3.
@ -592,6 +594,10 @@ func sqlInt64(i int64) sql.NullInt64 {
return sql.NullInt64{Int64: i, Valid: true}
}
func sqlInt32(i int32) sql.NullInt32 {
return sql.NullInt32{Int32: i, Valid: true}
}
// ModuloBufReader reads from a reader in block sizes that are exactly modulo
// modSize, with any remainder buffered until the next read.
type ModuloBufReader struct {
@ -628,3 +634,95 @@ func (r *ModuloBufReader) Read(p []byte) (int, error) {
return nr - rem, err
}
type VideoThumbnail struct {
Data []byte
Width, Height int
}
func (s *MediaSetService) GetVideoThumbnail(ctx context.Context, id uuid.UUID) (VideoThumbnail, error) {
mediaSet, err := s.store.GetMediaSet(ctx, id)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error getting media set: %v", err)
}
if mediaSet.VideoThumbnailS3UploadedAt.Valid {
return s.getThumbnailFromS3(ctx, mediaSet)
}
return s.getThumbnailFromYoutube(ctx, mediaSet)
}
func (s *MediaSetService) getThumbnailFromS3(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) {
input := s3.GetObjectInput{
Bucket: aws.String(mediaSet.VideoThumbnailS3Bucket.String),
Key: aws.String(mediaSet.VideoThumbnailS3Key.String),
}
output, err := s.s3.GetObject(ctx, &input)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail from s3: %v", err)
}
defer output.Body.Close()
imageData, err := io.ReadAll(output.Body)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error reading thumbnail from s3: %v", err)
}
return VideoThumbnail{
Width: int(mediaSet.VideoThumbnailWidth.Int32),
Height: int(mediaSet.VideoThumbnailHeight.Int32),
Data: imageData,
}, nil
}
func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) {
video, err := s.youtube.GetVideoContext(ctx, mediaSet.YoutubeID)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error fetching video: %v", err)
}
if len(video.Formats) == 0 {
return VideoThumbnail{}, errors.New("no format available")
}
thumbnails := video.Thumbnails
SortYoutubeThumbnails(thumbnails)
thumbnail := thumbnails[0]
resp, err := http.Get(thumbnail.URL)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail: %v", err)
}
defer resp.Body.Close()
imageData, err := io.ReadAll(resp.Body)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error reading thumbnail: %v", err)
}
// TODO: use mediaSet func to fetch s3Key
s3Key := fmt.Sprintf("media_sets/%s/thumbnail.jpg", mediaSet.ID)
uploader := newMultipartUploader(s.s3)
const mimeType = "application/jpeg"
_, err = uploader.Upload(ctx, bytes.NewReader(imageData), s3Bucket, s3Key, mimeType)
if err != nil {
return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err)
}
storeParams := store.SetVideoThumbnailUploadedParams{
ID: mediaSet.ID,
VideoThumbnailMimeType: sqlString(mimeType),
VideoThumbnailS3Bucket: sqlString(s3Bucket),
VideoThumbnailS3Key: sqlString(s3Key),
VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)),
VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)),
}
if _, err := s.store.SetVideoThumbnailUploaded(ctx, storeParams); err != nil {
return VideoThumbnail{}, fmt.Errorf("error updating media set: %v", err)
}
return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil
}

13
backend/media/youtube.go
View File

@ -51,3 +51,16 @@ func FilterYoutubeVideo(inFormats youtubev2.FormatList) youtubev2.FormatList {
})
return formats
}
// SortYoutubeThumbnails sorts the provided thumbnails ordered in descending preferred order.
func SortYoutubeThumbnails(thumbnails youtubev2.Thumbnails) {
sort.SliceStable(thumbnails, func(i, j int) bool {
// TODO: get rid of these magic 177s.
isMinSizeI := thumbnails[i].Width >= 177
isMinSizeJ := thumbnails[j].Width >= 177
if isMinSizeI && isMinSizeJ {
return thumbnails[i].Width < thumbnails[j].Width
}
return isMinSizeI
})
}

28
backend/media/youtube_test.go
View File

@ -86,3 +86,31 @@ func TestFilterVideo(t *testing.T) {
assert.Equal(t, formats[2], sortedFormats[1])
assert.Equal(t, formats[0], sortedFormats[2])
}
func TestSortThumbnails(t *testing.T) {
thumbnails := []youtubev2.Thumbnail{
{
Width: 250,
Height: 100,
},
{
Width: 1920,
Height: 1080,
},
{
Width: 50,
Height: 33,
},
{
Width: 75,
Height: 51,
},
}
media.SortYoutubeThumbnails(thumbnails)
assert.Equal(t, uint(250), thumbnails[0].Width)
assert.Equal(t, uint(1920), thumbnails[1].Width)
assert.Equal(t, uint(50), thumbnails[2].Width)
assert.Equal(t, uint(75), thumbnails[3].Width)
}

20
backend/server/server.go
View File

@ -211,6 +211,26 @@ func (c *mediaSetServiceController) GetVideo(request *pbmediaset.GetVideoRequest
return nil
}
func (c *mediaSetServiceController) GetVideoThumbnail(ctx context.Context, request *pbmediaset.GetVideoThumbnailRequest) (*pbmediaset.GetVideoThumbnailResponse, error) {
id, err := uuid.Parse(request.GetId())
if err != nil {
return nil, newResponseError(err)
}
thumbnail, err := c.mediaSetService.GetVideoThumbnail(ctx, id)
if err != nil {
return nil, newResponseError(err)
}
response := pbmediaset.GetVideoThumbnailResponse{
Image: thumbnail.Data,
Width: int32(thumbnail.Width),
Height: int32(thumbnail.Height),
}
return &response, nil
}
func Start(options Options) error {
logger, err := buildLogger(options.Environment)
if err != nil {

6
backend/sql/queries.sql
View File

@ -20,3 +20,9 @@ UPDATE media_sets
SET video_s3_bucket = $2, video_s3_key = $3, video_s3_uploaded_at = NOW(), updated_at = NOW()
WHERE id = $1
RETURNING *;
-- name: SetVideoThumbnailUploaded :one
UPDATE media_sets
SET video_thumbnail_width = $2, video_thumbnail_height = $3, video_thumbnail_mime_type = $4, video_thumbnail_s3_bucket = $5, video_thumbnail_s3_key = $6, video_thumbnail_s3_uploaded_at = NOW(), updated_at = NOW()
WHERE id = $1
RETURNING *;

1
frontend/src/App.tsx
View File

@ -181,6 +181,7 @@ function App(): JSX.Element {
/>
<VideoPreview
mediaSet={mediaSet}
video={video}
position={position}
duration={millisFromDuration(mediaSet.videoDuration)}

51
frontend/src/VideoPreview.tsx
View File

@ -1,6 +1,9 @@
import { MediaSet, MediaSetServiceClientImpl } from './generated/media_set';
import { newRPC } from './App';
import { useEffect, useRef } from 'react';
interface Props {
mediaSet: MediaSet;
position: number;
duration: number;
height: number;
@ -8,6 +11,7 @@ interface Props {
}
export const VideoPreview: React.FC<Props> = ({
mediaSet,
position,
duration,
height,
@ -17,6 +21,53 @@ export const VideoPreview: React.FC<Props> = ({
// effects
// load thumbnail, to display when the component is loaded for the first
// time. This is needed because of browser autoplay limitations.
useEffect(() => {
(async function () {
if (mediaSet == null) {
return;
}
const canvas = videoCanvasRef.current;
if (canvas == null) {
console.error('no canvas ref available');
return;
}
const ctx = canvas.getContext('2d');
if (ctx == null) {
console.error('no 2d context available');
return;
}
// Set aspect ratio.
canvas.width = canvas.height * (canvas.clientWidth / canvas.clientHeight);
console.log('getting video thumbnail...');
const rpc = newRPC();
const service = new MediaSetServiceClientImpl(rpc);
const thumbnail = await service.GetVideoThumbnail({ id: mediaSet.id });
console.log('got thumbnail', thumbnail);
const url = URL.createObjectURL(
new Blob([thumbnail.image], { type: 'image/jpeg' })
);
const img = new Image(thumbnail.width, thumbnail.height);
img.src = url;
console.log('img', img);
img.onerror = console.error;
img.onload = () => {
ctx.drawImage(img, 0, 0, 177, 100);
};
console.log('set src to', url);
})();
}, [mediaSet]);
// render canvas
useEffect(() => {
// TODO: not sure if requestAnimationFrame is recommended here.

11
proto/media_set.proto
View File

@ -56,9 +56,20 @@ message GetVideoProgress {
string url = 2;
}
message GetVideoThumbnailRequest {
string id = 1;
}
message GetVideoThumbnailResponse {
bytes image = 1;
int32 width = 2;
int32 height = 3;
}
service MediaSetService {
rpc Get(GetRequest) returns (MediaSet) {}
rpc GetAudio(GetAudioRequest) returns (stream GetAudioProgress) {}
rpc GetAudioSegment(GetAudioSegmentRequest) returns (GetAudioSegmentResponse) {}
rpc GetVideo(GetVideoRequest) returns (stream GetVideoProgress) {}
rpc GetVideoThumbnail(GetVideoThumbnailRequest) returns (GetVideoThumbnailResponse) {}
}