More progress

backend/cmd/progress-test/main.go

@@ -44,9 +44,14 @@ func main() {
 	// Create a MediaSetService
 	mediaSetService := media.NewMediaSetService(store, &youtubeClient, s3Client)
 
+	mediaSet, err := mediaSetService.Get(ctx, videoID)
+	if err != nil {
+		log.Fatalf("error calling fetch service: %v", err)
+	}
+
 	// Create a progressReader
-	// TODO: fix
+	id := uuid.MustParse(mediaSet.ID)
-	progressReader, err := mediaSetService.GetAudio(ctx, uuid.New(), 100)
+	progressReader, err := mediaSetService.GetAudio(ctx, id, 100)
 	if err != nil {
 		log.Fatalf("error calling fetch service: %v", err)
 	}

backend/media/service.go

@@ -22,6 +22,7 @@ const (
 	rawAudioCodec      = "pcm_s16le"
 	rawAudioFormat     = "s16le"
 	rawAudioSampleRate = 48_000
+	rawAudioMimeType   = "audio/raw"
 )
 
 const (
@@ -50,6 +51,7 @@ type Store interface {
 	GetMediaSet(ctx context.Context, id uuid.UUID) (store.MediaSet, error)
 	GetMediaSetByYoutubeID(ctx context.Context, youtubeID string) (store.MediaSet, error)
 	CreateMediaSet(ctx context.Context, arg store.CreateMediaSetParams) (store.MediaSet, error)
+	SetAudioUploaded(ctx context.Context, arg store.SetAudioUploadedParams) (store.MediaSet, error)
 }
 
 // S3Client wraps the AWS S3 service client.
@@ -127,7 +129,7 @@ func (s *MediaSetService) createMediaSet(ctx context.Context, youtubeID string)
 		AudioYoutubeItag:     int32(audioMetadata.YoutubeItag),
 		AudioChannels:        int32(audioMetadata.Channels),
 		AudioFramesApprox:    audioMetadata.ApproxFrames,
-		AudioSampleRateRaw:   int32(audioMetadata.SampleRate),
+		AudioSampleRate:      int32(audioMetadata.SampleRate),
 		AudioMimeTypeEncoded: audioMetadata.MimeType,
 		VideoYoutubeItag:     int32(videoMetadata.YoutubeItag),
 		VideoMimeType:        videoMetadata.MimeType,
@@ -162,8 +164,8 @@ func (s *MediaSetService) findMediaSet(ctx context.Context, youtubeID string) (*
 			Bytes:        0, // DEPRECATED
 			Channels:     int(mediaSet.AudioChannels),
 			ApproxFrames: int64(mediaSet.AudioFramesApprox),
-			Frames:       mediaSet.AudioFramesRaw.Int64,
+			Frames:       mediaSet.AudioFrames.Int64,
-			SampleRate:   int(mediaSet.AudioSampleRateRaw),
+			SampleRate:   int(mediaSet.AudioSampleRate),
 			MimeType:     mediaSet.AudioMimeTypeEncoded,
 		},
 		Video: Video{
@@ -258,13 +260,13 @@ func (s *MediaSetService) GetAudio(ctx context.Context, id uuid.UUID, numBins in
 		return nil, fmt.Errorf("error creating ffmpegreader: %v", err)
 	}
 
-	// set up uploader, this is writer 1
+	s3Key := fmt.Sprintf("media_sets/%s/audio.webm", id)
 	uploader, err := newMultipartUploadWriter(
 		ctx,
 		s.s3,
 		s3Bucket,
-		fmt.Sprintf("media_sets/%s/audio.webm", id),
+		s3Key,
-		"application/octet-stream",
+		rawAudioMimeType,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error creating uploader: %v", err)
@@ -276,24 +278,29 @@ func (s *MediaSetService) GetAudio(ctx context.Context, id uuid.UUID, numBins in
 		100,
 	)
 
-	state := fetchAudioState{
+	state := getAudioState{
 		fetchAudioProgressReader: fetchAudioProgressReader,
 		ffmpegReader:             ffmpegReader,
 		uploader:                 uploader,
+		s3Bucket:                 s3Bucket,
+		s3Key:                    s3Key,
+		store:                    s.store,
 	}
 	go state.run(ctx)
 
 	return &state, nil
 }
 
-type fetchAudioState struct {
+type getAudioState struct {
 	*fetchAudioProgressReader
 
-	ffmpegReader io.ReadCloser
+	ffmpegReader    io.ReadCloser
-	uploader     *multipartUploadWriter
+	uploader        *multipartUploadWriter
+	s3Bucket, s3Key string
+	store           Store
 }
 
-func (s *fetchAudioState) run(ctx context.Context) {
+func (s *getAudioState) run(ctx context.Context) {
 	mw := io.MultiWriter(s, s.uploader)
 	done := make(chan error)
 	var err error
@@ -326,6 +333,17 @@ outer:
 		}
 	}
 
+	if err == nil {
+		_, updateErr := s.store.SetAudioUploaded(ctx, store.SetAudioUploadedParams{
+			AudioS3Bucket: sqlString(s.s3Bucket),
+			AudioS3Key:    sqlString(s.s3Key),
+		})
+
+		if updateErr != nil {
+			err = updateErr
+		}
+	}
+
 	if err != nil {
 		newCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancel()
@@ -341,3 +359,7 @@ outer:
 		log.Printf("error closing peak iterator: %v", iterErr)
 	}
 }
+
+func sqlString(s string) sql.NullString {
+	return sql.NullString{String: s, Valid: true}
+}

backend/media/uploader.go

@@ -45,7 +45,8 @@ func newMultipartUploadWriter(ctx context.Context, s3Client S3Client, bucket, ke
 		return nil, fmt.Errorf("error creating multipart upload: %v", err)
 	}
 
-	b := make([]byte, 0, targetPartSizeBytes+16_384)
+	const bufferOverflowSize = 16_384
+	b := make([]byte, 0, targetPartSizeBytes+bufferOverflowSize)
 
 	return &multipartUploadWriter{
 		ctx:         ctx,

backend/server/server.go

@@ -65,20 +65,17 @@ func (c *mediaSetServiceController) Get(ctx context.Context, request *pbMediaSet
 	}
 
 	result := pbMediaSet.MediaSet{
-		Id: mediaSet.YoutubeID,
+		Id:                mediaSet.ID,
-		Audio: &pbMediaSet.MediaSet_Audio{
+		YoutubeId:         mediaSet.YoutubeID,
-			Channels:     int32(mediaSet.Audio.Channels),
+		AudioChannels:     int32(mediaSet.Audio.Channels),
-			Frames:       mediaSet.Audio.Frames,
+		AudioFrames:       mediaSet.Audio.Frames,
-			ApproxFrames: mediaSet.Audio.ApproxFrames,
+		AudioApproxFrames: mediaSet.Audio.ApproxFrames,
-			SampleRate:   int32(mediaSet.Audio.SampleRate),
+		AudioSampleRate:   int32(mediaSet.Audio.SampleRate),
-			YoutubeItag:  int32(mediaSet.Audio.YoutubeItag),
+		AudioYoutubeItag:  int32(mediaSet.Audio.YoutubeItag),
-			MimeType:     mediaSet.Audio.MimeType,
+		AudioMimeType:     mediaSet.Audio.MimeType,
-		},
+		VideoDuration:     durationpb.New(mediaSet.Video.Duration),
-		Video: &pbMediaSet.MediaSet_Video{
+		VideoYoutubeItag:  int32(mediaSet.Video.YoutubeItag),
-			Duration:    durationpb.New(mediaSet.Video.Duration),
+		VideoMimeType:     mediaSet.Video.MimeType,
-			YoutubeItag: int32(mediaSet.Video.YoutubeItag),
-			MimeType:    mediaSet.Video.MimeType,
-		},
 	}
 
 	return &result, nil

backend/sql/queries.sql

@@ -5,6 +5,11 @@ SELECT * FROM media_sets WHERE id = $1;
 SELECT * FROM media_sets WHERE youtube_id = $1;
 
 -- name: CreateMediaSet :one
-INSERT INTO media_sets (youtube_id, audio_youtube_itag, audio_channels, audio_frames_approx, audio_sample_rate_raw, audio_mime_type_encoded, video_youtube_itag, video_mime_type, video_duration_nanos, created_at, updated_at)
+INSERT INTO media_sets (youtube_id, audio_youtube_itag, audio_channels, audio_frames_approx, audio_sample_rate, audio_mime_type_encoded, video_youtube_itag, video_mime_type, video_duration_nanos, created_at, updated_at)
   VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW(), NOW())
   RETURNING *;
+
+-- name: SetAudioUploaded :one
+UPDATE media_sets
+  SET audio_s3_bucket = $1, audio_s3_key = $2, audio_s3_uploaded_at = NOW(), updated_at = NOW()
+  RETURNING *;

frontend/package.json

@@ -15,6 +15,7 @@
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
+    "ts-proto": "^1.83.3",
     "typescript": "^4.1.2",
     "web-vitals": "^1.0.1"
   },
@@ -49,6 +50,6 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-react": "^7.25.1",
     "prettier": "2.4.0",
-    "ts-protoc-gen": "^0.15.0"
+    "rxjs": "^7.4.0"
   }
 }

frontend/src/App.tsx

@@ -1,12 +1,15 @@
 import { grpc } from '@improbable-eng/grpc-web';
+// import {
+//   MediaSet as MediaSetPb,
+//   GetRequest,
+//   GetAudioRequest,
+//   GetAudioProgress,
+// } from './generated/media_set_pb';
 import {
-  MediaSet as MediaSetPb,
+  MediaSet,
-  GetRequest,
+  GrpcWebImpl,
-  GetAudioRequest,
+  MediaSetServiceClientImpl,
-  GetAudioProgress,
+} from './generated/media_set';
-} from './generated/media_set_pb';
-
-import { GetMediaSet } from './GrpcWrapper';
 
 import { useState, useEffect } from 'react';
 import { VideoPreview } from './VideoPreview';
@@ -15,32 +18,13 @@ import { Waveform } from './Waveform';
 import { ControlBar } from './ControlBar';
 import { SeekBar } from './SeekBar';
 import './App.css';
+import { Duration } from './generated/google/protobuf/duration';
 
 const grpcHost = 'http://localhost:8888';
 
-// Audio corresponds to media.Audio.
+// ported from backend, where should they live?
-export interface Audio {
+const thumbnailWidth = 177;
-  bytes: number;
+const thumbnailHeight = 100;
-  channels: number;
-  frames: number;
-  sampleRate: number;
-}
-
-// Video corresponds to media.Video.
-export interface Video {
-  bytes: number;
-  thumbnailWidth: number;
-  thumbnailHeight: number;
-  durationMillis: number;
-}
-
-// MediaSet corresponds to media.MediaSet.
-export interface MediaSet {
-  id: string;
-  source: string;
-  audio: Audio;
-  video: Video;
-}
 
 // Frames represents a selection of audio frames.
 export interface Frames {
@@ -65,12 +49,14 @@ function App(): JSX.Element {
   // fetch mediaset on page load:
   useEffect(() => {
     (async function () {
-      const request = new GetRequest();
+      const rpc = new GrpcWebImpl('http://localhost:8888', {});
-      request.setYoutubeId(videoID);
+      const service = new MediaSetServiceClientImpl(rpc);
+      const mediaSet = await service.Get({ youtubeId: videoID });
 
-      const mediaSet = await GetMediaSet(grpcHost, request);
       console.log('got media set:', mediaSet);
 
+      setMediaSet(mediaSet);
+
       //       const handleProgress = (progress: GetAudioProgress) => {
       //         console.log('got progress', progress);
       //       };
@@ -122,6 +108,8 @@ function App(): JSX.Element {
       return;
     }
 
+    return;
+
     video.src = `http://localhost:8888/api/media_sets/${videoID}/video`;
     video.muted = false;
     video.volume = 1;
@@ -134,7 +122,7 @@ function App(): JSX.Element {
       return;
     }
 
-    setViewport({ start: 0, end: mediaSet.audio.frames });
+    setViewport({ start: 0, end: mediaSet.audioFrames });
   }, [mediaSet]);
 
   useEffect(() => {
@@ -150,7 +138,7 @@ function App(): JSX.Element {
     }
 
     if (selection.start >= selection.end) {
-      setViewport({ start: 0, end: mediaSet.audio.frames });
+      setViewport({ start: 0, end: mediaSet.audioFrames });
       return;
     }
 
@@ -169,10 +157,7 @@ function App(): JSX.Element {
     flexDirection: 'column',
   } as React.CSSProperties;
 
-  let offsetPixels = 75;
+  const offsetPixels = Math.floor(thumbnailWidth / 2);
-  if (mediaSet != null) {
-    offsetPixels = Math.floor(mediaSet.video.thumbnailWidth / 2);
-  }
 
   if (mediaSet == null) {
     // TODO: improve
@@ -212,7 +197,7 @@ function App(): JSX.Element {
 
           <SeekBar
             position={video.currentTime}
-            duration={mediaSet.audio.frames / mediaSet.audio.sampleRate}
+            duration={mediaSet.audioFrames / mediaSet.audioSampleRate}
             offsetPixels={offsetPixels}
             onPositionChanged={(position: number) => {
               video.currentTime = position;
@@ -222,8 +207,8 @@ function App(): JSX.Element {
           <VideoPreview
             video={video}
             position={position}
-            duration={mediaSet.video.durationMillis}
+            duration={millisFromDuration(mediaSet.videoDuration)}
-            height={mediaSet.video.thumbnailHeight}
+            height={thumbnailHeight}
           />
         </div>
       </div>
@@ -232,3 +217,10 @@ function App(): JSX.Element {
 }
 
 export default App;
+
+function millisFromDuration(dur?: Duration): number {
+  if (dur == undefined) {
+    return 0;
+  }
+  return Math.floor(dur.seconds * 1000.0 + dur.nanos / 1000.0 / 1000.0);
+}

frontend/src/GrpcWrapper.tsx

@@ -1,55 +0,0 @@
-import { grpc } from '@improbable-eng/grpc-web';
-import { MediaSetService } from './generated/media_set_pb_service';
-import {
-  MediaSet,
-  GetRequest,
-  GetAudioProgress,
-  GetAudioRequest,
-} from './generated/media_set_pb';
-
-export const GetMediaSet = (
-  host: string,
-  request: GetRequest
-): Promise<MediaSet> => {
-  return new Promise<MediaSet>((resolve, reject) => {
-    let result: MediaSet;
-
-    grpc.invoke(MediaSetService.Get, {
-      host: host,
-      request: request,
-      onMessage: (mediaSet: MediaSet) => {
-        result = mediaSet;
-      },
-      onEnd: (
-        code: grpc.Code,
-        msg: string | undefined,
-        _trailers: grpc.Metadata
-      ) => {
-        if (code != 0) {
-          reject(new Error(`unexpected grpc code: ${code}, message: ${msg}`));
-          return;
-        }
-        resolve(result);
-      },
-    });
-  });
-};
-
-// export const etchMediaSetAudio = (
-//   host: string,
-//   request: FetchAudioRequest,
-//   onProgress: { (progress: FetchAudioProgress): void }
-// ) => {
-//   grpc.invoke(FetchService.FetchAudio, {
-//     host: 'http://localhost:8888',
-//     request: request,
-//     onMessage: onProgress,
-//     onEnd: (
-//       code: grpc.Code,
-//       msg: string | undefined,
-//       trailers: grpc.Metadata
-//     ) => {
-//       console.log('fetch audio request ended');
-//     },
-//   });
-// };

frontend/src/Overview.tsx

@@ -1,5 +1,6 @@
 import { useState, useEffect, useRef, MouseEvent } from 'react';
-import { MediaSet, Frames } from './App';
+import { MediaSet } from './generated/media_set';
+import { Frames } from './App';
 import { WaveformCanvas } from './WaveformCanvas';
 import { mouseEventToCanvasX } from './Helpers';
 import { secsToCanvasX } from './Helpers';
@@ -49,11 +50,11 @@ export const Overview: React.FC<Props> = ({
         return;
       }
 
-      const resp = await fetch(
+      // const resp = await fetch(
-        `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=0&end=${mediaSet.audio.frames}&bins=${CanvasLogicalWidth}`
+      //   `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=0&end=${mediaSet.audioFrames}&bins=${CanvasLogicalWidth}`
-      );
+      // );
-      const peaks = await resp.json();
+      // const peaks = await resp.json();
-      setPeaks(peaks);
+      // setPeaks(peaks);
     })();
   }, [mediaSet]);
 
@@ -84,9 +85,9 @@ export const Overview: React.FC<Props> = ({
 
       if (currentSelection.start < currentSelection.end) {
         const x1 =
-          (currentSelection.start / mediaSet.audio.frames) * CanvasLogicalWidth;
+          (currentSelection.start / mediaSet.audioFrames) * CanvasLogicalWidth;
         const x2 =
-          (currentSelection.end / mediaSet.audio.frames) * CanvasLogicalWidth;
+          (currentSelection.end / mediaSet.audioFrames) * CanvasLogicalWidth;
 
         ctx.beginPath();
         ctx.strokeStyle = 'red';
@@ -98,10 +99,10 @@ export const Overview: React.FC<Props> = ({
       }
 
       // draw position marker:
-      const fullSelection = { start: 0, end: mediaSet.audio.frames }; // constantize?
+      const fullSelection = { start: 0, end: mediaSet.audioFrames }; // constantize?
       const x = secsToCanvasX(
         position,
-        mediaSet.audio.sampleRate,
+        mediaSet.audioSampleRate,
         fullSelection
       );
       // should never happen:
@@ -135,7 +136,7 @@ export const Overview: React.FC<Props> = ({
     }
 
     const frame = Math.floor(
-      mediaSet.audio.frames *
+      mediaSet.audioFrames *
         (mouseEventToCanvasX(evt) / evt.currentTarget.width)
     );
 
@@ -155,7 +156,7 @@ export const Overview: React.FC<Props> = ({
     }
 
     const frame = Math.floor(
-      mediaSet.audio.frames *
+      mediaSet.audioFrames *
         (mouseEventToCanvasX(evt) / evt.currentTarget.width)
     );
 
@@ -164,8 +165,8 @@ export const Overview: React.FC<Props> = ({
       const frameCount = selection.end - selection.start;
       let start = Math.max(0, selection.start + diff);
       let end = start + frameCount;
-      if (end > mediaSet.audio.frames) {
+      if (end > mediaSet.audioFrames) {
-        end = mediaSet.audio.frames;
+        end = mediaSet.audioFrames;
         start = end - frameCount;
       }
       setNewSelection({

frontend/src/Waveform.tsx

@@ -1,5 +1,6 @@
 import { useEffect, useState, useRef } from 'react';
-import { Frames, MediaSet } from './App';
+import { Frames } from './App';
+import { MediaSet } from './generated/media_set';
 import { WaveformCanvas } from './WaveformCanvas';
 import { secsToCanvasX } from './Helpers';
 
@@ -33,14 +34,14 @@ export const Waveform: React.FC<Props> = ({
 
       let endFrame = viewport.end;
       if (endFrame <= viewport.start) {
-        endFrame = mediaSet.audio.frames;
+        endFrame = mediaSet.audioFrames;
       }
 
-      const resp = await fetch(
+      // const resp = await fetch(
-        `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=${viewport.start}&end=${endFrame}&bins=${CanvasLogicalWidth}`
+      //   `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=${viewport.start}&end=${endFrame}&bins=${CanvasLogicalWidth}`
-      );
+      // );
-      const newPeaks = await resp.json();
+      // const newPeaks = await resp.json();
-      setPeaks(newPeaks);
+      // setPeaks(newPeaks);
     })();
   }, [mediaSet, viewport]);
 
@@ -63,7 +64,7 @@ export const Waveform: React.FC<Props> = ({
       return;
     }
 
-    const x = secsToCanvasX(position, mediaSet.audio.sampleRate, viewport);
+    const x = secsToCanvasX(position, mediaSet.audioSampleRate, viewport);
     if (x == null) {
       return;
     }

frontend/yarn.lock

@@ -1484,6 +1484,59 @@
     schema-utils "^2.6.5"
     source-map "^0.7.3"
 
+"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf"
+  integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78=
+
+"@protobufjs/base64@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735"
+  integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==
+
+"@protobufjs/codegen@^2.0.4":
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb"
+  integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==
+
+"@protobufjs/eventemitter@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70"
+  integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A=
+
+"@protobufjs/fetch@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45"
+  integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.1"
+    "@protobufjs/inquire" "^1.1.0"
+
+"@protobufjs/float@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1"
+  integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=
+
+"@protobufjs/inquire@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089"
+  integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=
+
+"@protobufjs/path@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d"
+  integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=
+
+"@protobufjs/pool@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54"
+  integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=
+
+"@protobufjs/utf8@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570"
+  integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=
+
 "@rollup/plugin-node-resolve@^7.1.1":
   version "7.1.3"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz#80de384edfbd7bfc9101164910f86078151a3eca"
@@ -1812,6 +1865,11 @@
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
   integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4=
 
+"@types/long@^4.0.1":
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
+  integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==
+
 "@types/minimatch@*":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
@@ -1822,6 +1880,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.31.tgz#72286bd33d137aa0d152d47ec7c1762563d34055"
   integrity sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==
 
+"@types/node@>=13.7.0":
+  version "16.11.6"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.6.tgz#6bef7a2a0ad684cf6e90fcfe31cecabd9ce0a3ae"
+  integrity sha512-ua7PgUoeQFjmWPcoo9khiPum3Pd60k4/2ZGXt18sm2Slk0W0xZTqt5Y0Ny1NyBiN1EVQ/+FaF9NcY4Qe6rwk5w==
+
 "@types/node@^12.0.0":
   version "12.20.23"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.23.tgz#d0d5885bb885ee9b1ed114a04ea586540a1b2e2a"
@@ -1832,11 +1895,21 @@
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
   integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==
 
+"@types/object-hash@^1.3.0":
+  version "1.3.4"
+  resolved "https://registry.yarnpkg.com/@types/object-hash/-/object-hash-1.3.4.tgz#079ba142be65833293673254831b5e3e847fe58b"
+  integrity sha512-xFdpkAkikBgqBdG9vIlsqffDV8GpvnPEzs0IUtr1v3BEB97ijsFQ4RXVbUZwjFThhB4MDSTUfvmxUD5PGx0wXA==
+
 "@types/parse-json@^4.0.0":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
   integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==
 
+"@types/prettier@^1.19.0":
+  version "1.19.1"
+  resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-1.19.1.tgz#33509849f8e679e4add158959fdb086440e9553f"
+  integrity sha512-5qOlnZscTn4xxM5MeGXAMOsIOIKIbh9e85zJWfBRVPlRMEVawzoPhINYbRGkBZCI8LxvBe7tJCdWiarA99OZfQ==
+
 "@types/prettier@^2.0.0":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.2.1.tgz#374e31645d58cb18a07b3ecd8e9dede4deb2cccd"
@@ -4046,6 +4119,11 @@ data-urls@^2.0.0:
     whatwg-mimetype "^2.3.0"
     whatwg-url "^8.0.0"
 
+dataloader@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-1.4.0.tgz#bca11d867f5d3f1b9ed9f737bd15970c65dff5c8"
+  integrity sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw==
+
 debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -5618,7 +5696,7 @@ globby@^6.1.0:
     pify "^2.0.0"
     pinkie-promise "^2.0.0"
 
-google-protobuf@^3.15.5, google-protobuf@^3.19.0:
+google-protobuf@^3.19.0:
   version "3.19.0"
   resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.19.0.tgz#97f474323c92f19fd6737af1bb792e396991e0b8"
   integrity sha512-qXGAiv3OOlaJXJNeKOBKxbBAwjsxzhx+12ZdKOkZTsqsRkyiQRmr/nBkAkqnuQ8cmA9X5NVXvObQTpHVnXE2DQ==
@@ -7365,6 +7443,11 @@ loglevel@^1.6.8:
   resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.7.1.tgz#005fde2f5e6e47068f935ff28573e125ef72f197"
   integrity sha512-Hesni4s5UkWkwCGJMQGAh71PaLUmKFM60dHvq0zi/vDhhrzuk+4GgNbTXJ12YYQJn6ZKBDNIjYcuQGKudvqrIw==
 
+long@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
+  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
+
 loose-envify@^1.1.0, loose-envify@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
@@ -7940,6 +8023,11 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
+object-hash@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
+  integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==
+
 object-inspect@^1.11.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.11.0.tgz#9dceb146cedd4148a0d9e51ab88d34cf509922b1"
@@ -9155,6 +9243,11 @@ prettier@2.4.0:
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.0.tgz#85bdfe0f70c3e777cf13a4ffff39713ca6f64cba"
   integrity sha512-DsEPLY1dE5HF3BxCRBmD4uYZ+5DCbvatnolqTqcxEgKVZnL2kUfyu7b8pPQ5+hTBkdhU9SLUmK0/pHb07RE4WQ==
 
+prettier@^2.0.2:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.1.tgz#671e11c89c14a4cfc876ce564106c4a6726c9f5c"
+  integrity sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==
+
 pretty-bytes@^5.3.0:
   version "5.6.0"
   resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
@@ -9232,6 +9325,25 @@ prop-types@^15.7.2:
     object-assign "^4.1.1"
     react-is "^16.8.1"
 
+protobufjs@^6.8.8:
+  version "6.11.2"
+  resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.2.tgz#de39fabd4ed32beaa08e9bb1e30d08544c1edf8b"
+  integrity sha512-4BQJoPooKJl2G9j3XftkIXjoC9C0Av2NOrWmbLWT1vH32GcSUHjM0Arra6UfTsVyfMAuFzaLucXn1sadxJydAw==
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.2"
+    "@protobufjs/base64" "^1.1.2"
+    "@protobufjs/codegen" "^2.0.4"
+    "@protobufjs/eventemitter" "^1.1.0"
+    "@protobufjs/fetch" "^1.1.0"
+    "@protobufjs/float" "^1.0.2"
+    "@protobufjs/inquire" "^1.1.0"
+    "@protobufjs/path" "^1.1.2"
+    "@protobufjs/pool" "^1.1.0"
+    "@protobufjs/utf8" "^1.1.0"
+    "@types/long" "^4.0.1"
+    "@types/node" ">=13.7.0"
+    long "^4.0.0"
+
 proxy-addr@~2.0.5:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf"
@@ -9981,6 +10093,13 @@ run-queue@^1.0.0, run-queue@^1.0.3:
   dependencies:
     aproba "^1.1.1"
 
+rxjs@^7.4.0:
+  version "7.4.0"
+  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.4.0.tgz#a12a44d7eebf016f5ff2441b87f28c9a51cebc68"
+  integrity sha512-7SQDi7xeTMCJpqViXh8gL/lebcwlp3d831F05+9B44A4B0WfsEwUQHR64gsH1kvJ+Ep/J9K2+n1hVl1CsGN23w==
+  dependencies:
+    tslib "~2.1.0"
+
 safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
@@ -11060,12 +11179,34 @@ ts-pnp@1.2.0, ts-pnp@^1.1.6:
   resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
   integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw==
 
-ts-protoc-gen@^0.15.0:
+ts-poet@^4.5.0:
-  version "0.15.0"
+  version "4.6.1"
-  resolved "https://registry.yarnpkg.com/ts-protoc-gen/-/ts-protoc-gen-0.15.0.tgz#2fec5930b46def7dcc9fa73c060d770b7b076b7b"
+  resolved "https://registry.yarnpkg.com/ts-poet/-/ts-poet-4.6.1.tgz#015dc823d726655af9f095c900f84ed7c60e2dd3"
-  integrity sha512-TycnzEyrdVDlATJ3bWFTtra3SCiEP0W0vySXReAuEygXCUr1j2uaVyL0DhzjwuUdQoW5oXPwk6oZWeA0955V+g==
+  integrity sha512-DXJ+mBJIDp+jiaUgB4N5I/sczHHDU2FWacdbDNVAVS4Mh4hb7ckpvUWVW7m7/nAOcjR0r4Wt+7AoO7FeJKExfA==
   dependencies:
-    google-protobuf "^3.15.5"
+    "@types/prettier" "^1.19.0"
+    lodash "^4.17.15"
+    prettier "^2.0.2"
+
+ts-proto-descriptors@^1.2.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/ts-proto-descriptors/-/ts-proto-descriptors-1.3.1.tgz#760ebaaa19475b03662f7b358ffea45b9c5348f5"
+  integrity sha512-Cybb3fqceMwA6JzHdC32dIo8eVGVmXrM6TWhdk1XQVVHT/6OQqk0ioyX1dIdu3rCIBhRmWUhUE4HsyK+olmgMw==
+  dependencies:
+    long "^4.0.0"
+    protobufjs "^6.8.8"
+
+ts-proto@^1.83.3:
+  version "1.83.3"
+  resolved "https://registry.yarnpkg.com/ts-proto/-/ts-proto-1.83.3.tgz#ada7483035ddc946aa686dad1049e4fe45ae1d0f"
+  integrity sha512-r6MKFjoc4Og2kB4cNJ/bddLebdIwhouG5plu0Rry1jJMEqp2GKA7AE4FrR/FnTCIGbNPYP4622lBqckZd7UHcQ==
+  dependencies:
+    "@types/object-hash" "^1.3.0"
+    dataloader "^1.4.0"
+    object-hash "^1.3.1"
+    protobufjs "^6.8.8"
+    ts-poet "^4.5.0"
+    ts-proto-descriptors "^1.2.1"
 
 tsconfig-paths@^3.9.0:
   version "3.9.0"
@@ -11082,7 +11223,7 @@ tslib@^1.8.1, tslib@^1.9.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
-tslib@^2.0.3:
+tslib@^2.0.3, tslib@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a"
   integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==

proto/media_set.proto

@@ -6,25 +6,19 @@ option go_package = "pb/media_set";
 import "google/protobuf/duration.proto";
 
 message MediaSet {
-  message Audio {
-    int32 channels = 1;
-    int64 approx_frames = 2;
-    int64 frames = 3;
-    int32 sample_rate = 4;
-    int32 youtube_itag = 5;
-    string mime_type = 6;
-  };
-
-  message Video {
-    google.protobuf.Duration duration = 1;
-    int32 youtube_itag = 2;
-    string mime_type = 3;
-  };
-
   string id = 1;
   string youtube_id = 2;
-  Audio audio = 3;
+
-  Video video = 4;
+  int32 audio_channels = 3;
+  int64 audio_approx_frames = 4;
+  int64 audio_frames = 5;
+  int32 audio_sample_rate = 6;
+  int32 audio_youtube_itag = 7;
+  string audio_mime_type = 8;
+
+  google.protobuf.Duration video_duration = 9;
+  int32 video_youtube_itag = 10;
+  string video_mime_type = 11;
 };
 
 message GetAudioProgress {

protogen.sh

@@ -5,7 +5,7 @@
 # protobuf (pacman -S protobuf)
 # protoc-gen-go (go install google.golang.org/protobuf/cmd/protoc-gen-go@latest)
 # protoc-gen-go-grpc (go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest)
-# protoc-gen-ts (uses local copy from node_modules)
+# ts-proto (uses local copy from node_modules)
 #
 set -ex
 
@@ -17,9 +17,9 @@ mkdir -p $TARGET_FRONTEND
 
 protoc \
   -I./proto/ \
-  --plugin="protoc-gen-ts=./frontend/node_modules/.bin/protoc-gen-ts" \
+  --plugin="./frontend/node_modules/.bin/protoc-gen-ts_proto" \
   --go_out="$TARGET_BACKEND" \
   --go-grpc_out="$TARGET_BACKEND" \
   --js_out="import_style=commonjs,binary:$TARGET_FRONTEND" \
-  --ts_out="service=grpc-web:$TARGET_FRONTEND" \
+  --ts_proto_out="outputClientImpl=grpc-web,useOptionals=true,esModuleInterop=true:$TARGET_FRONTEND" \
   ./proto/*