From 2b1a668c9db892d959f47a3e23c66cc5224f5954 Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Wed, 29 Dec 2021 16:38:25 +0100 Subject: [PATCH] Add GetAudioSegment flow --- .../generated/pb/media_set/media_set.pb.go | 425 +++++++++++--- .../pb/media_set/media_set_grpc.pb.go | 65 ++- backend/media/get_segment.go | 159 ++++++ backend/media/get_segment_test.go | 264 +++++++++ backend/media/service.go | 136 ++--- backend/media/service_test.go | 5 +- backend/media/testdata/fake.mp3 | 1 + backend/media/testdata/fake.wav | 1 + backend/media/thumbnail.go | 98 ++++ backend/server/server.go | 49 ++ frontend/package.json | 1 + frontend/src/App.tsx | 36 +- frontend/src/ControlBar.tsx | 4 + frontend/src/generated/media_set.ts | 291 ++++++++++ frontend/src/generated/media_set_pb.js | 517 ++++++++++++++++++ frontend/yarn.lock | 5 + proto/media_set.proto | 18 + 17 files changed, 1885 insertions(+), 190 deletions(-) create mode 100644 backend/media/get_segment.go create mode 100644 backend/media/get_segment_test.go create mode 100644 backend/media/testdata/fake.mp3 create mode 100644 backend/media/testdata/fake.wav create mode 100644 backend/media/thumbnail.go diff --git a/backend/generated/pb/media_set/media_set.pb.go b/backend/generated/pb/media_set/media_set.pb.go index 09d2332..9c3311a 100644 --- a/backend/generated/pb/media_set/media_set.pb.go +++ b/backend/generated/pb/media_set/media_set.pb.go @@ -21,6 +21,52 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type AudioFormat int32 + +const ( + AudioFormat_WAV AudioFormat = 0 + AudioFormat_MP3 AudioFormat = 1 +) + +// Enum value maps for AudioFormat. +var ( + AudioFormat_name = map[int32]string{ + 0: "WAV", + 1: "MP3", + } + AudioFormat_value = map[string]int32{ + "WAV": 0, + "MP3": 1, + } +) + +func (x AudioFormat) Enum() *AudioFormat { + p := new(AudioFormat) + *p = x + return p +} + +func (x AudioFormat) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AudioFormat) Descriptor() protoreflect.EnumDescriptor { + return file_media_set_proto_enumTypes[0].Descriptor() +} + +func (AudioFormat) Type() protoreflect.EnumType { + return &file_media_set_proto_enumTypes[0] +} + +func (x AudioFormat) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AudioFormat.Descriptor instead. +func (AudioFormat) EnumDescriptor() ([]byte, []int) { + return file_media_set_proto_rawDescGZIP(), []int{0} +} + type MediaSet struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -431,6 +477,148 @@ func (x *GetPeaksForSegmentResponse) GetPeaks() []int32 { return nil } +type GetAudioSegmentRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + StartFrame int64 `protobuf:"varint,2,opt,name=start_frame,json=startFrame,proto3" json:"start_frame,omitempty"` + EndFrame int64 `protobuf:"varint,3,opt,name=end_frame,json=endFrame,proto3" json:"end_frame,omitempty"` + Format AudioFormat `protobuf:"varint,4,opt,name=format,proto3,enum=media_set.AudioFormat" json:"format,omitempty"` +} + +func (x *GetAudioSegmentRequest) Reset() { + *x = GetAudioSegmentRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_media_set_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetAudioSegmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetAudioSegmentRequest) ProtoMessage() {} + +func (x *GetAudioSegmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_media_set_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetAudioSegmentRequest.ProtoReflect.Descriptor instead. +func (*GetAudioSegmentRequest) Descriptor() ([]byte, []int) { + return file_media_set_proto_rawDescGZIP(), []int{6} +} + +func (x *GetAudioSegmentRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *GetAudioSegmentRequest) GetStartFrame() int64 { + if x != nil { + return x.StartFrame + } + return 0 +} + +func (x *GetAudioSegmentRequest) GetEndFrame() int64 { + if x != nil { + return x.EndFrame + } + return 0 +} + +func (x *GetAudioSegmentRequest) GetFormat() AudioFormat { + if x != nil { + return x.Format + } + return AudioFormat_WAV +} + +type GetAudioSegmentProgress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MimeType string `protobuf:"bytes,1,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + PercentComplete float32 `protobuf:"fixed32,3,opt,name=percent_complete,json=percentComplete,proto3" json:"percent_complete,omitempty"` + AudioData []byte `protobuf:"bytes,4,opt,name=audio_data,json=audioData,proto3" json:"audio_data,omitempty"` +} + +func (x *GetAudioSegmentProgress) Reset() { + *x = GetAudioSegmentProgress{} + if protoimpl.UnsafeEnabled { + mi := &file_media_set_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetAudioSegmentProgress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetAudioSegmentProgress) ProtoMessage() {} + +func (x *GetAudioSegmentProgress) ProtoReflect() protoreflect.Message { + mi := &file_media_set_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetAudioSegmentProgress.ProtoReflect.Descriptor instead. +func (*GetAudioSegmentProgress) Descriptor() ([]byte, []int) { + return file_media_set_proto_rawDescGZIP(), []int{7} +} + +func (x *GetAudioSegmentProgress) GetMimeType() string { + if x != nil { + return x.MimeType + } + return "" +} + +func (x *GetAudioSegmentProgress) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *GetAudioSegmentProgress) GetPercentComplete() float32 { + if x != nil { + return x.PercentComplete + } + return 0 +} + +func (x *GetAudioSegmentProgress) GetAudioData() []byte { + if x != nil { + return x.AudioData + } + return nil +} + type GetVideoRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -442,7 +630,7 @@ type GetVideoRequest struct { func (x *GetVideoRequest) Reset() { *x = GetVideoRequest{} if protoimpl.UnsafeEnabled { - mi := &file_media_set_proto_msgTypes[6] + mi := &file_media_set_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -455,7 +643,7 @@ func (x *GetVideoRequest) String() string { func (*GetVideoRequest) ProtoMessage() {} func (x *GetVideoRequest) ProtoReflect() protoreflect.Message { - mi := &file_media_set_proto_msgTypes[6] + mi := &file_media_set_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -468,7 +656,7 @@ func (x *GetVideoRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetVideoRequest.ProtoReflect.Descriptor instead. func (*GetVideoRequest) Descriptor() ([]byte, []int) { - return file_media_set_proto_rawDescGZIP(), []int{6} + return file_media_set_proto_rawDescGZIP(), []int{8} } func (x *GetVideoRequest) GetId() string { @@ -490,7 +678,7 @@ type GetVideoProgress struct { func (x *GetVideoProgress) Reset() { *x = GetVideoProgress{} if protoimpl.UnsafeEnabled { - mi := &file_media_set_proto_msgTypes[7] + mi := &file_media_set_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -503,7 +691,7 @@ func (x *GetVideoProgress) String() string { func (*GetVideoProgress) ProtoMessage() {} func (x *GetVideoProgress) ProtoReflect() protoreflect.Message { - mi := &file_media_set_proto_msgTypes[7] + mi := &file_media_set_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -516,7 +704,7 @@ func (x *GetVideoProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use GetVideoProgress.ProtoReflect.Descriptor instead. func (*GetVideoProgress) Descriptor() ([]byte, []int) { - return file_media_set_proto_rawDescGZIP(), []int{7} + return file_media_set_proto_rawDescGZIP(), []int{9} } func (x *GetVideoProgress) GetPercentComplete() float32 { @@ -544,7 +732,7 @@ type GetVideoThumbnailRequest struct { func (x *GetVideoThumbnailRequest) Reset() { *x = GetVideoThumbnailRequest{} if protoimpl.UnsafeEnabled { - mi := &file_media_set_proto_msgTypes[8] + mi := &file_media_set_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -557,7 +745,7 @@ func (x *GetVideoThumbnailRequest) String() string { func (*GetVideoThumbnailRequest) ProtoMessage() {} func (x *GetVideoThumbnailRequest) ProtoReflect() protoreflect.Message { - mi := &file_media_set_proto_msgTypes[8] + mi := &file_media_set_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -570,7 +758,7 @@ func (x *GetVideoThumbnailRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetVideoThumbnailRequest.ProtoReflect.Descriptor instead. func (*GetVideoThumbnailRequest) Descriptor() ([]byte, []int) { - return file_media_set_proto_rawDescGZIP(), []int{8} + return file_media_set_proto_rawDescGZIP(), []int{10} } func (x *GetVideoThumbnailRequest) GetId() string { @@ -593,7 +781,7 @@ type GetVideoThumbnailResponse struct { func (x *GetVideoThumbnailResponse) Reset() { *x = GetVideoThumbnailResponse{} if protoimpl.UnsafeEnabled { - mi := &file_media_set_proto_msgTypes[9] + mi := &file_media_set_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -606,7 +794,7 @@ func (x *GetVideoThumbnailResponse) String() string { func (*GetVideoThumbnailResponse) ProtoMessage() {} func (x *GetVideoThumbnailResponse) ProtoReflect() protoreflect.Message { - mi := &file_media_set_proto_msgTypes[9] + mi := &file_media_set_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -619,7 +807,7 @@ func (x *GetVideoThumbnailResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetVideoThumbnailResponse.ProtoReflect.Descriptor instead. func (*GetVideoThumbnailResponse) Descriptor() ([]byte, []int) { - return file_media_set_proto_rawDescGZIP(), []int{9} + return file_media_set_proto_rawDescGZIP(), []int{11} } func (x *GetVideoThumbnailResponse) GetImage() []byte { @@ -703,51 +891,78 @@ var file_media_set_proto_rawDesc = []byte{ 0x46, 0x72, 0x61, 0x6d, 0x65, 0x22, 0x32, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x65, 0x61, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x05, 0x52, 0x05, 0x70, 0x65, 0x61, 0x6b, 0x73, 0x22, 0x21, 0x0a, 0x0f, 0x47, 0x65, 0x74, - 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x4f, 0x0a, 0x10, - 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x12, 0x29, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, 0x70, 0x65, 0x72, 0x63, - 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, - 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x2a, 0x0a, - 0x18, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, - 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x5f, 0x0a, 0x19, 0x47, 0x65, 0x74, - 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x77, 0x69, 0x64, - 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x32, 0x9f, 0x03, 0x0a, 0x0f, 0x4d, - 0x65, 0x64, 0x69, 0x61, 0x53, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, - 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x15, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, - 0x74, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6d, - 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x53, 0x65, - 0x74, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x12, - 0x1a, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, - 0x65, 0x61, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x65, - 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, - 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x00, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x12, - 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, - 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, - 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, 0x72, - 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x47, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x1a, 0x2e, - 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, - 0x65, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x65, 0x64, 0x69, - 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x00, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x11, 0x47, 0x65, - 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x12, - 0x23, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, - 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, + 0x28, 0x05, 0x52, 0x05, 0x70, 0x65, 0x61, 0x6b, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x16, 0x47, 0x65, + 0x74, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x66, 0x72, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x5f, 0x66, 0x72, 0x61, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x46, 0x72, 0x61, + 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x41, + 0x75, 0x64, 0x69, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x22, 0x9a, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x53, + 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1b, + 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, + 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, + 0x0f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x44, 0x61, 0x74, 0x61, 0x22, + 0x21, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x22, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x50, 0x72, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, + 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, + 0x52, 0x0f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x75, 0x72, 0x6c, 0x22, 0x2a, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, + 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, + 0x5f, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, + 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, + 0x2a, 0x1f, 0x0a, 0x0b, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, + 0x07, 0x0a, 0x03, 0x57, 0x41, 0x56, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4d, 0x50, 0x33, 0x10, + 0x01, 0x32, 0xfd, 0x03, 0x0a, 0x0f, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x53, 0x65, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x15, 0x2e, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, + 0x4d, 0x65, 0x64, 0x69, 0x61, 0x53, 0x65, 0x74, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x08, 0x47, 0x65, + 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x12, 0x1a, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, + 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, + 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, + 0x00, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, + 0x6f, 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x64, 0x69, + 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, + 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x25, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x50, + 0x65, 0x61, 0x6b, 0x73, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x41, + 0x75, 0x64, 0x69, 0x6f, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x2e, 0x6d, 0x65, + 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x64, 0x69, 0x6f, + 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, + 0x64, 0x69, 0x6f, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x22, 0x00, 0x30, 0x01, 0x12, 0x47, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, + 0x65, 0x6f, 0x12, 0x1a, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, + 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, + 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, + 0x64, 0x65, 0x6f, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x00, 0x30, 0x01, 0x12, + 0x60, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, + 0x6e, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x2e, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, 0x75, 0x6d, 0x62, 0x6e, 0x61, - 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x0e, 0x5a, 0x0c, - 0x70, 0x62, 0x2f, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, 0x74, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x65, 0x64, 0x69, + 0x61, 0x5f, 0x73, 0x65, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x54, 0x68, + 0x75, 0x6d, 0x62, 0x6e, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x0e, 0x5a, 0x0c, 0x70, 0x62, 0x2f, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x73, 0x65, + 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -762,37 +977,44 @@ func file_media_set_proto_rawDescGZIP() []byte { return file_media_set_proto_rawDescData } -var file_media_set_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_media_set_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_media_set_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_media_set_proto_goTypes = []interface{}{ - (*MediaSet)(nil), // 0: media_set.MediaSet - (*GetRequest)(nil), // 1: media_set.GetRequest - (*GetPeaksRequest)(nil), // 2: media_set.GetPeaksRequest - (*GetPeaksProgress)(nil), // 3: media_set.GetPeaksProgress - (*GetPeaksForSegmentRequest)(nil), // 4: media_set.GetPeaksForSegmentRequest - (*GetPeaksForSegmentResponse)(nil), // 5: media_set.GetPeaksForSegmentResponse - (*GetVideoRequest)(nil), // 6: media_set.GetVideoRequest - (*GetVideoProgress)(nil), // 7: media_set.GetVideoProgress - (*GetVideoThumbnailRequest)(nil), // 8: media_set.GetVideoThumbnailRequest - (*GetVideoThumbnailResponse)(nil), // 9: media_set.GetVideoThumbnailResponse - (*durationpb.Duration)(nil), // 10: google.protobuf.Duration + (AudioFormat)(0), // 0: media_set.AudioFormat + (*MediaSet)(nil), // 1: media_set.MediaSet + (*GetRequest)(nil), // 2: media_set.GetRequest + (*GetPeaksRequest)(nil), // 3: media_set.GetPeaksRequest + (*GetPeaksProgress)(nil), // 4: media_set.GetPeaksProgress + (*GetPeaksForSegmentRequest)(nil), // 5: media_set.GetPeaksForSegmentRequest + (*GetPeaksForSegmentResponse)(nil), // 6: media_set.GetPeaksForSegmentResponse + (*GetAudioSegmentRequest)(nil), // 7: media_set.GetAudioSegmentRequest + (*GetAudioSegmentProgress)(nil), // 8: media_set.GetAudioSegmentProgress + (*GetVideoRequest)(nil), // 9: media_set.GetVideoRequest + (*GetVideoProgress)(nil), // 10: media_set.GetVideoProgress + (*GetVideoThumbnailRequest)(nil), // 11: media_set.GetVideoThumbnailRequest + (*GetVideoThumbnailResponse)(nil), // 12: media_set.GetVideoThumbnailResponse + (*durationpb.Duration)(nil), // 13: google.protobuf.Duration } var file_media_set_proto_depIdxs = []int32{ - 10, // 0: media_set.MediaSet.video_duration:type_name -> google.protobuf.Duration - 1, // 1: media_set.MediaSetService.Get:input_type -> media_set.GetRequest - 2, // 2: media_set.MediaSetService.GetPeaks:input_type -> media_set.GetPeaksRequest - 4, // 3: media_set.MediaSetService.GetPeaksForSegment:input_type -> media_set.GetPeaksForSegmentRequest - 6, // 4: media_set.MediaSetService.GetVideo:input_type -> media_set.GetVideoRequest - 8, // 5: media_set.MediaSetService.GetVideoThumbnail:input_type -> media_set.GetVideoThumbnailRequest - 0, // 6: media_set.MediaSetService.Get:output_type -> media_set.MediaSet - 3, // 7: media_set.MediaSetService.GetPeaks:output_type -> media_set.GetPeaksProgress - 5, // 8: media_set.MediaSetService.GetPeaksForSegment:output_type -> media_set.GetPeaksForSegmentResponse - 7, // 9: media_set.MediaSetService.GetVideo:output_type -> media_set.GetVideoProgress - 9, // 10: media_set.MediaSetService.GetVideoThumbnail:output_type -> media_set.GetVideoThumbnailResponse - 6, // [6:11] is the sub-list for method output_type - 1, // [1:6] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 13, // 0: media_set.MediaSet.video_duration:type_name -> google.protobuf.Duration + 0, // 1: media_set.GetAudioSegmentRequest.format:type_name -> media_set.AudioFormat + 2, // 2: media_set.MediaSetService.Get:input_type -> media_set.GetRequest + 3, // 3: media_set.MediaSetService.GetPeaks:input_type -> media_set.GetPeaksRequest + 5, // 4: media_set.MediaSetService.GetPeaksForSegment:input_type -> media_set.GetPeaksForSegmentRequest + 7, // 5: media_set.MediaSetService.GetAudioSegment:input_type -> media_set.GetAudioSegmentRequest + 9, // 6: media_set.MediaSetService.GetVideo:input_type -> media_set.GetVideoRequest + 11, // 7: media_set.MediaSetService.GetVideoThumbnail:input_type -> media_set.GetVideoThumbnailRequest + 1, // 8: media_set.MediaSetService.Get:output_type -> media_set.MediaSet + 4, // 9: media_set.MediaSetService.GetPeaks:output_type -> media_set.GetPeaksProgress + 6, // 10: media_set.MediaSetService.GetPeaksForSegment:output_type -> media_set.GetPeaksForSegmentResponse + 8, // 11: media_set.MediaSetService.GetAudioSegment:output_type -> media_set.GetAudioSegmentProgress + 10, // 12: media_set.MediaSetService.GetVideo:output_type -> media_set.GetVideoProgress + 12, // 13: media_set.MediaSetService.GetVideoThumbnail:output_type -> media_set.GetVideoThumbnailResponse + 8, // [8:14] is the sub-list for method output_type + 2, // [2:8] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_media_set_proto_init() } @@ -874,7 +1096,7 @@ func file_media_set_proto_init() { } } file_media_set_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVideoRequest); i { + switch v := v.(*GetAudioSegmentRequest); i { case 0: return &v.state case 1: @@ -886,7 +1108,7 @@ func file_media_set_proto_init() { } } file_media_set_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVideoProgress); i { + switch v := v.(*GetAudioSegmentProgress); i { case 0: return &v.state case 1: @@ -898,7 +1120,7 @@ func file_media_set_proto_init() { } } file_media_set_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVideoThumbnailRequest); i { + switch v := v.(*GetVideoRequest); i { case 0: return &v.state case 1: @@ -910,6 +1132,30 @@ func file_media_set_proto_init() { } } file_media_set_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetVideoProgress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_media_set_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetVideoThumbnailRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_media_set_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetVideoThumbnailResponse); i { case 0: return &v.state @@ -927,13 +1173,14 @@ func file_media_set_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_media_set_proto_rawDesc, - NumEnums: 0, - NumMessages: 10, + NumEnums: 1, + NumMessages: 12, NumExtensions: 0, NumServices: 1, }, GoTypes: file_media_set_proto_goTypes, DependencyIndexes: file_media_set_proto_depIdxs, + EnumInfos: file_media_set_proto_enumTypes, MessageInfos: file_media_set_proto_msgTypes, }.Build() File_media_set_proto = out.File diff --git a/backend/generated/pb/media_set/media_set_grpc.pb.go b/backend/generated/pb/media_set/media_set_grpc.pb.go index 1302ec5..f84f544 100644 --- a/backend/generated/pb/media_set/media_set_grpc.pb.go +++ b/backend/generated/pb/media_set/media_set_grpc.pb.go @@ -21,6 +21,7 @@ type MediaSetServiceClient interface { Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*MediaSet, error) GetPeaks(ctx context.Context, in *GetPeaksRequest, opts ...grpc.CallOption) (MediaSetService_GetPeaksClient, error) GetPeaksForSegment(ctx context.Context, in *GetPeaksForSegmentRequest, opts ...grpc.CallOption) (*GetPeaksForSegmentResponse, error) + GetAudioSegment(ctx context.Context, in *GetAudioSegmentRequest, opts ...grpc.CallOption) (MediaSetService_GetAudioSegmentClient, error) GetVideo(ctx context.Context, in *GetVideoRequest, opts ...grpc.CallOption) (MediaSetService_GetVideoClient, error) GetVideoThumbnail(ctx context.Context, in *GetVideoThumbnailRequest, opts ...grpc.CallOption) (*GetVideoThumbnailResponse, error) } @@ -83,8 +84,40 @@ func (c *mediaSetServiceClient) GetPeaksForSegment(ctx context.Context, in *GetP return out, nil } +func (c *mediaSetServiceClient) GetAudioSegment(ctx context.Context, in *GetAudioSegmentRequest, opts ...grpc.CallOption) (MediaSetService_GetAudioSegmentClient, error) { + stream, err := c.cc.NewStream(ctx, &MediaSetService_ServiceDesc.Streams[1], "/media_set.MediaSetService/GetAudioSegment", opts...) + if err != nil { + return nil, err + } + x := &mediaSetServiceGetAudioSegmentClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type MediaSetService_GetAudioSegmentClient interface { + Recv() (*GetAudioSegmentProgress, error) + grpc.ClientStream +} + +type mediaSetServiceGetAudioSegmentClient struct { + grpc.ClientStream +} + +func (x *mediaSetServiceGetAudioSegmentClient) Recv() (*GetAudioSegmentProgress, error) { + m := new(GetAudioSegmentProgress) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *mediaSetServiceClient) GetVideo(ctx context.Context, in *GetVideoRequest, opts ...grpc.CallOption) (MediaSetService_GetVideoClient, error) { - stream, err := c.cc.NewStream(ctx, &MediaSetService_ServiceDesc.Streams[1], "/media_set.MediaSetService/GetVideo", opts...) + stream, err := c.cc.NewStream(ctx, &MediaSetService_ServiceDesc.Streams[2], "/media_set.MediaSetService/GetVideo", opts...) if err != nil { return nil, err } @@ -131,6 +164,7 @@ type MediaSetServiceServer interface { Get(context.Context, *GetRequest) (*MediaSet, error) GetPeaks(*GetPeaksRequest, MediaSetService_GetPeaksServer) error GetPeaksForSegment(context.Context, *GetPeaksForSegmentRequest) (*GetPeaksForSegmentResponse, error) + GetAudioSegment(*GetAudioSegmentRequest, MediaSetService_GetAudioSegmentServer) error GetVideo(*GetVideoRequest, MediaSetService_GetVideoServer) error GetVideoThumbnail(context.Context, *GetVideoThumbnailRequest) (*GetVideoThumbnailResponse, error) mustEmbedUnimplementedMediaSetServiceServer() @@ -149,6 +183,9 @@ func (UnimplementedMediaSetServiceServer) GetPeaks(*GetPeaksRequest, MediaSetSer func (UnimplementedMediaSetServiceServer) GetPeaksForSegment(context.Context, *GetPeaksForSegmentRequest) (*GetPeaksForSegmentResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetPeaksForSegment not implemented") } +func (UnimplementedMediaSetServiceServer) GetAudioSegment(*GetAudioSegmentRequest, MediaSetService_GetAudioSegmentServer) error { + return status.Errorf(codes.Unimplemented, "method GetAudioSegment not implemented") +} func (UnimplementedMediaSetServiceServer) GetVideo(*GetVideoRequest, MediaSetService_GetVideoServer) error { return status.Errorf(codes.Unimplemented, "method GetVideo not implemented") } @@ -225,6 +262,27 @@ func _MediaSetService_GetPeaksForSegment_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _MediaSetService_GetAudioSegment_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(GetAudioSegmentRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MediaSetServiceServer).GetAudioSegment(m, &mediaSetServiceGetAudioSegmentServer{stream}) +} + +type MediaSetService_GetAudioSegmentServer interface { + Send(*GetAudioSegmentProgress) error + grpc.ServerStream +} + +type mediaSetServiceGetAudioSegmentServer struct { + grpc.ServerStream +} + +func (x *mediaSetServiceGetAudioSegmentServer) Send(m *GetAudioSegmentProgress) error { + return x.ServerStream.SendMsg(m) +} + func _MediaSetService_GetVideo_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(GetVideoRequest) if err := stream.RecvMsg(m); err != nil { @@ -290,6 +348,11 @@ var MediaSetService_ServiceDesc = grpc.ServiceDesc{ Handler: _MediaSetService_GetPeaks_Handler, ServerStreams: true, }, + { + StreamName: "GetAudioSegment", + Handler: _MediaSetService_GetAudioSegment_Handler, + ServerStreams: true, + }, { StreamName: "GetVideo", Handler: _MediaSetService_GetVideo_Handler, diff --git a/backend/media/get_segment.go b/backend/media/get_segment.go new file mode 100644 index 0000000..068f16f --- /dev/null +++ b/backend/media/get_segment.go @@ -0,0 +1,159 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strconv" + "sync" +) + +// CommandFunc is a function that builds an *exec.Cmd from a context, name and +// args. +type CommandFunc func(ctx context.Context, name string, arg ...string) *exec.Cmd + +// AudioFormat represents an abstract audio format, e.g. MP3 or WAV. +type AudioFormat int + +const ( + AudioFormatWAV AudioFormat = iota + AudioFormatMP3 +) + +// String implements fmt.Stringer. +func (f AudioFormat) String() string { + switch f { + case AudioFormatWAV: + return "wav" + case AudioFormatMP3: + return "mp3" + default: + panic("unknown audio format") + } +} + +// AudioSegmentProgress represents a progress update for an AudioSegmentStream, +// and contains a byte slice of audio data and indication of the approximate +// progress. +type AudioSegmentProgress struct { + PercentComplete float32 + Data []byte +} + +// AudioSegmentStream is a stream of AudioSegmentProgress structs. +type AudioSegmentStream struct { + progressChan chan AudioSegmentProgress + errorChan chan error +} + +// send publishes a new partial segment and progress update to the strean. +func (s *AudioSegmentStream) send(p []byte, percentComplete float32) { + s.progressChan <- AudioSegmentProgress{ + Data: p, + PercentComplete: percentComplete, + } +} + +// close signals the successful end of the stream of data. +func (s *AudioSegmentStream) close() { + close(s.progressChan) +} + +// closeWithError signals the unsuccessful end of a stream of data. +func (s *AudioSegmentStream) closeWithError(err error) { + s.errorChan <- err +} + +// audioSegmentGetter gets an audio segment and streams it to the caller. +type audioSegmentGetter struct { + mu sync.Mutex + commandFunc CommandFunc + rawAudio io.ReadCloser + channels int32 + outFormat AudioFormat + stream *AudioSegmentStream + bytesRead, bytesExpected int64 +} + +// newAudioSegmentGetter returns a new audioSegmentGetter. The io.ReadCloser +// will be consumed and closed by the getAudioSegment() function. +func newAudioSegmentGetter(commandFunc CommandFunc, rawAudio io.ReadCloser, channels int32, bytesExpected int64, outFormat AudioFormat) *audioSegmentGetter { + return &audioSegmentGetter{ + commandFunc: commandFunc, + rawAudio: rawAudio, + channels: channels, + bytesExpected: bytesExpected, + outFormat: outFormat, + stream: &AudioSegmentStream{ + progressChan: make(chan AudioSegmentProgress), + errorChan: make(chan error, 1), + }, + } +} + +// Read implements io.Reader and is consumed by the stdin of the FFMPEG +// command. It is called from a separate goroutine to Write(). +func (s *audioSegmentGetter) Read(p []byte) (int, error) { + n, err := s.rawAudio.Read(p) + + s.mu.Lock() + defer s.mu.Unlock() + s.bytesRead += int64(n) + + return n, err +} + +// Write implements io.Writer and consumes the stdout of the FFMPEG command. It +// is called from a separate goroutine to Read(). +func (s *audioSegmentGetter) Write(p []byte) (int, error) { + s.stream.send(p, s.percentComplete()) + return len(p), nil +} + +func (s *audioSegmentGetter) percentComplete() float32 { + s.mu.Lock() + defer s.mu.Unlock() + + return (float32(s.bytesRead) / float32(s.bytesExpected)) * 100 +} + +// Next implements AudioSegmentStream. +func (s *AudioSegmentStream) Next(ctx context.Context) (AudioSegmentProgress, error) { + select { + case progress, ok := <-s.progressChan: + if !ok { + return AudioSegmentProgress{}, io.EOF + } + return progress, nil + case err := <-s.errorChan: + return AudioSegmentProgress{}, err + case <-ctx.Done(): + return AudioSegmentProgress{}, ctx.Err() + } +} + +func (s *audioSegmentGetter) getAudioSegment(ctx context.Context) { + defer s.rawAudio.Close() + + var stdErr bytes.Buffer + cmd := s.commandFunc(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-f", "s16le", "-ac", itoa(int(s.channels)), "-ar", itoa(rawAudioSampleRate), "-i", "-", "-f", s.outFormat.String(), "-") + cmd.Stderr = &stdErr + cmd.Stdin = s + cmd.Stdout = s + + if err := cmd.Start(); err != nil { + s.stream.closeWithError(fmt.Errorf("error starting command: %v, output: %s", err, stdErr.String())) + return + } + + if err := cmd.Wait(); err != nil { + s.stream.closeWithError(fmt.Errorf("error waiting for ffmpeg: %v, output: %s", err, stdErr.String())) + return + } + + s.stream.close() +} + +func itoa(i int) string { return strconv.Itoa(i) } diff --git a/backend/media/get_segment_test.go b/backend/media/get_segment_test.go new file mode 100644 index 0000000..2b1d0bd --- /dev/null +++ b/backend/media/get_segment_test.go @@ -0,0 +1,264 @@ +package media_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "testing" + + "git.netflux.io/rob/clipper/config" + "git.netflux.io/rob/clipper/generated/mocks" + "git.netflux.io/rob/clipper/generated/store" + "git.netflux.io/rob/clipper/media" + "github.com/google/uuid" + "github.com/jackc/pgx/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +const inFixturePath = "testdata/tone-44100-stereo-int16-30000ms.raw" + +func helperCommand(t *testing.T, wantCommand, stdoutFile, stderrString string, forceExitCode int) media.CommandFunc { + return func(ctx context.Context, name string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", name} + cs = append(cs, args...) + cmd := exec.CommandContext(ctx, os.Args[0], cs...) + cmd.Env = []string{ + "GO_WANT_HELPER_PROCESS=1", + "GO_WANT_COMMAND=" + wantCommand, + "GO_STDOUT_FILE=" + stdoutFile, + "GO_STDERR_STRING=" + stderrString, + "GO_FORCE_EXIT_CODE=" + strconv.Itoa(forceExitCode), + } + return cmd + } +} + +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + defer func() { + // Stop the helper process writing to stdout after the test has finished. + // This prevents it from writing the "PASS" string which is unwanted in + // this context. + if !t.Failed() { + os.Stdout, _ = os.Open(os.DevNull) + } + }() + + if exitCode := os.Getenv("GO_FORCE_EXIT_CODE"); exitCode != "0" { + c, _ := strconv.Atoi(exitCode) + os.Stderr.WriteString(os.Getenv("GO_STDERR_STRING")) + os.Exit(c) + } + + if wantCommand := os.Getenv("GO_WANT_COMMAND"); wantCommand != "" { + gotCmd := strings.Split(strings.Join(os.Args, " "), " -- ")[1] + if wantCommand != gotCmd { + fmt.Printf("GO_WANT_COMMAND assertion failed:\nwant = %v\ngot = %v", wantCommand, gotCmd) + return + } + } + + // Copy stdin to /dev/null. This is required to avoid broken pipe errors in + // the tests: + _, err := io.Copy(io.Discard, os.Stdin) + require.NoError(t, err) + + // If an output file is provided, then copy that to stdout: + if fname := os.Getenv("GO_STDOUT_FILE"); fname != "" { + fptr, err := os.Open(fname) + require.NoError(t, err) + + _, err = io.Copy(os.Stdout, fptr) + require.NoError(t, err) + } +} + +func fixtureReader(t *testing.T, limit int64) io.ReadCloser { + fptr, err := os.Open(inFixturePath) + require.NoError(t, err) + + // limitReader to make the mock work realistically, not intended for assertions: + return struct { + io.Reader + io.Closer + }{ + Reader: io.LimitReader(fptr, limit), + Closer: fptr, + } +} + +func TestGetSegment(t *testing.T) { + mediaSetID := uuid.MustParse("4c440241-cca9-436f-adb0-be074588cf2b") + + t.Run("invalid range", func(t *testing.T) { + var mockStore mocks.Store + var fileStore mocks.FileStore + service := media.NewMediaSetService(&mockStore, nil, &fileStore, nil, config.Config{}, zap.NewNop().Sugar()) + + stream, err := service.GetAudioSegment(context.Background(), mediaSetID, 1, 0, media.AudioFormatMP3) + require.Nil(t, stream) + require.EqualError(t, err, "invalid range") + }) + + t.Run("error fetching media set", func(t *testing.T) { + var mockStore mocks.Store + mockStore.On("GetMediaSet", mock.Anything, mediaSetID).Return(store.MediaSet{}, pgx.ErrNoRows) + var fileStore mocks.FileStore + service := media.NewMediaSetService(&mockStore, nil, &fileStore, nil, config.Config{}, zap.NewNop().Sugar()) + + stream, err := service.GetAudioSegment(context.Background(), mediaSetID, 0, 1, media.AudioFormatMP3) + require.Nil(t, stream) + require.EqualError(t, err, "error getting media set: no rows in result set") + }) + + t.Run("error fetching audio data", func(t *testing.T) { + mediaSet := store.MediaSet{ID: mediaSetID, AudioChannels: 2} + + var mockStore mocks.Store + mockStore.On("GetMediaSet", mock.Anything, mediaSetID).Return(mediaSet, nil) + + var fileStore mocks.FileStore + fileStore.On("GetObjectWithRange", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, errors.New("network error")) + + service := media.NewMediaSetService(&mockStore, nil, &fileStore, nil, config.Config{}, zap.NewNop().Sugar()) + + stream, err := service.GetAudioSegment(context.Background(), mediaSetID, 0, 1, media.AudioFormatMP3) + require.Nil(t, stream) + require.EqualError(t, err, "error getting object from store: network error") + }) + + t.Run("ffmpeg returns non-zero error code", func(t *testing.T) { + mediaSet := store.MediaSet{ID: mediaSetID, AudioChannels: 2} + + var mockStore mocks.Store + mockStore.On("GetMediaSet", mock.Anything, mediaSetID).Return(mediaSet, nil) + + var fileStore mocks.FileStore + fileStore.On("GetObjectWithRange", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(fixtureReader(t, 1), nil) + + cmd := helperCommand(t, "", "", "something bad happened", 2) + service := media.NewMediaSetService(&mockStore, nil, &fileStore, cmd, config.Config{}, zap.NewNop().Sugar()) + + stream, err := service.GetAudioSegment(context.Background(), mediaSetID, 0, 1, media.AudioFormatMP3) + require.NoError(t, err) + + _, err = stream.Next(context.Background()) + require.EqualError(t, err, "error waiting for ffmpeg: exit status 2, output: something bad happened") + + }) + + testCases := []struct { + name string + audioFormat media.AudioFormat + audioChannels int32 + inStartFrame, inEndFrame int64 + wantStartByte, wantEndByte int64 + outFixturePath string + wantCommand string + wantOutput string + }{ + { + name: "mono to mp3", + audioFormat: media.AudioFormatMP3, + audioChannels: 1, + inStartFrame: 500, + inEndFrame: 2_000, + wantStartByte: 1_000, + wantEndByte: 4_000, + outFixturePath: "testdata/fake.mp3", + wantCommand: "ffmpeg -hide_banner -loglevel error -f s16le -ac 1 -ar 48000 -i - -f mp3 -", + wantOutput: "this is a fake mp3", + }, + { + name: "stereo to mp3", + audioFormat: media.AudioFormatMP3, + audioChannels: 2, + inStartFrame: 0, + inEndFrame: 1_323_000, + wantStartByte: 0, + wantEndByte: 5_292_000, + outFixturePath: "testdata/fake.mp3", + wantCommand: "ffmpeg -hide_banner -loglevel error -f s16le -ac 2 -ar 48000 -i - -f mp3 -", + wantOutput: "this is a fake mp3", + }, + { + name: "mono to wav", + audioFormat: media.AudioFormatWAV, + audioChannels: 1, + inStartFrame: 16_384, + inEndFrame: 32_768, + wantStartByte: 32_768, + wantEndByte: 65_536, + outFixturePath: "testdata/fake.wav", + wantCommand: "ffmpeg -hide_banner -loglevel error -f s16le -ac 1 -ar 48000 -i - -f wav -", + wantOutput: "this is a fake wav", + }, + { + name: "stereo to wav", + audioFormat: media.AudioFormatWAV, + audioChannels: 2, + inStartFrame: 2_048, + inEndFrame: 4_096, + wantStartByte: 8_192, + wantEndByte: 16_384, + outFixturePath: "testdata/fake.wav", + wantCommand: "ffmpeg -hide_banner -loglevel error -f s16le -ac 2 -ar 48000 -i - -f wav -", + wantOutput: "this is a fake wav", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + + mediaSet := store.MediaSet{ID: mediaSetID, AudioChannels: tc.audioChannels} + var mockStore mocks.Store + mockStore.On("GetMediaSet", mock.Anything, mediaSetID).Return(mediaSet, nil) + defer mockStore.AssertExpectations(t) + + var fileStore mocks.FileStore + fileStore. + On("GetObjectWithRange", mock.Anything, "media_sets/4c440241-cca9-436f-adb0-be074588cf2b/audio.raw", tc.wantStartByte, tc.wantEndByte). + Return(fixtureReader(t, tc.wantEndByte-tc.wantStartByte), nil) + defer fileStore.AssertExpectations(t) + + cmd := helperCommand(t, tc.wantCommand, tc.outFixturePath, "", 0) + service := media.NewMediaSetService(&mockStore, nil, &fileStore, cmd, config.Config{}, zap.NewNop().Sugar()) + + stream, err := service.GetAudioSegment(ctx, mediaSetID, tc.inStartFrame, tc.inEndFrame, tc.audioFormat) + require.NoError(t, err) + + var data bytes.Buffer + var lastPercentComplete float32 + var progress media.AudioSegmentProgress + for { + progress, err = stream.Next(ctx) + if err == io.EOF { + break + } + + require.NoError(t, err) + assert.GreaterOrEqual(t, progress.PercentComplete, lastPercentComplete) + lastPercentComplete = progress.PercentComplete + data.Write(progress.Data) + } + + assert.Equal(t, tc.wantOutput, data.String()) + assert.Equal(t, float32(100), lastPercentComplete) + }) + } +} diff --git a/backend/media/service.go b/backend/media/service.go index cf70cc2..88305b6 100644 --- a/backend/media/service.go +++ b/backend/media/service.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "io" - "net/http" "strconv" "time" @@ -34,20 +33,22 @@ const ( // MediaSetService exposes logical flows handling MediaSets. type MediaSetService struct { - store Store - youtube YoutubeClient - fileStore FileStore - config config.Config - logger *zap.SugaredLogger + store Store + youtube YoutubeClient + fileStore FileStore + commandFunc CommandFunc + config config.Config + logger *zap.SugaredLogger } -func NewMediaSetService(store Store, youtubeClient YoutubeClient, fileStore FileStore, config config.Config, logger *zap.SugaredLogger) *MediaSetService { +func NewMediaSetService(store Store, youtubeClient YoutubeClient, fileStore FileStore, commandFunc CommandFunc, config config.Config, logger *zap.SugaredLogger) *MediaSetService { return &MediaSetService{ - store: store, - youtube: youtubeClient, - fileStore: fileStore, - config: config, - logger: logger, + store: store, + youtube: youtubeClient, + fileStore: fileStore, + commandFunc: commandFunc, + config: config, + logger: logger, } } @@ -250,7 +251,7 @@ func (s *MediaSetService) GetVideo(ctx context.Context, id uuid.UUID) (GetVideoP ) } -// GetAudio fetches the audio part of a MediaSet. +// GetPeaks fetches the audio part of a MediaSet. func (s *MediaSetService) GetPeaks(ctx context.Context, id uuid.UUID, numBins int) (GetPeaksProgressReader, error) { mediaSet, err := s.store.GetMediaSet(ctx, id) if err != nil { @@ -437,101 +438,30 @@ func (s *MediaSetService) GetPeaksForSegment(ctx context.Context, id uuid.UUID, return peaks, nil } -func sqlString(s string) sql.NullString { - return sql.NullString{String: s, Valid: true} -} +func (s *MediaSetService) GetAudioSegment(ctx context.Context, id uuid.UUID, startFrame, endFrame int64, outFormat AudioFormat) (*AudioSegmentStream, error) { + if startFrame > endFrame { + return nil, errors.New("invalid range") + } -func sqlInt64(i int64) sql.NullInt64 { - return sql.NullInt64{Int64: i, Valid: true} -} - -func sqlInt32(i int32) sql.NullInt32 { - return sql.NullInt32{Int32: i, Valid: true} -} - -type VideoThumbnail struct { - Data []byte - Width, Height int -} - -func (s *MediaSetService) GetVideoThumbnail(ctx context.Context, id uuid.UUID) (VideoThumbnail, error) { mediaSet, err := s.store.GetMediaSet(ctx, id) if err != nil { - return VideoThumbnail{}, fmt.Errorf("error getting media set: %v", err) - } - - if mediaSet.VideoThumbnailS3UploadedAt.Valid { - return s.getThumbnailFromFileStore(ctx, mediaSet) - } - - return s.getThumbnailFromYoutube(ctx, mediaSet) -} - -func (s *MediaSetService) getThumbnailFromFileStore(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { - object, err := s.fileStore.GetObject(ctx, mediaSet.VideoThumbnailS3Key.String) - if err != nil { - return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail from file store: %v", err) - } - defer object.Close() - - imageData, err := io.ReadAll(object) - if err != nil { - return VideoThumbnail{}, fmt.Errorf("error reading thumbnail from file store: %v", err) - } - - return VideoThumbnail{ - Width: int(mediaSet.VideoThumbnailWidth.Int32), - Height: int(mediaSet.VideoThumbnailHeight.Int32), - Data: imageData, - }, nil -} - -func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { - video, err := s.youtube.GetVideoContext(ctx, mediaSet.YoutubeID) - if err != nil { - return VideoThumbnail{}, fmt.Errorf("error fetching video: %v", err) - } - - if len(video.Formats) == 0 { - return VideoThumbnail{}, errors.New("no format available") - } - - thumbnails := video.Thumbnails - SortYoutubeThumbnails(thumbnails) - thumbnail := thumbnails[0] - - resp, err := http.Get(thumbnail.URL) - if err != nil { - return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail: %v", err) - } - defer resp.Body.Close() - - imageData, err := io.ReadAll(resp.Body) - if err != nil { - return VideoThumbnail{}, fmt.Errorf("error reading thumbnail: %v", err) + return nil, fmt.Errorf("error getting media set: %v", err) } // TODO: use mediaSet func to fetch key - thumbnailKey := fmt.Sprintf("media_sets/%s/thumbnail.jpg", mediaSet.ID) + key := fmt.Sprintf("media_sets/%s/audio.raw", mediaSet.ID) + startByte := startFrame * int64(mediaSet.AudioChannels) * SizeOfInt16 + endByte := endFrame * int64(mediaSet.AudioChannels) * SizeOfInt16 - const mimeType = "application/jpeg" - _, err = s.fileStore.PutObject(ctx, thumbnailKey, bytes.NewReader(imageData), mimeType) + rawAudio, err := s.fileStore.GetObjectWithRange(ctx, key, startByte, endByte) if err != nil { - return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err) + return nil, fmt.Errorf("error getting object from store: %v", err) } - storeParams := store.SetVideoThumbnailUploadedParams{ - ID: mediaSet.ID, - VideoThumbnailMimeType: sqlString(mimeType), - VideoThumbnailS3Key: sqlString(thumbnailKey), - VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)), - VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)), - } - if _, err := s.store.SetVideoThumbnailUploaded(ctx, storeParams); err != nil { - return VideoThumbnail{}, fmt.Errorf("error updating media set: %v", err) - } + g := newAudioSegmentGetter(s.commandFunc, rawAudio, mediaSet.AudioChannels, endByte-startByte, outFormat) + go g.getAudioSegment(ctx) - return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil + return g.stream, nil } // logProgressReader is a reader that prints progress logs as it reads. @@ -558,3 +488,15 @@ func (r *logProgressReader) Read(p []byte) (int, error) { return n, err } + +func sqlString(s string) sql.NullString { + return sql.NullString{String: s, Valid: true} +} + +func sqlInt64(i int64) sql.NullInt64 { + return sql.NullInt64{Int64: i, Valid: true} +} + +func sqlInt32(i int32) sql.NullInt32 { + return sql.NullInt32{Int32: i, Valid: true} +} diff --git a/backend/media/service_test.go b/backend/media/service_test.go index 99a5944..2615edd 100644 --- a/backend/media/service_test.go +++ b/backend/media/service_test.go @@ -6,6 +6,7 @@ import ( "database/sql" "io" "os" + "os/exec" "testing" "git.netflux.io/rob/clipper/config" @@ -110,7 +111,7 @@ func TestPeaksForSegment(t *testing.T) { On("GetObjectWithRange", mock.Anything, "foo", startByte, endByte). Return(audioData, nil) - service := media.NewMediaSetService(store, nil, fileStore, config.Config{}, zap.NewNop().Sugar()) + service := media.NewMediaSetService(store, nil, fileStore, exec.CommandContext, config.Config{}, zap.NewNop().Sugar()) peaks, err := service.GetPeaksForSegment(context.Background(), mediaSet.ID, tc.startFrame, tc.endFrame, tc.numBins) if tc.wantErr == "" { @@ -153,7 +154,7 @@ func BenchmarkGetPeaksForSegment(b *testing.B) { On("GetObjectWithRange", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(readCloser, nil) - service := media.NewMediaSetService(store, nil, fileStore, config.Config{}, zap.NewNop().Sugar()) + service := media.NewMediaSetService(store, nil, fileStore, exec.CommandContext, config.Config{}, zap.NewNop().Sugar()) b.StartTimer() _, err = service.GetPeaksForSegment(context.Background(), mediaSetID, startFrame, endFrame, numBins) diff --git a/backend/media/testdata/fake.mp3 b/backend/media/testdata/fake.mp3 new file mode 100644 index 0000000..c44160b --- /dev/null +++ b/backend/media/testdata/fake.mp3 @@ -0,0 +1 @@ +this is a fake mp3 \ No newline at end of file diff --git a/backend/media/testdata/fake.wav b/backend/media/testdata/fake.wav new file mode 100644 index 0000000..c3cf5f2 --- /dev/null +++ b/backend/media/testdata/fake.wav @@ -0,0 +1 @@ +this is a fake wav \ No newline at end of file diff --git a/backend/media/thumbnail.go b/backend/media/thumbnail.go new file mode 100644 index 0000000..021946b --- /dev/null +++ b/backend/media/thumbnail.go @@ -0,0 +1,98 @@ +package media + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + + "git.netflux.io/rob/clipper/generated/store" + "github.com/google/uuid" +) + +type VideoThumbnail struct { + Data []byte + Width, Height int +} + +func (s *MediaSetService) GetVideoThumbnail(ctx context.Context, id uuid.UUID) (VideoThumbnail, error) { + mediaSet, err := s.store.GetMediaSet(ctx, id) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error getting media set: %v", err) + } + + if mediaSet.VideoThumbnailS3UploadedAt.Valid { + return s.getThumbnailFromFileStore(ctx, mediaSet) + } + + return s.getThumbnailFromYoutube(ctx, mediaSet) +} + +func (s *MediaSetService) getThumbnailFromFileStore(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { + object, err := s.fileStore.GetObject(ctx, mediaSet.VideoThumbnailS3Key.String) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail from file store: %v", err) + } + defer object.Close() + + imageData, err := io.ReadAll(object) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error reading thumbnail from file store: %v", err) + } + + return VideoThumbnail{ + Width: int(mediaSet.VideoThumbnailWidth.Int32), + Height: int(mediaSet.VideoThumbnailHeight.Int32), + Data: imageData, + }, nil +} + +func (s *MediaSetService) getThumbnailFromYoutube(ctx context.Context, mediaSet store.MediaSet) (VideoThumbnail, error) { + video, err := s.youtube.GetVideoContext(ctx, mediaSet.YoutubeID) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching video: %v", err) + } + + if len(video.Formats) == 0 { + return VideoThumbnail{}, errors.New("no format available") + } + + thumbnails := video.Thumbnails + SortYoutubeThumbnails(thumbnails) + thumbnail := thumbnails[0] + + resp, err := http.Get(thumbnail.URL) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error fetching thumbnail: %v", err) + } + defer resp.Body.Close() + + imageData, err := io.ReadAll(resp.Body) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error reading thumbnail: %v", err) + } + + // TODO: use mediaSet func to fetch key + thumbnailKey := fmt.Sprintf("media_sets/%s/thumbnail.jpg", mediaSet.ID) + + const mimeType = "application/jpeg" + _, err = s.fileStore.PutObject(ctx, thumbnailKey, bytes.NewReader(imageData), mimeType) + if err != nil { + return VideoThumbnail{}, fmt.Errorf("error uploading thumbnail: %v", err) + } + + storeParams := store.SetVideoThumbnailUploadedParams{ + ID: mediaSet.ID, + VideoThumbnailMimeType: sqlString(mimeType), + VideoThumbnailS3Key: sqlString(thumbnailKey), + VideoThumbnailWidth: sqlInt32(int32(thumbnail.Width)), + VideoThumbnailHeight: sqlInt32(int32(thumbnail.Height)), + } + if _, err := s.store.SetVideoThumbnailUploaded(ctx, storeParams); err != nil { + return VideoThumbnail{}, fmt.Errorf("error updating media set: %v", err) + } + + return VideoThumbnail{Width: int(thumbnail.Width), Height: int(thumbnail.Height), Data: imageData}, nil +} diff --git a/backend/server/server.go b/backend/server/server.go index c7c6d7a..84af76e 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -2,9 +2,11 @@ package server import ( "context" + "errors" "fmt" "io" "net/http" + "os/exec" "time" "git.netflux.io/rob/clipper/config" @@ -35,6 +37,7 @@ const ( const ( getPeaksTimeout = time.Minute * 5 getPeaksForSegmentTimeout = time.Second * 10 + getAudioSegmentTimeout = time.Minute * 2 getVideoTimeout = time.Minute * 5 ) @@ -167,6 +170,51 @@ func (c *mediaSetServiceController) GetPeaksForSegment(ctx context.Context, requ return &pbmediaset.GetPeaksForSegmentResponse{Peaks: peaks32}, nil } +func (c *mediaSetServiceController) GetAudioSegment(request *pbmediaset.GetAudioSegmentRequest, outStream pbmediaset.MediaSetService_GetAudioSegmentServer) error { + ctx, cancel := context.WithTimeout(context.Background(), getPeaksForSegmentTimeout) + defer cancel() + + id, err := uuid.Parse(request.GetId()) + if err != nil { + return newResponseError(err) + } + + var format media.AudioFormat + switch request.Format { + case pbmediaset.AudioFormat_MP3: + format = media.AudioFormatMP3 + case pbmediaset.AudioFormat_WAV: + format = media.AudioFormatWAV + default: + return newResponseError(errors.New("unknown format")) + } + + stream, err := c.mediaSetService.GetAudioSegment(ctx, id, request.StartFrame, request.EndFrame, format) + if err != nil { + return newResponseError(err) + } + + for { + progress, err := stream.Next(ctx) + if err != nil && err != io.EOF { + return newResponseError(err) + } + + progressPb := pbmediaset.GetAudioSegmentProgress{ + PercentComplete: progress.PercentComplete, + AudioData: progress.Data, + } + + outStream.Send(&progressPb) + + if err == io.EOF { + break + } + } + + return nil +} + func (c *mediaSetServiceController) GetVideo(request *pbmediaset.GetVideoRequest, stream pbmediaset.MediaSetService_GetVideoServer) error { // TODO: reduce timeout when already fetched from Youtube ctx, cancel := context.WithTimeout(context.Background(), getVideoTimeout) @@ -227,6 +275,7 @@ func Start(options Options) error { options.Store, options.YoutubeClient, options.FileStore, + exec.CommandContext, options.Config, options.Logger.Sugar().Named("mediaSetService"), ) diff --git a/frontend/package.json b/frontend/package.json index eec789e..a5b8f89 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -43,6 +43,7 @@ ] }, "devDependencies": { + "@types/wicg-file-system-access": "^2020.9.4", "@typescript-eslint/eslint-plugin": "^4.31.0", "@typescript-eslint/parser": "^4.31.0", "eslint": "^7.32.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6363dc8..e105a0c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -7,6 +7,7 @@ import { } from './generated/media_set'; import { useState, useEffect, useRef, useCallback } from 'react'; +import { AudioFormat } from './generated/media_set'; import { VideoPreview } from './VideoPreview'; import { Overview, CanvasLogicalWidth } from './Overview'; import { Waveform } from './Waveform'; @@ -246,6 +247,35 @@ function App(): JSX.Element { } }, [audio, video, selection]); + const handleClip = useCallback(() => { + (async function () { + console.debug('clip', selection); + + if (mediaSet == null) { + return; + } + + // TODO: support File System Access API fallback + const h = await window.showSaveFilePicker({ suggestedName: 'clip.mp3' }); + const fileStream = await h.createWritable(); + + const rpc = newRPC(); + const service = new MediaSetServiceClientImpl(rpc); + const stream = service.GetAudioSegment({ + id: mediaSet.id, + format: AudioFormat.MP3, + startFrame: selection.start, + endFrame: selection.end, + }); + + await stream.forEach((p) => fileStream.write(p.audioData)); + console.debug('finished writing stream'); + + await fileStream.close(); + console.debug('closed stream'); + })(); + }, [mediaSet, selection]); + const setPositionFromFrame = useCallback( (frame: number) => { if (mediaSet == null) { @@ -297,7 +327,11 @@ function App(): JSX.Element { <>
- + void; onPause: () => void; + onClip: () => void; } const ControlBar: React.FC = React.memo((props: Props) => { @@ -26,6 +27,9 @@ const ControlBar: React.FC = React.memo((props: Props) => { +
); diff --git a/frontend/src/generated/media_set.ts b/frontend/src/generated/media_set.ts index e45ebf7..81b1c7b 100644 --- a/frontend/src/generated/media_set.ts +++ b/frontend/src/generated/media_set.ts @@ -9,6 +9,38 @@ import { share } from "rxjs/operators"; export const protobufPackage = "media_set"; +export enum AudioFormat { + WAV = 0, + MP3 = 1, + UNRECOGNIZED = -1, +} + +export function audioFormatFromJSON(object: any): AudioFormat { + switch (object) { + case 0: + case "WAV": + return AudioFormat.WAV; + case 1: + case "MP3": + return AudioFormat.MP3; + case -1: + case "UNRECOGNIZED": + default: + return AudioFormat.UNRECOGNIZED; + } +} + +export function audioFormatToJSON(object: AudioFormat): string { + switch (object) { + case AudioFormat.WAV: + return "WAV"; + case AudioFormat.MP3: + return "MP3"; + default: + return "UNKNOWN"; + } +} + export interface MediaSet { id: string; youtubeId: string; @@ -49,6 +81,20 @@ export interface GetPeaksForSegmentResponse { peaks: number[]; } +export interface GetAudioSegmentRequest { + id: string; + startFrame: number; + endFrame: number; + format: AudioFormat; +} + +export interface GetAudioSegmentProgress { + mimeType: string; + message: string; + percentComplete: number; + audioData: Uint8Array; +} + export interface GetVideoRequest { id: string; } @@ -652,6 +698,213 @@ export const GetPeaksForSegmentResponse = { }, }; +const baseGetAudioSegmentRequest: object = { + id: "", + startFrame: 0, + endFrame: 0, + format: 0, +}; + +export const GetAudioSegmentRequest = { + encode( + message: GetAudioSegmentRequest, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.id !== "") { + writer.uint32(10).string(message.id); + } + if (message.startFrame !== 0) { + writer.uint32(16).int64(message.startFrame); + } + if (message.endFrame !== 0) { + writer.uint32(24).int64(message.endFrame); + } + if (message.format !== 0) { + writer.uint32(32).int32(message.format); + } + return writer; + }, + + decode( + input: _m0.Reader | Uint8Array, + length?: number + ): GetAudioSegmentRequest { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = { ...baseGetAudioSegmentRequest } as GetAudioSegmentRequest; + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.id = reader.string(); + break; + case 2: + message.startFrame = longToNumber(reader.int64() as Long); + break; + case 3: + message.endFrame = longToNumber(reader.int64() as Long); + break; + case 4: + message.format = reader.int32() as any; + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }, + + fromJSON(object: any): GetAudioSegmentRequest { + const message = { ...baseGetAudioSegmentRequest } as GetAudioSegmentRequest; + message.id = + object.id !== undefined && object.id !== null ? String(object.id) : ""; + message.startFrame = + object.startFrame !== undefined && object.startFrame !== null + ? Number(object.startFrame) + : 0; + message.endFrame = + object.endFrame !== undefined && object.endFrame !== null + ? Number(object.endFrame) + : 0; + message.format = + object.format !== undefined && object.format !== null + ? audioFormatFromJSON(object.format) + : 0; + return message; + }, + + toJSON(message: GetAudioSegmentRequest): unknown { + const obj: any = {}; + message.id !== undefined && (obj.id = message.id); + message.startFrame !== undefined && (obj.startFrame = message.startFrame); + message.endFrame !== undefined && (obj.endFrame = message.endFrame); + message.format !== undefined && + (obj.format = audioFormatToJSON(message.format)); + return obj; + }, + + fromPartial, I>>( + object: I + ): GetAudioSegmentRequest { + const message = { ...baseGetAudioSegmentRequest } as GetAudioSegmentRequest; + message.id = object.id ?? ""; + message.startFrame = object.startFrame ?? 0; + message.endFrame = object.endFrame ?? 0; + message.format = object.format ?? 0; + return message; + }, +}; + +const baseGetAudioSegmentProgress: object = { + mimeType: "", + message: "", + percentComplete: 0, +}; + +export const GetAudioSegmentProgress = { + encode( + message: GetAudioSegmentProgress, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.mimeType !== "") { + writer.uint32(10).string(message.mimeType); + } + if (message.message !== "") { + writer.uint32(18).string(message.message); + } + if (message.percentComplete !== 0) { + writer.uint32(29).float(message.percentComplete); + } + if (message.audioData.length !== 0) { + writer.uint32(34).bytes(message.audioData); + } + return writer; + }, + + decode( + input: _m0.Reader | Uint8Array, + length?: number + ): GetAudioSegmentProgress { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = { + ...baseGetAudioSegmentProgress, + } as GetAudioSegmentProgress; + message.audioData = new Uint8Array(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.mimeType = reader.string(); + break; + case 2: + message.message = reader.string(); + break; + case 3: + message.percentComplete = reader.float(); + break; + case 4: + message.audioData = reader.bytes(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }, + + fromJSON(object: any): GetAudioSegmentProgress { + const message = { + ...baseGetAudioSegmentProgress, + } as GetAudioSegmentProgress; + message.mimeType = + object.mimeType !== undefined && object.mimeType !== null + ? String(object.mimeType) + : ""; + message.message = + object.message !== undefined && object.message !== null + ? String(object.message) + : ""; + message.percentComplete = + object.percentComplete !== undefined && object.percentComplete !== null + ? Number(object.percentComplete) + : 0; + message.audioData = + object.audioData !== undefined && object.audioData !== null + ? bytesFromBase64(object.audioData) + : new Uint8Array(); + return message; + }, + + toJSON(message: GetAudioSegmentProgress): unknown { + const obj: any = {}; + message.mimeType !== undefined && (obj.mimeType = message.mimeType); + message.message !== undefined && (obj.message = message.message); + message.percentComplete !== undefined && + (obj.percentComplete = message.percentComplete); + message.audioData !== undefined && + (obj.audioData = base64FromBytes( + message.audioData !== undefined ? message.audioData : new Uint8Array() + )); + return obj; + }, + + fromPartial, I>>( + object: I + ): GetAudioSegmentProgress { + const message = { + ...baseGetAudioSegmentProgress, + } as GetAudioSegmentProgress; + message.mimeType = object.mimeType ?? ""; + message.message = object.message ?? ""; + message.percentComplete = object.percentComplete ?? 0; + message.audioData = object.audioData ?? new Uint8Array(); + return message; + }, +}; + const baseGetVideoRequest: object = { id: "" }; export const GetVideoRequest = { @@ -938,6 +1191,10 @@ export interface MediaSetService { request: DeepPartial, metadata?: grpc.Metadata ): Promise; + GetAudioSegment( + request: DeepPartial, + metadata?: grpc.Metadata + ): Observable; GetVideo( request: DeepPartial, metadata?: grpc.Metadata @@ -956,6 +1213,7 @@ export class MediaSetServiceClientImpl implements MediaSetService { this.Get = this.Get.bind(this); this.GetPeaks = this.GetPeaks.bind(this); this.GetPeaksForSegment = this.GetPeaksForSegment.bind(this); + this.GetAudioSegment = this.GetAudioSegment.bind(this); this.GetVideo = this.GetVideo.bind(this); this.GetVideoThumbnail = this.GetVideoThumbnail.bind(this); } @@ -993,6 +1251,17 @@ export class MediaSetServiceClientImpl implements MediaSetService { ); } + GetAudioSegment( + request: DeepPartial, + metadata?: grpc.Metadata + ): Observable { + return this.rpc.invoke( + MediaSetServiceGetAudioSegmentDesc, + GetAudioSegmentRequest.fromPartial(request), + metadata + ); + } + GetVideo( request: DeepPartial, metadata?: grpc.Metadata @@ -1086,6 +1355,28 @@ export const MediaSetServiceGetPeaksForSegmentDesc: UnaryMethodDefinitionish = { } as any, }; +export const MediaSetServiceGetAudioSegmentDesc: UnaryMethodDefinitionish = { + methodName: "GetAudioSegment", + service: MediaSetServiceDesc, + requestStream: false, + responseStream: true, + requestType: { + serializeBinary() { + return GetAudioSegmentRequest.encode(this).finish(); + }, + } as any, + responseType: { + deserializeBinary(data: Uint8Array) { + return { + ...GetAudioSegmentProgress.decode(data), + toObject() { + return this; + }, + }; + }, + } as any, +}; + export const MediaSetServiceGetVideoDesc: UnaryMethodDefinitionish = { methodName: "GetVideo", service: MediaSetServiceDesc, diff --git a/frontend/src/generated/media_set_pb.js b/frontend/src/generated/media_set_pb.js index d1fbaf7..bde4297 100644 --- a/frontend/src/generated/media_set_pb.js +++ b/frontend/src/generated/media_set_pb.js @@ -17,6 +17,9 @@ var global = Function('return this')(); var google_protobuf_duration_pb = require('google-protobuf/google/protobuf/duration_pb.js'); goog.object.extend(proto, google_protobuf_duration_pb); +goog.exportSymbol('proto.media_set.AudioFormat', null, global); +goog.exportSymbol('proto.media_set.GetAudioSegmentProgress', null, global); +goog.exportSymbol('proto.media_set.GetAudioSegmentRequest', null, global); goog.exportSymbol('proto.media_set.GetPeaksForSegmentRequest', null, global); goog.exportSymbol('proto.media_set.GetPeaksForSegmentResponse', null, global); goog.exportSymbol('proto.media_set.GetPeaksProgress', null, global); @@ -153,6 +156,48 @@ if (goog.DEBUG && !COMPILED) { */ proto.media_set.GetPeaksForSegmentResponse.displayName = 'proto.media_set.GetPeaksForSegmentResponse'; } +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.media_set.GetAudioSegmentRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.media_set.GetAudioSegmentRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.media_set.GetAudioSegmentRequest.displayName = 'proto.media_set.GetAudioSegmentRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.media_set.GetAudioSegmentProgress = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.media_set.GetAudioSegmentProgress, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.media_set.GetAudioSegmentProgress.displayName = 'proto.media_set.GetAudioSegmentProgress'; +} /** * Generated by JsPbCodeGenerator. * @param {Array=} opt_data Optional initial data array, typically from a @@ -1577,6 +1622,470 @@ proto.media_set.GetPeaksForSegmentResponse.prototype.clearPeaksList = function() +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.media_set.GetAudioSegmentRequest.prototype.toObject = function(opt_includeInstance) { + return proto.media_set.GetAudioSegmentRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.media_set.GetAudioSegmentRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.media_set.GetAudioSegmentRequest.toObject = function(includeInstance, msg) { + var f, obj = { + id: jspb.Message.getFieldWithDefault(msg, 1, ""), + startFrame: jspb.Message.getFieldWithDefault(msg, 2, 0), + endFrame: jspb.Message.getFieldWithDefault(msg, 3, 0), + format: jspb.Message.getFieldWithDefault(msg, 4, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.media_set.GetAudioSegmentRequest} + */ +proto.media_set.GetAudioSegmentRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.media_set.GetAudioSegmentRequest; + return proto.media_set.GetAudioSegmentRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.media_set.GetAudioSegmentRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.media_set.GetAudioSegmentRequest} + */ +proto.media_set.GetAudioSegmentRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setId(value); + break; + case 2: + var value = /** @type {number} */ (reader.readInt64()); + msg.setStartFrame(value); + break; + case 3: + var value = /** @type {number} */ (reader.readInt64()); + msg.setEndFrame(value); + break; + case 4: + var value = /** @type {!proto.media_set.AudioFormat} */ (reader.readEnum()); + msg.setFormat(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.media_set.GetAudioSegmentRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.media_set.GetAudioSegmentRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.media_set.GetAudioSegmentRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.media_set.GetAudioSegmentRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getId(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getStartFrame(); + if (f !== 0) { + writer.writeInt64( + 2, + f + ); + } + f = message.getEndFrame(); + if (f !== 0) { + writer.writeInt64( + 3, + f + ); + } + f = message.getFormat(); + if (f !== 0.0) { + writer.writeEnum( + 4, + f + ); + } +}; + + +/** + * optional string id = 1; + * @return {string} + */ +proto.media_set.GetAudioSegmentRequest.prototype.getId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.media_set.GetAudioSegmentRequest} returns this + */ +proto.media_set.GetAudioSegmentRequest.prototype.setId = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional int64 start_frame = 2; + * @return {number} + */ +proto.media_set.GetAudioSegmentRequest.prototype.getStartFrame = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.media_set.GetAudioSegmentRequest} returns this + */ +proto.media_set.GetAudioSegmentRequest.prototype.setStartFrame = function(value) { + return jspb.Message.setProto3IntField(this, 2, value); +}; + + +/** + * optional int64 end_frame = 3; + * @return {number} + */ +proto.media_set.GetAudioSegmentRequest.prototype.getEndFrame = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 3, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.media_set.GetAudioSegmentRequest} returns this + */ +proto.media_set.GetAudioSegmentRequest.prototype.setEndFrame = function(value) { + return jspb.Message.setProto3IntField(this, 3, value); +}; + + +/** + * optional AudioFormat format = 4; + * @return {!proto.media_set.AudioFormat} + */ +proto.media_set.GetAudioSegmentRequest.prototype.getFormat = function() { + return /** @type {!proto.media_set.AudioFormat} */ (jspb.Message.getFieldWithDefault(this, 4, 0)); +}; + + +/** + * @param {!proto.media_set.AudioFormat} value + * @return {!proto.media_set.GetAudioSegmentRequest} returns this + */ +proto.media_set.GetAudioSegmentRequest.prototype.setFormat = function(value) { + return jspb.Message.setProto3EnumField(this, 4, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.media_set.GetAudioSegmentProgress.prototype.toObject = function(opt_includeInstance) { + return proto.media_set.GetAudioSegmentProgress.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.media_set.GetAudioSegmentProgress} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.media_set.GetAudioSegmentProgress.toObject = function(includeInstance, msg) { + var f, obj = { + mimeType: jspb.Message.getFieldWithDefault(msg, 1, ""), + message: jspb.Message.getFieldWithDefault(msg, 2, ""), + percentComplete: jspb.Message.getFloatingPointFieldWithDefault(msg, 3, 0.0), + audioData: msg.getAudioData_asB64() + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.media_set.GetAudioSegmentProgress} + */ +proto.media_set.GetAudioSegmentProgress.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.media_set.GetAudioSegmentProgress; + return proto.media_set.GetAudioSegmentProgress.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.media_set.GetAudioSegmentProgress} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.media_set.GetAudioSegmentProgress} + */ +proto.media_set.GetAudioSegmentProgress.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setMimeType(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setMessage(value); + break; + case 3: + var value = /** @type {number} */ (reader.readFloat()); + msg.setPercentComplete(value); + break; + case 4: + var value = /** @type {!Uint8Array} */ (reader.readBytes()); + msg.setAudioData(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.media_set.GetAudioSegmentProgress.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.media_set.GetAudioSegmentProgress.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.media_set.GetAudioSegmentProgress} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.media_set.GetAudioSegmentProgress.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getMimeType(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getMessage(); + if (f.length > 0) { + writer.writeString( + 2, + f + ); + } + f = message.getPercentComplete(); + if (f !== 0.0) { + writer.writeFloat( + 3, + f + ); + } + f = message.getAudioData_asU8(); + if (f.length > 0) { + writer.writeBytes( + 4, + f + ); + } +}; + + +/** + * optional string mime_type = 1; + * @return {string} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getMimeType = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.media_set.GetAudioSegmentProgress} returns this + */ +proto.media_set.GetAudioSegmentProgress.prototype.setMimeType = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional string message = 2; + * @return {string} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getMessage = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.media_set.GetAudioSegmentProgress} returns this + */ +proto.media_set.GetAudioSegmentProgress.prototype.setMessage = function(value) { + return jspb.Message.setProto3StringField(this, 2, value); +}; + + +/** + * optional float percent_complete = 3; + * @return {number} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getPercentComplete = function() { + return /** @type {number} */ (jspb.Message.getFloatingPointFieldWithDefault(this, 3, 0.0)); +}; + + +/** + * @param {number} value + * @return {!proto.media_set.GetAudioSegmentProgress} returns this + */ +proto.media_set.GetAudioSegmentProgress.prototype.setPercentComplete = function(value) { + return jspb.Message.setProto3FloatField(this, 3, value); +}; + + +/** + * optional bytes audio_data = 4; + * @return {!(string|Uint8Array)} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getAudioData = function() { + return /** @type {!(string|Uint8Array)} */ (jspb.Message.getFieldWithDefault(this, 4, "")); +}; + + +/** + * optional bytes audio_data = 4; + * This is a type-conversion wrapper around `getAudioData()` + * @return {string} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getAudioData_asB64 = function() { + return /** @type {string} */ (jspb.Message.bytesAsB64( + this.getAudioData())); +}; + + +/** + * optional bytes audio_data = 4; + * Note that Uint8Array is not supported on all browsers. + * @see http://caniuse.com/Uint8Array + * This is a type-conversion wrapper around `getAudioData()` + * @return {!Uint8Array} + */ +proto.media_set.GetAudioSegmentProgress.prototype.getAudioData_asU8 = function() { + return /** @type {!Uint8Array} */ (jspb.Message.bytesAsU8( + this.getAudioData())); +}; + + +/** + * @param {!(string|Uint8Array)} value + * @return {!proto.media_set.GetAudioSegmentProgress} returns this + */ +proto.media_set.GetAudioSegmentProgress.prototype.setAudioData = function(value) { + return jspb.Message.setProto3BytesField(this, 4, value); +}; + + + + + if (jspb.Message.GENERATE_TO_OBJECT) { /** * Creates an object representation of this proto. @@ -2208,4 +2717,12 @@ proto.media_set.GetVideoThumbnailResponse.prototype.setHeight = function(value) }; +/** + * @enum {number} + */ +proto.media_set.AudioFormat = { + WAV: 0, + MP3: 1 +}; + goog.object.extend(exports, proto.media_set); diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 3a63e3e..672bec9 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2056,6 +2056,11 @@ anymatch "^3.0.0" source-map "^0.6.0" +"@types/wicg-file-system-access@^2020.9.4": + version "2020.9.4" + resolved "https://registry.yarnpkg.com/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.4.tgz#83f255d6bd20b0ae131d555693473d15a0574e92" + integrity sha512-o43jUljwP0ZrQ927mPjGdJaBMfS12nf3VPj6Z52fMucxILrSs8tnfLbMDSn6cP3hrrLChc3SYneeEvecknNVtA== + "@types/yargs-parser@*": version "20.2.1" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.1.tgz#3b9ce2489919d9e4fea439b76916abc34b2df129" diff --git a/proto/media_set.proto b/proto/media_set.proto index c226ec3..2a425c8 100644 --- a/proto/media_set.proto +++ b/proto/media_set.proto @@ -49,6 +49,23 @@ message GetPeaksForSegmentResponse { repeated int32 peaks = 1; } +enum AudioFormat { + WAV = 0; + MP3 = 1; +} + +message GetAudioSegmentRequest { + string id = 1; + int64 start_frame = 2; + int64 end_frame = 3; + AudioFormat format = 4; +} + +message GetAudioSegmentProgress { + float percent_complete = 3; + bytes audio_data = 4; +} + message GetVideoRequest { string id = 1; } @@ -72,6 +89,7 @@ service MediaSetService { rpc Get(GetRequest) returns (MediaSet) {} rpc GetPeaks(GetPeaksRequest) returns (stream GetPeaksProgress) {} rpc GetPeaksForSegment(GetPeaksForSegmentRequest) returns (GetPeaksForSegmentResponse) {} + rpc GetAudioSegment(GetAudioSegmentRequest) returns (stream GetAudioSegmentProgress) {} rpc GetVideo(GetVideoRequest) returns (stream GetVideoProgress) {} rpc GetVideoThumbnail(GetVideoThumbnailRequest) returns (GetVideoThumbnailResponse) {} }