Decoding and rendering works, with a few bugs of course.

player/src/video/decoder.ts

@@ -46,19 +46,26 @@ export class Decoder {
             }
         });
 
-        decoder.configure({
-            codec: video.codec,
-            codedHeight: video.track_height,
-            codedWidth: video.track_width,
-            // optimizeForLatency: true
-        })
-
 		const input = MP4.New();
 
         input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => {
             for (let sample of samples) {
-                const timestamp = sample.dts / (1000 / info.timescale) // milliseconds
+                const timestamp = 1000 * sample.dts / sample.timescale // milliseconds
-                console.log(sample)
+
+                if (sample.is_sync) {
+                    // Configure the decoder using the AVC box for H.264
+                    const avcc = sample.description.avcC;
+                    const description = new MP4.Stream(new Uint8Array(avcc.size), 0, false)
+                    avcc.write(description)
+
+                    decoder.configure({
+                        codec: video.codec,
+                        codedHeight: video.track_height,
+                        codedWidth: video.track_width,
+                        description: description.buffer?.slice(8),
+                        // optimizeForLatency: true
+                    })
+                }
 
                 decoder.decode(new EncodedVideoChunk({
                     data: sample.data,

player/src/video/renderer.ts

@@ -46,7 +46,8 @@ export class Renderer {
 
         let frame = this.queue[0]
         if (frame.timestamp > target) {
-            // nothing to render yet
+            // nothing to render yet, wait for the next animation frame
+            this.render = self.requestAnimationFrame(this.draw.bind(this))
             return
         }
 
@@ -59,14 +60,25 @@ export class Renderer {
                 break
             }
 
+            console.log("dropping frame")
+
+            frame.close()
+
             this.queue.shift()
             frame = next
         }
 
         const ctx = this.canvas.getContext("2d");
-        ctx?.drawImage(frame, 0, 0) // TODO aspect ratio
+        ctx?.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) // TODO aspect ratio
 
         this.last = frame.timestamp;
         frame.close()
+
+        if (this.queue.length > 0) {
+            this.render = self.requestAnimationFrame(this.draw.bind(this))
+        } else {
+            // Break the loop for now
+            this.render = 0
+        }
     }
 }