Ignore empty video frames (#52)

.changeset/brave-ties-search.md

@@ -0,0 +1,5 @@
+---
+"@livekit/track-processors": patch
+---
+
+Ignore empty video frames

src/ProcessorWrapper.ts

@@ -32,7 +32,6 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
   constructor(transformer: TrackTransformer<TransformerOptions>, name: string) {
     this.name = name;
     this.transformer = transformer;
-    this.transformer.restart;
   }
 
   private async setup(opts: ProcessorOptions<Track.Kind>) {
@@ -46,15 +45,19 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
       // in order to prevent this, we force the resize mode to "none"
       resizeMode: 'none',
     });
+
     this.sourceSettings = this.source.getSettings();
     this.sourceDummy = opts.element;
+
+    if (!(this.sourceDummy instanceof HTMLVideoElement)) {
+      throw TypeError('Currently only video transformers are supported');
+    }
+
     if (this.sourceDummy instanceof HTMLVideoElement) {
       this.sourceDummy.height = this.sourceSettings.height ?? 300;
       this.sourceDummy.width = this.sourceSettings.width ?? 300;
     }
-    if (!(this.sourceDummy instanceof HTMLVideoElement)) {
-      throw TypeError('Currently only video transformers are supported');
-    }
+
     // TODO explore if we can do all the processing work in a webworker
     this.processor = new MediaStreamTrackProcessor({ track: this.source });
 
@@ -75,15 +78,16 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
       throw new TypeError('Expected both canvas and processor to be defined after setup');
     }
 
-    let readableStream = this.processor.readable;
+    const readableStream = this.processor.readable;
 
     await this.transformer.init({
       outputCanvas: this.canvas,
       inputElement: this.sourceDummy as HTMLVideoElement,
     });
-    readableStream = readableStream.pipeThrough(this.transformer!.transformer!);
 
-    readableStream
+    const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);
+
+    pipedStream
       .pipeTo(this.trackGenerator.writable)
       .catch((e) => console.error('error when trying to pipe', e))
       .finally(() => this.destroy());

src/transformers/BackgroundTransformer.ts

@@ -58,7 +58,9 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
 
     // Skip loading the image here if update already loaded the image below
     if (this.options?.imagePath && !this.backgroundImage) {
-      await this.loadBackground(this.options.imagePath).catch((err) => console.error("Error while loading processor background image: ", err));
+      await this.loadBackground(this.options.imagePath).catch((err) =>
+        console.error('Error while loading processor background image: ', err),
+      );
     }
   }
 
@@ -83,6 +85,10 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
 
   async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
     try {
+      if (!(frame instanceof VideoFrame)) {
+        console.debug('empty frame detected, ignoring');
+        return;
+      }
       if (this.isDisabled) {
         controller.enqueue(frame);
         return;
@@ -107,7 +113,7 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
       });
       controller.enqueue(newFrame);
     } finally {
-      frame.close();
+      frame?.close();
     }
   }
 

src/transformers/VideoTransformer.ts

@@ -20,6 +20,7 @@ export default abstract class VideoTransformer<Options extends Record<string, un
     if (!(inputVideo instanceof HTMLVideoElement)) {
       throw TypeError('Video transformer needs a HTMLVideoElement as input');
     }
+
     this.transformer = new TransformStream({
       transform: (frame, controller) => this.transform(frame, controller),
     });

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
-    "lib": ["DOM", "ES2017"],
-    "target": "ES2017" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
+    "lib": ["DOM", "ES2018"],
+    "target": "ES2018" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
     "module": "ES2020" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
     "rootDir": "./",
     "declarationDir": "./dist",