Add background blur video frame processor (#1686)

* Add background blur video frame processor

* Fix change log and package.json for blur

* Updated documents and fixed removing observer for background blur

.github/workflows/background-blur-integration.yml

@@ -0,0 +1,79 @@
+  name: Background Blur Integration Workflow
+
+  on:
+    pull_request:
+      branches:
+        - master
+        - release-1.x
+
+  env:
+    SELENIUM_GRID_PROVIDER: saucelabs
+    CLOUD_WATCH_METRIC: false
+    TEST_TYPE: Github-Action
+    SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}}
+    SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}}
+
+  jobs:
+    integ-background-blur:
+      name: Background Blur Integration Test
+      runs-on: ubuntu-latest
+
+      steps:
+        - name: Checkout Package
+          uses: actions/checkout@v2
+          with:
+            fetch-depth: 0
+        - name: Pack the Chime SDK and install the tarball into the Demo
+          run: |
+            current_version=$(.github/script/get-current-version)
+            echo "Packing current version:" $current_version
+            npm run build
+            npm pack
+            cd demos/browser
+            npm uninstall amazon-chime-sdk-js
+            npm install ../../amazon-chime-sdk-js-$current_version.tgz
+        - name: Check if needed to run
+          id: test_needed
+          run: |
+            source ${GITHUB_WORKSPACE}/integration/js/script/need-integ-test
+            check_if_integ_tests_required
+            echo ::set-output name=integ_test_required::$requires_integration_test
+        - name: Create a Job ID
+          id: create-job-id
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          uses: filipstefansson/uuid-action@ce29ebbb0981ac2448c2e406e848bfaa30ddf04c
+        - name: Set JOB_ID Env Variable
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: echo "JOB_ID=${{ steps.create-job-id.outputs.uuid }}" >> $GITHUB_ENV
+        - name: Echo Job ID
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: echo "${{ steps.create-job-id.outputs.uuid }}"
+        - name: Configure AWS Credentials
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          uses: aws-actions/configure-aws-credentials@v1
+          with:
+            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            aws-region: us-east-1
+        - name: Setup Sauce Connect
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          uses: saucelabs/sauce-connect-action@v1
+          with:
+            username: ${{ secrets.SAUCE_USERNAME }}
+            accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+            noSSLBumpDomains: all
+            tunnelIdentifier: ${{ steps.create-job-id.outputs.uuid }}
+        - name: Setup Node.js - 14.x
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          uses: actions/setup-node@v1
+          with:
+            node-version: 14.x
+        - name: Install Kite
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: integration/js/script/install-kite
+        - name: Clean Install
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: npm ci
+        - name: Run Background Blur Integration Test
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: npm run test:integration-background-blur

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- Add background blur video frame processor to enable background blur on streaming video. See [guide](https://aws.github.io/amazon-chime-sdk-js/modules/backgroundfilter_video_processor.html)
+
+### Removed
+
+### Fixed
+
+### Changed
+
 ## [2.19.0] - 2021-10-14
 
 ### Added

README.md

@@ -59,6 +59,7 @@ The following developer guides cover specific topics for a technical audience.
 - [Meeting events](https://aws.github.io/amazon-chime-sdk-js/modules/meetingevents.html)
 - [Integrating Amazon Voice Focus into your application](https://aws.github.io/amazon-chime-sdk-js/modules/amazonvoice_focus.html)
 - [Adding frame-by-frame processing to an outgoing video stream](https://aws.github.io/amazon-chime-sdk-js/modules/videoprocessor.html)
+- [Adding background filtering to an outgoing video stream](https://aws.github.io/amazon-chime-sdk-js/modules/backgroundfilter_video_processor.html)
 - [Adapting video to limited bandwidth using a priority-based video downlink policy](https://aws.github.io/amazon-chime-sdk-js/modules/prioritybased_downlink_policy.html)
 - [Client event ingestion](https://aws.github.io/amazon-chime-sdk-js/modules/clientevent_ingestion.html)
 - [Content Security Policy](https://aws.github.io/amazon-chime-sdk-js/modules/contentsecurity_policy.html)
@@ -1144,10 +1145,9 @@ messagingSession.start();
 
 ## Notice
 
-The use of Amazon Voice Focus via this SDK involves the downloading and execution of code at runtime by end users.
-
-The use of Amazon Voice Focus runtime code is subject to additional notices. See [this NOTICES file](https://static.sdkassets.chime.aws/workers/NOTICES.txt) for details. You agree to make these additional notices available to all end users who use Amazon Voice Focus runtime code via this SDK.
+The use of Amazon Voice Focus and background blur via this SDK involves the downloading and execution of code at runtime by end users.
 
+The use of Amazon Voice Focus and background blur runtime code is subject to additional notices. See [this Amazon Voice Focus NOTICES file](https://static.sdkassets.chime.aws/workers/NOTICES.txt) and [background blur NOTICES file](https://static.sdkassets.chime.aws/bgblur/workers/NOTICES.txt) for details. You agree to make these additional notices available to all end users who use Amazon Voice Focus and background blur runtime code via this SDK.
 
 The browser demo applications in the [demos directory](https://github.com/aws/amazon-chime-sdk-js/tree/master/demos) use [TensorFlow.js](https://github.com/tensorflow/tfjs) and pre-trained [TensorFlow.js models](https://github.com/tensorflow/tfjs-models) for image segmentation. Use of these third party models involves downloading and execution of code at runtime from [jsDelivr](https://www.jsdelivr.com/) by end user browsers. For the jsDelivr Acceptable Use Policy, please visit this [link](https://www.jsdelivr.com/terms/acceptable-use-policy-jsdelivr-net).
 

demos/browser/app/meetingV2/meetingV2.ts

@@ -11,6 +11,9 @@ import {
   AudioProfile,
   AudioVideoFacade,
   AudioVideoObserver,
+  BackgroundBlurProcessor,
+  BackgroundBlurVideoFrameProcessor,
+  BackgroundBlurVideoFrameProcessorObserver,
   ClientMetricReport,
   ClientVideoStreamReceivingReport,
   ConsoleLogger,
@@ -70,6 +73,9 @@ import {
   VoiceFocusTransformDevice,
   isAudioTransformDevice,
   isDestroyable,
+  BackgroundFilterSpec,
+  BackgroundFilterPaths,
+  ModelSpecBuilder,
 } from 'amazon-chime-sdk-js';
 
 import CircularCut from './videofilter/CircularCut';
@@ -292,7 +298,25 @@ const VOICE_FOCUS_SPEC = {
 
 const MAX_VOICE_FOCUS_COMPLEXITY: VoiceFocusModelComplexity | undefined = undefined;
 
-type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'Segmentation' | 'Resize (9/16)' | 'None';
+const BACKGROUND_BLUR_CDN = search.get('blurCDN') || undefined;
+const BACKGROUND_BLUR_ASSET_GROUP = search.get('blurAssetGroup') || undefined;
+const BACKGROUND_BLUR_REVISION_ID = search.get('blurRevisionID') || undefined;
+
+const BACKGROUND_BLUR_PATHS: BackgroundFilterPaths = BACKGROUND_BLUR_CDN && {
+  worker: `${BACKGROUND_BLUR_CDN}/bgblur/workers/worker.js`,
+  wasm: `${BACKGROUND_BLUR_CDN}/bgblur/wasm/_cwt-wasm.wasm`,
+  simd: `${BACKGROUND_BLUR_CDN}/bgblur/wasm/_cwt-wasm-simd.wasm`,
+};
+const BACKGROUND_BLUR_MODEL = BACKGROUND_BLUR_CDN && ModelSpecBuilder.builder()
+    .withSelfieSegmentationDefaults()
+    .withPath(`${BACKGROUND_BLUR_CDN}/bgblur/models/selfie_segmentation_landscape.tflite`)
+    .build();
+const BACKGROUND_BLUR_ASSET_SPEC = (BACKGROUND_BLUR_ASSET_GROUP || BACKGROUND_BLUR_REVISION_ID) && {
+  assetGroup: BACKGROUND_BLUR_ASSET_GROUP,
+  revisionID: BACKGROUND_BLUR_REVISION_ID,
+}
+
+type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'Segmentation' | 'Resize (9/16)' | 'BackgroundBlur' | 'None';
 
 const VIDEO_FILTERS: VideoFilterName[] = ['Emojify', 'CircularCut', 'NoOp', 'Resize (9/16)'];
 
@@ -426,6 +450,7 @@ export class DemoMeetingApp
 
   attendeeIdPresenceHandler: (undefined | ((attendeeId: string, present: boolean, externalUserId: string, dropped: boolean) => void)) = undefined;
   activeSpeakerHandler: (undefined | ((attendeeIds: string[]) => void)) = undefined;
+  blurObserver: (undefined | BackgroundBlurVideoFrameProcessorObserver ) = undefined;
 
   showActiveSpeakerScores = false;
   activeSpeakerLayout = true;
@@ -485,6 +510,8 @@ export class DemoMeetingApp
   enableVoiceFocus = false;
   voiceFocusIsActive = false;
 
+  supportsBackgroundBlur = false;
+
   enableLiveTranscription = false;
   noWordSeparatorForTranscription = false;
 
@@ -499,6 +526,8 @@ export class DemoMeetingApp
   voiceFocusTransformer: VoiceFocusDeviceTransformer | undefined;
   voiceFocusDevice: VoiceFocusTransformDevice | undefined;
 
+  bbprocessor: BackgroundBlurProcessor | undefined;
+
   // This is an extremely minimal reactive programming approach: these elements
   // will be updated when the Amazon Voice Focus display state changes.
   voiceFocusDisplayables: HTMLElement[] = [];
@@ -670,6 +699,17 @@ export class DemoMeetingApp
     document.getElementById('voice-focus-setting').classList.toggle('hidden', true);
   }
 
+  async initBackgroundBlur(): Promise<void> {
+      const logger = new ConsoleLogger('SDK', LogLevel.DEBUG);
+      try {
+        this.supportsBackgroundBlur = await BackgroundBlurVideoFrameProcessor.isSupported(this.getBackgroundBlurSpec());
+      }
+      catch (e) {
+      logger.warn(`[DEMO] Does not support background blur: ${e.message}`);
+        this.supportsBackgroundBlur = false;
+      }
+  }
+
   private async onVoiceFocusSettingChanged(): Promise<void> {
     this.log('[DEMO] Amazon Voice Focus setting toggled to', this.enableVoiceFocus);
     this.openAudioInputFromSelectionAndPreview();
@@ -788,6 +828,7 @@ export class DemoMeetingApp
           (document.getElementById('info-name') as HTMLSpanElement).innerText = this.name;
 
           await this.initVoiceFocus();
+          await this.initBackgroundBlur();
           await this.populateAllDeviceLists();
           await this.populateVideoFilterInputList(false);
           await this.populateVideoFilterInputList(true);
@@ -2416,6 +2457,14 @@ export class DemoMeetingApp
     this.chosenVideoTransformDevice?.stop();
   }
 
+  private getBackgroundBlurSpec(): BackgroundFilterSpec {
+    return {
+      paths: BACKGROUND_BLUR_PATHS,
+      model: BACKGROUND_BLUR_MODEL,
+      ...BACKGROUND_BLUR_ASSET_SPEC
+    };
+  }
+
   private async populateVideoFilterInputList(isPreviewWindow: boolean): Promise<void> {
     const genericName = 'Filter';
     let filters: VideoFilterName[] = ['None'];
@@ -2434,6 +2483,10 @@ export class DemoMeetingApp
           this.log('Could not load BodyPix dependency', err);
         });
       }
+
+      if (this.supportsBackgroundBlur) {
+        filters.push('BackgroundBlur');
+      }
     }
 
     this.populateFilterList(isPreviewWindow, genericName, filters);
@@ -2939,7 +2992,7 @@ export class DemoMeetingApp
     return value;
   }
 
-  private videoFilterToProcessor(videoFilter: VideoFilterName): VideoFrameProcessor | null {
+  private async videoFilterToProcessor(videoFilter: VideoFilterName): Promise<VideoFrameProcessor | null> {
     this.log(`Choosing video filter ${videoFilter}`);
 
     if (videoFilter === 'Emojify') {
@@ -2962,6 +3015,21 @@ export class DemoMeetingApp
       return new ResizeProcessor(0.5625);  // 16/9 Aspect Ratio
     }
 
+    if (videoFilter === 'BackgroundBlur') {
+      console.log("background blur - create called from videoFilterToProcessor!")
+
+      // In the event that frames start being dropped we should take some action to remove the background blur.
+      this.blurObserver = {
+        filterFrameDurationHigh: (event) => {
+          this.log(`background filter duration high: framed dropped - ${event.framesDropped}, avg - ${event.avgFilterDurationMillis} ms, frame rate - ${event.framerate}, period - ${event.periodMillis} ms`);
+        }
+      };
+
+      this.bbprocessor = await BackgroundBlurVideoFrameProcessor.create(this.getBackgroundBlurSpec());
+      this.bbprocessor.addObserver(this.blurObserver);
+      return this.bbprocessor;
+    }
+
     return null;
   }
 
@@ -2990,7 +3058,7 @@ export class DemoMeetingApp
       await this.chosenVideoTransformDevice.stop();
     }
 
-    const proc = this.videoFilterToProcessor(this.selectedVideoFilterItem);
+    const proc = await this.videoFilterToProcessor(this.selectedVideoFilterItem);
     this.chosenVideoFilter = this.selectedVideoFilterItem;
     this.chosenVideoTransformDevice = new DefaultVideoTransformDevice(
       this.meetingLogger,
@@ -3203,6 +3271,9 @@ export class DemoMeetingApp
       await this.audioVideo.chooseAudioOutputDevice(null);
       this.audioVideo.unbindAudioElement();
 
+      // remove blur event observer
+      this.bbprocessor?.removeObserver(this.blurObserver);
+
       // Stop any video processor.
       await this.chosenVideoTransformDevice?.stop();
 
@@ -3228,6 +3299,7 @@ export class DemoMeetingApp
       this.activeSpeakerHandler = undefined;
       this.currentAudioInputDevice = undefined;
       this.eventReporter = undefined;
+      this.bbprocessor = undefined;
     };
 
     const onLeftMeeting = async () => {