Upgrade body segmentation model (#1171)

* Upgrade body segmentation model

* remove comments

* fix lint

* Update to 5.0.4

* fix lint

* update

Author: fengwuyao

body-segmentation/demos/live_video/package.json

@@ -13,10 +13,9 @@
     "@mediapipe/selfie_segmentation": "~0.1.0",
     "@tensorflow-models/body-segmentation": "file:../../dist",
     "@tensorflow-models/pose-detection": "file:../../../pose-detection/dist",
-    "@tensorflow/tfjs-backend-wasm": "^3.13.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.13.0",
-    "@tensorflow/tfjs-converter": "^3.13.0",
-    "@tensorflow/tfjs-core": "^3.13.0",
+    "@tensorflow/tfjs-backend-wasm": "^4.9.0",
+    "@tensorflow/tfjs-backend-webgl": "^4.9.0",
+    "@tensorflow/tfjs-converter": "^4.9.0",
     "scatter-gl": "0.0.8"
   },
   "scripts": {

body-segmentation/demos/live_video/yarn.lock

@@ -996,62 +996,43 @@
 "@tensorflow-models/pose-detection@file:../../../pose-detection/dist":
   version "0.0.0"
 
-"@tensorflow/tfjs-backend-cpu@3.13.0":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.13.0.tgz#2a141256fe0a5de0670c31a9ba6990465708dee4"
-  integrity sha512-POmzUoAP8HooYYTZ72O1ZYkpVZB0f+8PeAkbTxIG0oahcJccj6a0Vovp1A6xWKfljUoPlJb3jWVC++S603ZL8w==
+"@tensorflow/tfjs-backend-cpu@4.9.0":
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.9.0.tgz#76ce8f4e83814293b662404f16d70f65926195a2"
+  integrity sha512-AHIfI3iD1fyQgQKeoQmtkI3exPWRfOo+W0Ws/bxOdapTXcAYWGg0179t52j8XPDwsl8WopfaTINNgYNG6FnP3Q==
   dependencies:
-    "@types/seedrandom" "2.4.27"
-    seedrandom "2.4.3"
+    "@types/seedrandom" "^2.4.28"
+    seedrandom "^3.0.5"
 
-"@tensorflow/tfjs-backend-wasm@^3.13.0":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-3.13.0.tgz#3465c6040542752d2632df7e0c4ac0e5a9ef801a"
-  integrity sha512-h5kNS4xvljoySzfcFwqbdFB6QZGR06IA9/Xq/PjBeZt18XEoJGqKHbOCYupmUlr5pxo/gnXzPhAC2h4SfZXPXw==
+"@tensorflow/tfjs-backend-wasm@^4.9.0":
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-4.9.0.tgz#1e1fec3406b312efc814f8238c5c25052e01650e"
+  integrity sha512-HsRr9qyu/1pdpkmSoF+1QMtpuE61QOVpH55j6HGNcN+DGuF8TSh80imzZa9JC+A7DXyQdHBKCbUqytuisBQDaA==
   dependencies:
-    "@tensorflow/tfjs-backend-cpu" "3.13.0"
+    "@tensorflow/tfjs-backend-cpu" "4.9.0"
     "@types/emscripten" "~0.0.34"
 
-"@tensorflow/tfjs-backend-webgl@^3.13.0":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.13.0.tgz#f99df51253de21e20dae195991d332195ab30b4b"
-  integrity sha512-ZuJS11tCoZx2F1Eq7wqiqu8euJpPW/JV0qOKBehlRpV2qQrR+wHMpBT1hhDl4qU4LdgFTtSggKIRg/L8b0ScUQ==
+"@tensorflow/tfjs-backend-webgl@^4.9.0":
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.9.0.tgz#272812af5c61d02af3b541ed543c3484f5eb8618"
+  integrity sha512-lSEOjR9zi6vb1V9yhrby8jWt6SS+wWBXRa3sDE5GCbpcHMWHv41wZktB2WQyIXDqJQcw1lRZBDoYneibMqr2uQ==
   dependencies:
-    "@tensorflow/tfjs-backend-cpu" "3.13.0"
+    "@tensorflow/tfjs-backend-cpu" "4.9.0"
     "@types/offscreencanvas" "~2019.3.0"
-    "@types/seedrandom" "2.4.27"
+    "@types/seedrandom" "^2.4.28"
     "@types/webgl-ext" "0.0.30"
-    "@types/webgl2" "0.0.6"
-    seedrandom "2.4.3"
+    seedrandom "^3.0.5"
 
-"@tensorflow/tfjs-converter@^3.13.0":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-3.13.0.tgz#3affc86d94c3948b01673a91309a35feb10e5eac"
-  integrity sha512-H2VpDTv9Ve0HBt7ttzz46DmnsPaiT0B+yJjVH3NebGZbgY9C8boBgJIsdyqfiqEWBS3WxF8h4rh58Hv5XXMgaQ==
-
-"@tensorflow/tfjs-core@^3.13.0":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.13.0.tgz#0cfd707c668250969564991c5c101fb52e51e1aa"
-  integrity sha512-18qBEVIB/4u2OUK9nA5P1XT3e3LyarElD1UKNSNDpnMLxhLTUVZaCR71eHJcpl9wP2Q0cciaTJCTpJdPv1tNDQ==
-  dependencies:
-    "@types/long" "^4.0.1"
-    "@types/offscreencanvas" "~2019.3.0"
-    "@types/seedrandom" "2.4.27"
-    "@types/webgl-ext" "0.0.30"
-    long "4.0.0"
-    node-fetch "~2.6.1"
-    seedrandom "2.4.3"
+"@tensorflow/tfjs-converter@^4.9.0":
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.9.0.tgz#7958f563043b3c54f757f47d2b5b08e963f466ad"
+  integrity sha512-mRlzdG3jVsxMkFfHFgDNY10HMoh+vtfPPIghtY+Fc4U/ZnBUFvSfZqwEFyXfOJAewn4fY4BX8+6RE4a0kRXqGA==
 
 "@types/emscripten@~0.0.34":
   version "0.0.34"
   resolved "https://registry.yarnpkg.com/@types/emscripten/-/emscripten-0.0.34.tgz#12b4a344274fb102ff2f6c877b37587bc3e46008"
   integrity sha512-QSb9ojDincskc+uKMI0KXp8e1NALFINCrMlp8VGKGcTSxeEyRTTKyjWw75NYrCZHUsVEEEpr1tYHpbtaC++/sQ==
 
-"@types/long@^4.0.1":
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
-  integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==
-
 "@types/offscreencanvas@~2019.3.0":
   version "2019.3.0"
   resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553"
@@ -1062,21 +1043,16 @@
   resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.5.tgz#75a2a8e7d8ab4b230414505d92335d1dcb53a6df"
   integrity sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ==
 
-"@types/[email protected].27":
-  version "2.4.27"
-  resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41"
-  integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=
+"@types/seedrandom@^2.4.28":
+  version "2.4.30"
+  resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.30.tgz#d2efe425869b84163c2d56e779dddadb9372cbfa"
+  integrity sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==
 
 "@types/[email protected]":
   version "0.0.30"
   resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d"
   integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==
 
-"@types/[email protected]":
-  version "0.0.6"
-  resolved "https://registry.yarnpkg.com/@types/webgl2/-/webgl2-0.0.6.tgz#1ea2db791362bd8521548d664dbd3c5311cdf4b6"
-  integrity sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==
-
 abab@^2.0.0:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a"
@@ -4250,11 +4226,6 @@ log-symbols@^2.2.0:
   dependencies:
     chalk "^2.0.1"
 
-[email protected]:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
-  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
-
 loose-envify@^1.0.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
@@ -4468,13 +4439,6 @@ node-addon-api@^1.7.1:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d"
   integrity sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==
 
-node-fetch@~2.6.1:
-  version "2.6.6"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
-  integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^0.10.0:
   version "0.10.0"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
@@ -5736,10 +5700,10 @@ [email protected]:
   dependencies:
     three "^0.113"
 
-seedrandom@2.4.3:
-  version "2.4.3"
-  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc"
-  integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=
+seedrandom@^3.0.5:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
+  integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
 
 [email protected]:
   version "7.0.0"
@@ -6314,11 +6278,6 @@ tr46@^1.0.1:
   dependencies:
     punycode "^2.1.0"
 
-tr46@~0.0.3:
-  version "0.0.3"
-  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
-  integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
-
 [email protected]:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"
@@ -6565,11 +6524,6 @@ wcwidth@^1.0.1:
   dependencies:
     defaults "^1.0.3"
 
-webidl-conversions@^3.0.0:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
-  integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
-
 webidl-conversions@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
@@ -6587,14 +6541,6 @@ whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0:
   resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf"
   integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==
 
-whatwg-url@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
-  integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
-  dependencies:
-    tr46 "~0.0.3"
-    webidl-conversions "^3.0.0"
-
 whatwg-url@^7.0.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06"

body-segmentation/package.json

@@ -14,20 +14,21 @@
   },
   "peerDependencies": {
     "@mediapipe/selfie_segmentation": "~0.1.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.13.0",
-    "@tensorflow/tfjs-converter": "^3.13.0",
-    "@tensorflow/tfjs-core": "^3.13.0"
+    "@tensorflow/tfjs-backend-webgl": "^4.9.0",
+    "@tensorflow/tfjs-converter": "^4.9.0",
+    "@tensorflow/tfjs-core": "^4.9.0"
   },
   "devDependencies": {
     "@babel/polyfill": "^7.10.4",
     "@mediapipe/selfie_segmentation": "^0.1.0",
     "@rollup/plugin-commonjs": "^11.0.2",
     "@rollup/plugin-node-resolve": "^7.1.1",
     "@rollup/plugin-typescript": "^3.0.0",
-    "@tensorflow/tfjs-backend-cpu": "^3.13.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.13.0",
-    "@tensorflow/tfjs-converter": "^3.13.0",
-    "@tensorflow/tfjs-core": "^3.13.0",
+    "@tensorflow/tfjs-backend-cpu": "^4.9.0",
+    "@tensorflow/tfjs-backend-webgl": "^4.9.0",
+    "@tensorflow/tfjs-converter": "^4.9.0",
+    "@tensorflow/tfjs-backend-webgpu": "4.9.0",
+    "@tensorflow/tfjs-core": "^4.9.0",
     "@types/jasmine": "~3.4.0",
     "babel-core": "~6.26.0",
     "babel-plugin-transform-runtime": "~6.23.0",
@@ -46,7 +47,7 @@
     "ts-node": "~8.8.2",
     "tslint": "~6.1.3",
     "tslint-no-circular-imports": "~0.7.0",
-    "typescript": "~3.9.9",
+    "typescript": "~4.4.0",
     "yalc": "~1.0.0-pre.21"
   },
   "scripts": {

body-segmentation/src/body_pix/body_pix_test.ts

@@ -16,20 +16,20 @@
  */
 
 // tslint:disable-next-line: no-imports-from-dist
-import {BROWSER_ENVS, describeWithFlags} from '@tensorflow/tfjs-core/dist/jasmine_util';
+import { BROWSER_ENVS, describeWithFlags } from '@tensorflow/tfjs-core/dist/jasmine_util';
 
 import * as bodySegmentation from '../index';
-import {Mask} from '../shared/calculators/interfaces/common_interfaces';
-import {toImageDataLossy, toTensorLossy} from '../shared/calculators/mask_util';
+import { Mask } from '../shared/calculators/interfaces/common_interfaces';
+import { toImageDataLossy, toTensorLossy } from '../shared/calculators/mask_util';
 import * as renderUtil from '../shared/calculators/render_util';
-import {loadImage} from '../shared/test_util';
-import {BodyPixSegmentationConfig} from './types';
+import { loadImage } from '../shared/test_util';
+import { BodyPixSegmentationConfig } from './types';
 
 // Measured in channels.
-const DIFF_IMAGE = 30;
+const DIFF_IMAGE_RATIO = 0.001;
 
 class CanvasImageSourceMask implements Mask {
-  constructor(private mask: CanvasImageSource) {}
+  constructor(private mask: CanvasImageSource) { }
 
   async toCanvasImageSource() {
     return this.mask;
@@ -44,28 +44,28 @@ class CanvasImageSourceMask implements Mask {
   }
 
   getUnderlyingType() {
-    return 'canvasimagesource' as const ;
+    return 'canvasimagesource' as const;
   }
 }
 
 async function getSegmentation(
-    image: HTMLImageElement, config: BodyPixSegmentationConfig) {
+  image: HTMLImageElement, config: BodyPixSegmentationConfig) {
   const segmenter = await bodySegmentation.createSegmenter(
-      bodySegmentation.SupportedModels.BodyPix);
+    bodySegmentation.SupportedModels.BodyPix);
 
   const segmentations = await segmenter.segmentPeople(image, config);
   return Promise.all(segmentations.map(async segmentation => {
     return {
       maskValueToLabel: segmentation.maskValueToLabel,
       // Convert to canvas image source to apply alpha-premultiplication.
       mask: new CanvasImageSourceMask(
-          await segmentation.mask.toCanvasImageSource())
+        await segmentation.mask.toCanvasImageSource())
     };
   }));
 }
 
 async function getBinaryMask(
-    image: HTMLImageElement, expectedNumSegmentations?: number) {
+  image: HTMLImageElement, expectedNumSegmentations?: number) {
   const segmentation = await getSegmentation(image, {
     multiSegmentation: expectedNumSegmentations != null,
     segmentBodyParts: false
@@ -76,13 +76,13 @@ async function getBinaryMask(
   }
 
   const binaryMask = await renderUtil.toBinaryMask(
-      segmentation, {r: 255, g: 255, b: 255, a: 255},
-      {r: 0, g: 0, b: 0, a: 255});
+    segmentation, { r: 255, g: 255, b: 255, a: 255 },
+    { r: 0, g: 0, b: 0, a: 255 });
   return binaryMask;
 }
 
 async function getColoredMask(
-    image: HTMLImageElement, expectedNumSegmentations?: number) {
+  image: HTMLImageElement, expectedNumSegmentations?: number) {
   const segmentation = await getSegmentation(image, {
     multiSegmentation: expectedNumSegmentations != null,
     segmentBodyParts: true
@@ -93,8 +93,8 @@ async function getColoredMask(
   }
 
   const coloredMask = await renderUtil.toColoredMask(
-      segmentation, bodySegmentation.bodyPixMaskValueToRainbowColor,
-      {r: 255, g: 255, b: 255, a: 255});
+    segmentation, bodySegmentation.bodyPixMaskValueToRainbowColor,
+    { r: 255, g: 255, b: 255, a: 255 });
 
   return coloredMask;
 }
@@ -104,12 +104,13 @@ const HEIGHT = 861;
 
 async function expectImage(actual: ImageData, imageName: string) {
   const expectedImage = await loadImage(imageName, WIDTH, HEIGHT)
-                            .then(async image => toImageDataLossy(image));
+    .then(async image => toImageDataLossy(image));
   const mismatchedChannels = actual.data.reduce(
-      (mismatched, channel, i) =>
-          mismatched + +(channel !== expectedImage.data[i]),
-      0);
-  expect(mismatchedChannels).toBeLessThanOrEqual(DIFF_IMAGE);
+    (mismatched, channel, i) =>
+      mismatched + +(channel !== expectedImage.data[i]),
+    0);
+  expect(mismatchedChannels)
+    .toBeLessThanOrEqual(expectedImage.data.length * DIFF_IMAGE_RATIO);
 }
 
 describeWithFlags('renderUtil', BROWSER_ENVS, () => {