Skip to content

Commit 4e8fa79

Browse files
haoyunfeixpyu10055
haoyunfeix
and
pyu10055
authored
[face-detection] Add WebGPU backend (#1046)
* [face-detection] Add WebGPU backend 1. Add tfjs-webgpu backend and user warnings 2. Update tfjs to 4.0.0 3. Fix backend changing issue in upload_video demo 4. Add URL backend parameter * update message Co-authored-by: Ping Yu <[email protected]>
1 parent 654e441 commit 4e8fa79

File tree

11 files changed

+332
-211
lines changed

11 files changed

+332
-211
lines changed

‎face-detection/demos/live_video/package.json

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@
1111
"dependencies": {
1212
"@mediapipe/face_detection": "~0.4.0",
1313
"@tensorflow-models/face-detection": "file:../../dist",
14-
"@tensorflow/tfjs-backend-wasm": "^3.12.0",
15-
"@tensorflow/tfjs-backend-webgl": "^3.12.0",
16-
"@tensorflow/tfjs-converter": "^3.12.0",
17-
"@tensorflow/tfjs-core": "^3.12.0",
14+
"@tensorflow/tfjs-backend-wasm": "^4.0.0",
15+
"@tensorflow/tfjs-backend-webgl": "^4.0.0",
16+
"@tensorflow/tfjs-backend-webgpu": "^0.0.1-alpha.14",
17+
"@tensorflow/tfjs-converter": "^4.0.0",
18+
"@tensorflow/tfjs-core": "^4.0.0",
1819
"scatter-gl": "0.0.8"
1920
},
2021
"scripts": {

‎face-detection/demos/live_video/src/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717

1818
import '@tensorflow/tfjs-backend-webgl';
19+
import '@tensorflow/tfjs-backend-webgpu';
1920

2021
import * as tfjsWasm from '@tensorflow/tfjs-backend-wasm';
2122

‎face-detection/demos/live_video/yarn.lock

Lines changed: 63 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,51 +1025,67 @@
10251025
"@tensorflow-models/face-detection@file:../../dist":
10261026
version "0.0.0"
10271027

1028-
"@tensorflow/tfjs-backend-cpu@3.14.0":
1029-
version "3.14.0"
1030-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.14.0.tgz#f4db5cd17609e274709f86a0c34233d35cb886b2"
1031-
integrity sha512-Sk0B8p1QUqxEVsOmBNxxX2BUgeR8mfXVc6JZM5lWKP79bYy8YGzuiitrSrcxAhEFAANgmDVvM9FTTVR25a0CWg==
1028+
"@tensorflow/tfjs-backend-cpu@3.21.0":
1029+
version "3.21.0"
1030+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz#ee0274bf14627d08b81c4e229230da66dc9c3d92"
1031+
integrity sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==
10321032
dependencies:
1033-
"@types/seedrandom" "2.4.27"
1034-
seedrandom "2.4.3"
1033+
"@types/seedrandom" "^2.4.28"
1034+
seedrandom "^3.0.5"
10351035

1036-
"@tensorflow/tfjs-backend-wasm@^3.12.0":
1037-
version "3.14.0"
1038-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-3.14.0.tgz#56ba7987a89029ff33ea5c677483fb57a234a4c6"
1039-
integrity sha512-7dCG5eJRyK3CpXXbisbMcFyoi//fx62EQ+bvIra4/9YaTzkCoREDw9TAyyyTG4/wCxtOlU4+lmKD9675AXf97Q==
1036+
"@tensorflow/[email protected]":
1037+
version "4.0.0"
1038+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.0.0.tgz#ed957eedca558d5b0ce9caa129f180b88039501b"
1039+
integrity sha512-Y9ok6VBMir1MVbkcK+h34hF4ZzHZKOrdjAOeE5eFbSuegJHY/AXWcZEU589VTnEE4AU+SUdjjfp6yvAtU17GAA==
1040+
dependencies:
1041+
"@types/seedrandom" "^2.4.28"
1042+
seedrandom "^3.0.5"
1043+
1044+
"@tensorflow/tfjs-backend-wasm@^4.0.0":
1045+
version "4.0.0"
1046+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-4.0.0.tgz#195c976335487767fe95763d5790d3af7d28d15c"
1047+
integrity sha512-Z5gqdmqywauB2ZrWFduvFc82aMMTBFN7EtWEhGAGRXVcVH2L4nIXx0bs/5aUBA0Qs9B7hiy83b+NquWrDdBeGQ==
10401048
dependencies:
1041-
"@tensorflow/tfjs-backend-cpu" "3.14.0"
1049+
"@tensorflow/tfjs-backend-cpu" "4.0.0"
10421050
"@types/emscripten" "~0.0.34"
10431051

1044-
"@tensorflow/tfjs-backend-webgl@^3.12.0":
1045-
version "3.14.0"
1046-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.14.0.tgz#25c33c1059d4f05d1f8977ccc5d11df6e0f5ba89"
1047-
integrity sha512-P3qB9LmC69+9ut9cC76mGUS4tLIFk95qmWUaUP1Zk4R3iLkiRnLE4xmBxGnm3rNckoWS77Ujpel58i4QK1BmCw==
1052+
"@tensorflow/tfjs-backend-webgl@^4.0.0":
1053+
version "4.0.0"
1054+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.0.0.tgz#df21ff8786e17035744d6425286a506c57068004"
1055+
integrity sha512-JfTtmdwlnJbV4CRxXLJr5MM0Yzj6WfMXAwM/bjTuDKQsyYdNzq75E72A4B0c+XwSIZb7MnVyUmRkZYeGTCqwtg==
10481056
dependencies:
1049-
"@tensorflow/tfjs-backend-cpu" "3.14.0"
1057+
"@tensorflow/tfjs-backend-cpu" "4.0.0"
10501058
"@types/offscreencanvas" "~2019.3.0"
1051-
"@types/seedrandom" "2.4.27"
1059+
"@types/seedrandom" "^2.4.28"
10521060
"@types/webgl-ext" "0.0.30"
10531061
"@types/webgl2" "0.0.6"
1054-
seedrandom "2.4.3"
1062+
seedrandom "^3.0.5"
10551063

1056-
"@tensorflow/tfjs-converter@^3.12.0":
1057-
version "3.14.0"
1058-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-3.14.0.tgz#e127fb37bfc68915c79ed2edf14a272c5fd07320"
1059-
integrity sha512-cz8dpfOU5kOeY8SyNdmg5Pv836fYmZGH/6j3VaslALkqK63TOgVGpWUzi7f10KEZCZUk29TTVeu3u9zqobSBpA==
1064+
"@tensorflow/tfjs-backend-webgpu@^0.0.1-alpha.14":
1065+
version "0.0.1-alpha.14"
1066+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgpu/-/tfjs-backend-webgpu-0.0.1-alpha.14.tgz#03595cc4c121b20b52b220fb16c2f50737de3c16"
1067+
integrity sha512-4TLkuVY9i/o3LBBQomEkxWOgdUG2++BrPNDX6Nwjk7p4AuJbgdoALGn5Ab4uotSukIe88binMMxffMMfMIVtGw==
1068+
dependencies:
1069+
"@tensorflow/tfjs-backend-cpu" "3.21.0"
10601070

1061-
"@tensorflow/tfjs-core@^3.12.0":
1062-
version "3.14.0"
1063-
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.14.0.tgz#2d73e76e2ae740c2f91c2cbf15312b9e61ec1c72"
1064-
integrity sha512-bS/iuI9BpDVZuqEPfLzIFLoHYd+ihNIiux+EXveuFO8phPx7FkgPqakatHYLzrdYlOfAXurIxIlGNAzVKNQOUQ==
1071+
"@tensorflow/tfjs-converter@^4.0.0":
1072+
version "4.0.0"
1073+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.0.0.tgz#c3182f8e7b1a246e294951ce6df2e36ebb324a2d"
1074+
integrity sha512-VU53ZB9blQyF1geB0xbloedYI6d52yQ0U56uPSS9AcjexrAShnN7VIjfNtCFYeLDmVe3M/frkAuIZmWAv66iMw==
1075+
1076+
"@tensorflow/tfjs-core@^4.0.0":
1077+
version "4.0.0"
1078+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-4.0.0.tgz#80a8a45afb74c81ecb906100008125a3702b9140"
1079+
integrity sha512-RrTgHPT8Xo6vsBvaRkee1YOoH3lv0lEtS3ISGRcHsUy8mkT8ZIMNou4zaJuIBQ7bVPXOeOFFeTbjDvWVtYypxQ==
10651080
dependencies:
10661081
"@types/long" "^4.0.1"
1067-
"@types/offscreencanvas" "~2019.3.0"
1068-
"@types/seedrandom" "2.4.27"
1082+
"@types/offscreencanvas" "~2019.7.0"
1083+
"@types/seedrandom" "^2.4.28"
10691084
"@types/webgl-ext" "0.0.30"
1085+
"@webgpu/types" "0.1.21"
10701086
long "4.0.0"
10711087
node-fetch "~2.6.1"
1072-
seedrandom "2.4.3"
1088+
seedrandom "^3.0.5"
10731089

10741090
"@types/emscripten@~0.0.34":
10751091
version "0.0.34"
@@ -1086,15 +1102,20 @@
10861102
resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553"
10871103
integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==
10881104

1105+
"@types/offscreencanvas@~2019.7.0":
1106+
version "2019.7.0"
1107+
resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.0.tgz#e4a932069db47bb3eabeb0b305502d01586fa90d"
1108+
integrity sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg==
1109+
10891110
"@types/q@^1.5.1":
10901111
version "1.5.5"
10911112
resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.5.tgz#75a2a8e7d8ab4b230414505d92335d1dcb53a6df"
10921113
integrity sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ==
10931114

1094-
"@types/[email protected].27":
1095-
version "2.4.27"
1096-
resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41"
1097-
integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=
1115+
"@types/seedrandom@^2.4.28":
1116+
version "2.4.30"
1117+
resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.30.tgz#d2efe425869b84163c2d56e779dddadb9372cbfa"
1118+
integrity sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==
10981119

10991120
"@types/[email protected]":
11001121
version "0.0.30"
@@ -1106,6 +1127,11 @@
11061127
resolved "https://registry.yarnpkg.com/@types/webgl2/-/webgl2-0.0.6.tgz#1ea2db791362bd8521548d664dbd3c5311cdf4b6"
11071128
integrity sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==
11081129

1130+
"@webgpu/[email protected]":
1131+
version "0.1.21"
1132+
resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.21.tgz#b181202daec30d66ccd67264de23814cfd176d3a"
1133+
integrity sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow==
1134+
11091135
abab@^2.0.0:
11101136
version "2.0.5"
11111137
resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a"
@@ -5773,10 +5799,10 @@ [email protected]:
57735799
dependencies:
57745800
three "^0.113"
57755801

5776-
seedrandom@2.4.3:
5777-
version "2.4.3"
5778-
resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc"
5779-
integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=
5802+
seedrandom@^3.0.5:
5803+
version "3.0.5"
5804+
resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
5805+
integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
57805806

57815807
[email protected]:
57825808
version "7.0.0"

‎face-detection/demos/shared/option_panel.js

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,14 @@ import * as params from './params';
2727
let TUNABLE_FLAG_DEFAULT_VALUE_MAP;
2828

2929
const stringValueMap = {};
30+
let backendFolder;
3031

3132
export async function setupModelFolder(gui, urlParams) {
3233
// The model folder contains options for model selection.
3334
const modelFolder = gui.addFolder('Model');
3435

3536
const model = urlParams.get('model');
37+
const backendFromURL = urlParams.get('backend');
3638

3739
switch (model) {
3840
case 'mediapipe_face_detector':
@@ -56,7 +58,8 @@ export async function setupModelFolder(gui, urlParams) {
5658

5759
modelFolder.open();
5860

59-
const backendFolder = gui.addFolder('Backend');
61+
backendFolder = gui.addFolder('Backend');
62+
params.STATE.backend = backendFromURL;
6063

6164
showBackendConfigs(backendFolder);
6265

@@ -65,7 +68,10 @@ export async function setupModelFolder(gui, urlParams) {
6568
return gui;
6669
}
6770

68-
async function showBackendConfigs(folderController) {
71+
export async function showBackendConfigs(folderController) {
72+
if (folderController == null) {
73+
folderController = backendFolder;
74+
}
6975
// Clean up backend configs for the previous model.
7076
const fixedSelectionCount = 0;
7177
while (folderController.__controllers.length > fixedSelectionCount) {
@@ -74,8 +80,10 @@ async function showBackendConfigs(folderController) {
7480
.__controllers[folderController.__controllers.length - 1]);
7581
}
7682
const backends = params.MODEL_BACKEND_MAP[params.STATE.model];
77-
// The first element of the array is the default backend for the model.
78-
params.STATE.backend = backends[0];
83+
if(params.STATE.backend == null) {
84+
// The first element of the array is the default backend for the model.
85+
params.STATE.backend = backends[0];
86+
}
7987
const backendController =
8088
folderController.add(params.STATE, 'backend', backends);
8189
backendController.name('runtime-backend');

‎face-detection/demos/shared/params.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,13 @@ export const BACKEND_FLAGS_MAP = {
9494
'WEBGL_FORCE_F16_TEXTURES', 'WEBGL_RENDER_FLOAT32_CAPABLE',
9595
'WEBGL_FLUSH_THRESHOLD'
9696
],
97+
['tfjs-webgpu']: [],
9798
['mediapipe-gpu']: []
9899
};
99100

100101
export const MODEL_BACKEND_MAP = {
101102
[faceDetection.SupportedModels.MediaPipeFaceDetector]:
102-
['mediapipe-gpu', 'tfjs-webgl']
103+
['mediapipe-gpu', 'tfjs-webgl', 'tfjs-webgpu']
103104
}
104105

105106
export const TUNABLE_FLAG_NAME_MAP = {

‎face-detection/demos/shared/util.js

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
* =============================================================================
1616
*/
1717
import * as tf from '@tensorflow/tfjs-core';
18-
19-
import {GREEN, NUM_KEYPOINTS, RED, TUNABLE_FLAG_VALUE_RANGE_MAP} from './params';
18+
import {showBackendConfigs} from './option_panel';
19+
import {GREEN, NUM_KEYPOINTS, RED, STATE, TUNABLE_FLAG_VALUE_RANGE_MAP} from './params';
2020

2121
export function isiOS() {
2222
return /iPhone|iPad|iPod/i.test(navigator.userAgent);
@@ -38,7 +38,14 @@ export function isMobile() {
3838
async function resetBackend(backendName) {
3939
const ENGINE = tf.engine();
4040
if (!(backendName in ENGINE.registryFactory)) {
41-
throw new Error(`${backendName} backend is not registed.`);
41+
if(backendName === 'webgpu') {
42+
alert('webgpu backend is not registered. Your browser may not support WebGPU yet. To test this backend, please use a supported browser, e.g. Chrome canary with --enable-unsafe-webgpu flag');
43+
STATE.backend = !!STATE.lastTFJSBackend ? STATE.lastTFJSBackend : 'tfjs-webgl';
44+
showBackendConfigs();
45+
return;
46+
} else {
47+
throw new Error(`${backendName} backend is not registered.`);
48+
}
4249
}
4350

4451
if (backendName in ENGINE.registry) {
@@ -48,6 +55,7 @@ async function resetBackend(backendName) {
4855
}
4956

5057
await tf.setBackend(backendName);
58+
STATE.lastTFJSBackend = `tfjs-${backendName}`;
5159
}
5260

5361
/**

‎face-detection/demos/upload_video/package.json

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@
1111
"dependencies": {
1212
"@mediapipe/face_detection": "~0.4.0",
1313
"@tensorflow-models/face-detection": "file:../../dist",
14-
"@tensorflow/tfjs-backend-wasm": "^3.12.0",
15-
"@tensorflow/tfjs-backend-webgl": "^3.12.0",
16-
"@tensorflow/tfjs-converter": "^3.12.0",
17-
"@tensorflow/tfjs-core": "^3.12.0"
14+
"@tensorflow/tfjs-backend-wasm": "^4.0.0",
15+
"@tensorflow/tfjs-backend-webgl": "^4.0.0",
16+
"@tensorflow/tfjs-backend-webgpu": "^0.0.1-alpha.14",
17+
"@tensorflow/tfjs-converter": "^4.0.0",
18+
"@tensorflow/tfjs-core": "^4.0.0"
1819
},
1920
"scripts": {
2021
"watch": "cross-env NODE_ENV=development parcel index.html --no-hmr --open",

‎face-detection/demos/upload_video/src/index.js

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717

1818
import '@tensorflow/tfjs-backend-webgl';
19+
import '@tensorflow/tfjs-backend-webgpu';
1920

2021
import * as tfjsWasm from '@tensorflow/tfjs-backend-wasm';
2122

@@ -96,12 +97,6 @@ async function renderResult() {
9697
}
9798
}
9899

99-
async function checkUpdate() {
100-
await checkGuiUpdate();
101-
102-
requestAnimationFrame(checkUpdate);
103-
};
104-
105100
async function updateVideo(event) {
106101
// Clear reference to any previous uploaded video.
107102
URL.revokeObjectURL(camera.video.currentSrc);
@@ -128,6 +123,7 @@ async function updateVideo(event) {
128123
}
129124

130125
async function runFrame() {
126+
await checkGuiUpdate();
131127
if (video.paused) {
132128
// video has finished.
133129
camera.mediaRecorder.stop();
@@ -178,18 +174,16 @@ async function app() {
178174

179175
await setupDatGui(urlParams);
180176
stats = setupStats();
181-
detector = await createDetector();
182177
camera = new Context();
183178

184179
await setBackendAndEnvFlags(STATE.flags, STATE.backend);
180+
detector = await createDetector();
185181

186182
const runButton = document.getElementById('submit');
187183
runButton.onclick = run;
188184

189185
const uploadButton = document.getElementById('videofile');
190186
uploadButton.onchange = updateVideo;
191-
192-
checkUpdate();
193187
};
194188

195189
app();

0 commit comments

Comments
 (0)