Skip to content

Commit ba3211a

Browse files
ahmedsabielina128
ahmedsabie
and
lina128
authored
Add SelfieSegmentation MediaPipe implementation (#875)
* Add SelfieSegmentation MediaPipe implementation * Rename to MediaPipeSelfieSegmentation * Fix documentation Co-authored-by: Na Li <[email protected]>
1 parent 5082426 commit ba3211a

23 files changed

+1100
-11
lines changed

‎body-segmentation/.npmignore

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.DS_Store
2+
.yalc/
3+
.vscode/
4+
.rpt2_cache/
5+
demos/
6+
scripts/
7+
src/
8+
test_data/
9+
coverage/
10+
node_modules/
11+
karma.conf.js
12+
*.tgz
13+
.travis.yml
14+
.npmignore
15+
tslint.json
16+
yarn.lock
17+
yalc.lock
18+
cloudbuild.yml
19+
dist/*_test.js
20+
dist/*_test.js.map
21+
dist/test_util*

‎body-segmentation/README.md

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Body Segmentation
2+
3+
This package provides models for running real-time body segmentation.
4+
5+
Currently, we provide 1 model option:
6+
7+
#### MediaPipe SelfieSegmentation:
8+
9+
MediaPipe SelfieSegmentation segments the prominent humans in the scene. It can run in real-time on both smartphones and laptops. The intended use cases include selfie effects and video conferencing, where the person is close (< 2m) to the camera..
10+
11+
-------------------------------------------------------------------------------
12+
## Table of Contents
13+
1. [How to Run It](#how-to-run-it)
14+
2. [Example Code and Demos](#example-code-and-demos)
15+
16+
-------------------------------------------------------------------------------
17+
## How to Run It
18+
In general there are two steps:
19+
20+
You first create a detector by choosing one of the models from `SupportedModels`,
21+
including `MediaPipeSelfieSegmentation`.
22+
23+
For example:
24+
25+
```javascript
26+
const model = bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation;
27+
const segmenter = await bodySegmentation.createDetector(model);
28+
```
29+
30+
Then you can use the segmenter to segment people in the image.
31+
32+
```
33+
const people = await segmenter.segmentPeople(image);
34+
```
35+
36+
The returned segmentation list contains the detected people in the image.
37+
Note that it is not necessarily the case that there will be one segmentation per
38+
one person. Each model will have its own semantics for the segmentation output.
39+
40+
MediaPipe SelfieSegmentation returns exactly one segmentation corresponding to all people in the input image.
41+
42+
Example output:
43+
```
44+
[
45+
{
46+
maskValueToLabel: (maskValue: number) => { return 'person' },
47+
mask: {
48+
toCanvasImageSource(): ...
49+
toImageData(): ...
50+
toTensor(): ...
51+
getUnderlyingType(): ...
52+
}
53+
}
54+
]
55+
```
56+
57+
The `mask` key stores an object which provides access to the underlying mask image using the conversion functions toCanvasImageSource, toImageData, and toTensor depending on the desired output type. Note that getUnderlyingType can be queried to determine what is the type being used underneath the hood to avoid expensive conversions (such as from tensor to image data).
58+
59+
The semantics of the RGBA values of the `mask` is as follows: the image mask is the same size as the input image, where green and blue channels are always set to 0. Different red values denote different body parts (see maskValueToLabel key below). Different alpha values denote the probability of pixel being a body part pixel (0 being lowest probability and 255 being highest).
60+
61+
`maskValueToLabel` maps a foreground pixel’s red value to the segmented part name of that pixel. Should throw error for unsupported input values. This is not necessarily the same across different models (for example MediaPipeSelfieSegmentation will always return 'person' since it does not distinguish individual body parts).
62+
63+
Refer to each model's documentation for specific configurations for the model
64+
and their performance.
65+
66+
[MediaPipeSelfieSegmentation MediaPipe Documentation](https://github.com/tensorflow/tfjs-models/tree/master/body-segmentation/src/selfie_segmentation_mediapipe)
67+
68+
-------------------------------------------------------------------------------
69+
70+
## Example Code and Demos
71+
You may reference the demos for code examples. Details for how to run the demos
72+
are included in the `demos/`
73+
[folder](https://github.com/tensorflow/tfjs-models/tree/master/body-segmentation/demos).

‎body-segmentation/cloudbuild.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install body-segmentation dependencies.
10+
- name: 'node:10'
11+
dir: 'body-segmentation'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Lint.
18+
- name: 'node:10'
19+
dir: 'body-segmentation'
20+
entrypoint: 'yarn'
21+
id: 'lint'
22+
args: ['lint']
23+
waitFor: ['yarn']
24+
25+
# Build.
26+
- name: 'node:10'
27+
dir: 'body-segmentation'
28+
entrypoint: 'yarn'
29+
id: 'build'
30+
args: ['build']
31+
waitFor: ['yarn']
32+
33+
# Run browser tests.
34+
- name: 'node:10'
35+
dir: 'body-segmentation'
36+
entrypoint: 'yarn'
37+
id: 'test'
38+
args: ['test-ci']
39+
env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
40+
secretEnv: ['BROWSERSTACK_KEY']
41+
waitFor: ['build']
42+
43+
# General configuration
44+
secrets:
45+
- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
46+
secretEnv:
47+
BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
48+
timeout: 1800s
49+
logsBucket: 'gs://tfjs-build-logs'
50+
substitutions:
51+
_NIGHTLY: ''
52+
options:
53+
logStreamingOption: 'STREAM_ON'
54+
substitution_option: 'ALLOW_LOOSE'

‎body-segmentation/karma.conf.js

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
* =============================================================================
16+
*/
17+
18+
const karmaTypescriptConfig = {
19+
tsconfig: 'tsconfig.test.json',
20+
// Disable coverage reports and instrumentation by default for tests
21+
coverageOptions: {instrumentation: false},
22+
reports: {},
23+
bundlerOptions: {
24+
sourceMap: true,
25+
// Process any non es5 code through karma-typescript-es6-transform (babel)
26+
acornOptions: {ecmaVersion: 8},
27+
transforms: [
28+
require('karma-typescript-es6-transform')({
29+
presets: [
30+
// ensure we get es5 by adding IE 11 as a target
31+
['@babel/env', {'targets': {'ie': '11'}, 'loose': true}]
32+
]
33+
}),
34+
]
35+
}
36+
};
37+
38+
const devConfig = {
39+
frameworks: ['jasmine', 'karma-typescript'],
40+
files: [
41+
{pattern: './node_modules/@babel/polyfill/dist/polyfill.js'},
42+
{
43+
// Serve test data as static resources.
44+
pattern: 'test_data/**',
45+
watched: true,
46+
included: false,
47+
served: true,
48+
nocache: true
49+
},
50+
{
51+
// Serve MediaPipe wasm/binarypb/data as static resources.
52+
pattern: './node_modules/@mediapipe/selfie_segmentation/**',
53+
watched: true,
54+
included: false,
55+
served: true,
56+
nocache: true
57+
},
58+
'src/setup_test.ts',
59+
{pattern: 'src/**/*.ts'},
60+
],
61+
preprocessors: {'**/*.ts': ['karma-typescript']},
62+
karmaTypescriptConfig,
63+
reporters: ['dots', 'karma-typescript']
64+
};
65+
66+
const browserstackConfig = {
67+
...devConfig,
68+
hostname: 'bs-local.com',
69+
port: 9886
70+
};
71+
72+
module.exports = function(config) {
73+
const args = [];
74+
if (config.testEnv) {
75+
args.push('--testEnv', config.testEnv);
76+
}
77+
if (config.grep) {
78+
args.push('--grep', config.grep);
79+
}
80+
if (config.flags) {
81+
args.push('--flags', config.flags);
82+
}
83+
84+
let extraConfig = null;
85+
86+
if (config.browserstack) {
87+
extraConfig = browserstackConfig;
88+
} else {
89+
extraConfig = devConfig;
90+
}
91+
92+
let exclude = [];
93+
if (config.excludeTest != null) {
94+
exclude.push(config.excludeTest);
95+
}
96+
97+
config.set({
98+
...extraConfig,
99+
singleRun: true,
100+
exclude,
101+
browsers: ['Chrome'],
102+
browserStack: {
103+
username: process.env.BROWSERSTACK_USERNAME,
104+
accessKey: process.env.BROWSERSTACK_KEY,
105+
timeout: 1800,
106+
tunnelIdentifier:
107+
`body_segmentation_${Date.now()}_${Math.floor(Math.random() * 1000)}`
108+
},
109+
captureTimeout: 3e5,
110+
reportSlowerThan: 500,
111+
browserNoActivityTimeout: 3e5,
112+
browserDisconnectTimeout: 3e5,
113+
browserDisconnectTolerance: 3,
114+
browserSocketTimeout: 1.2e5,
115+
customLaunchers: {
116+
// For browserstack configs see:
117+
// https://www.browserstack.com/automate/node
118+
bs_chrome_mac: {
119+
base: 'BrowserStack',
120+
browser: 'chrome',
121+
browser_version: 'latest',
122+
os: 'OS X',
123+
os_version: 'High Sierra'
124+
},
125+
bs_firefox_mac: {
126+
base: 'BrowserStack',
127+
browser: 'firefox',
128+
browser_version: 'latest',
129+
os: 'OS X',
130+
os_version: 'High Sierra'
131+
},
132+
bs_safari_mac: {
133+
base: 'BrowserStack',
134+
browser: 'safari',
135+
browser_version: 'latest',
136+
os: 'OS X',
137+
os_version: 'High Sierra'
138+
},
139+
bs_ios_11: {
140+
base: 'BrowserStack',
141+
device: 'iPhone X',
142+
os: 'iOS',
143+
os_version: '11.0',
144+
real_mobile: true
145+
},
146+
bs_android_9: {
147+
base: 'BrowserStack',
148+
device: 'Google Pixel 3 XL',
149+
os: 'android',
150+
os_version: '9.0',
151+
real_mobile: true
152+
},
153+
win_10_chrome: {
154+
base: 'BrowserStack',
155+
browser: 'chrome',
156+
// Latest Chrome on Windows has WebGL problems:
157+
// https://github.com/tensorflow/tfjs/issues/2272
158+
browser_version: '77.0',
159+
os: 'Windows',
160+
os_version: '10'
161+
},
162+
},
163+
client: {jasmine: {random: false}, args: args}
164+
})
165+
}

‎body-segmentation/package.json

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
{
2+
"name": "@tensorflow-models/body-segmentation",
3+
"version": "0.0.1",
4+
"description": "Pretrained body segmentation model",
5+
"main": "dist/index.js",
6+
"jsnext:main": "dist/body-segmentation.esm.js",
7+
"module": "dist/body-segmentation.esm.js",
8+
"unpkg": "dist/body-segmentation.min.js",
9+
"jsdelivr": "dist/body-segmentation.min.js",
10+
"types": "dist/index.d.ts",
11+
"repository": {
12+
"type": "git",
13+
"url": "https://github.com/tensorflow/tfjs-models.git"
14+
},
15+
"peerDependencies": {
16+
"@mediapipe/selfie_segmentation": "~0.1.0",
17+
"@tensorflow/tfjs-backend-webgl": "^3.11.0",
18+
"@tensorflow/tfjs-converter": "^3.11.0",
19+
"@tensorflow/tfjs-core": "^3.11.0"
20+
},
21+
"devDependencies": {
22+
"@babel/polyfill": "^7.10.4",
23+
"@mediapipe/selfie_segmentation": "^0.1.0",
24+
"@rollup/plugin-commonjs": "^11.0.2",
25+
"@rollup/plugin-node-resolve": "^7.1.1",
26+
"@rollup/plugin-typescript": "^3.0.0",
27+
"@tensorflow/tfjs-backend-cpu": "^3.11.0",
28+
"@tensorflow/tfjs-backend-webgl": "^3.11.0",
29+
"@tensorflow/tfjs-converter": "^3.11.0",
30+
"@tensorflow/tfjs-core": "^3.11.0",
31+
"@types/jasmine": "~3.4.0",
32+
"@types/node": "~10.3.4",
33+
"babel-core": "~6.26.0",
34+
"babel-plugin-transform-runtime": "~6.23.0",
35+
"jasmine": "~3.1.0",
36+
"jasmine-core": "~3.1.0",
37+
"karma": "~6.3.1",
38+
"karma-browserstack-launcher": "~1.6.0",
39+
"karma-chrome-launcher": "~2.2.0",
40+
"karma-jasmine": "~1.1.0",
41+
"karma-spec-reporter": "~0.0.32",
42+
"karma-typescript": "~5.5.1",
43+
"karma-typescript-es6-transform": "^5.1.0",
44+
"rollup": "~2.3.2",
45+
"rollup-plugin-terser": "~7.0.2",
46+
"rollup-plugin-visualizer": "~3.3.2",
47+
"ts-node": "~8.8.2",
48+
"tslint": "~6.1.3",
49+
"tslint-no-circular-imports": "~0.7.0",
50+
"typescript": "~3.5.3",
51+
"yalc": "~1.0.0-pre.21"
52+
},
53+
"scripts": {
54+
"bundle": "rollup -c",
55+
"build": "rimraf dist && tsc && yarn bundle",
56+
"publish-local": "yarn build && rollup -c && yalc publish",
57+
"run-browserstack": "karma start --browserstack",
58+
"test-node": "NODE_PRESERVE_SYMLINKS=1 ts-node --skip-ignore --project tsconfig.test.json run_tests.ts",
59+
"test": "yarn && karma start",
60+
"test-ci": "yarn run-browserstack --browsers=bs_chrome_mac",
61+
"build-npm": "yarn build && yarn bundle",
62+
"lint": "tslint -p . -t verbose"
63+
},
64+
"license": "Apache-2.0",
65+
"dependencies": {
66+
"rimraf": "^3.0.2"
67+
}
68+
}

0 commit comments

Comments
 (0)