Create a tfjs-node-gpu directory. (tensorflow#1925)

Each of the following get copied during "prepare" phase, which is on local NPM install (but not when a dependent package installs): https://docs.npmjs.com/misc/scripts

- tfjs-node/src => tfjs-node-gpu/src
- tfjs-node/binding => tfjs-node-gpu/binding
- tfjs-node/scripts => tfjs-node-gpu/scripts
- tfjs-node/binding.gyp => tfjs-node-gpu/binding.gyp

We do this with a copy because symlinks have the incorrect local directory structure.

This PR also fixes a bug where the node backend file caches the backend (this is incorrect behavior because it may switch out during unit tests or if there are accidentally 2 copies of tfjs-node as a dependency).

DEV

Author: Nikhil Thorat

.gitignore

@@ -26,6 +26,14 @@ deps/
 dist/
 tfjs-layers/integration_tests/tfjs2keras/test-data/
 tfjs-layers/integration/typescript/yarn.lock
+
+# Ignore the src, binding, scripts of tfjs-node-gpu since it is copied over when
+# building.
+/tfjs-node-gpu/src
+/tfjs-node-gpu/binding
+/tfjs-node-gpu/scripts
+/tfjs-node-gpu/binding.gyp
+
 lib/
 local.log
 node_modules/

cloudbuild.yml

@@ -71,13 +71,20 @@ steps:
   args: ['./scripts/run-build.sh', 'tfjs-react-native']
   waitFor: ['diff']
 
-# Node.
+# Node CPU.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-node'
   args: ['./scripts/run-build.sh', 'tfjs-node']
   waitFor: ['diff']
 
+# Node GPU.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  id: 'tfjs-node-gpu'
+  args: ['./scripts/run-build.sh', 'tfjs-node-gpu']
+  waitFor: ['diff']
+
 # Integration tests
 - name: 'node:10'
   dir: 'tfjs-core'

scripts/publish-npm.sh

@@ -62,7 +62,7 @@ fi
 ./scripts/make-version.js $1
 
 cd $1
-yarn build-npm
+yarn build-npm for-publish
 cd ..
 
 ./scripts/tag-version.js $1

tfjs-node-gpu/.npmignore

@@ -0,0 +1 @@
+../tfjs-node/.npmignore

tfjs-node-gpu/.nycrc

@@ -0,0 +1 @@
+../tfjs-node/.nycrc

tfjs-node-gpu/cloudbuild.yml

@@ -0,0 +1,44 @@
+steps:
+# Install common dependencies.
+- name: 'node:10'
+  id: 'yarn-common'
+  entrypoint: 'yarn'
+  args: ['install']
+
+# Install tfjs-node dependencies.
+- name: 'node:10'
+  dir: 'tfjs-node-gpu'
+  entrypoint: 'yarn'
+  id: 'yarn'
+  args: ['install']
+  waitFor: ['yarn-common']
+
+# Unit tests.
+- name: 'node:10'
+  dir: 'tfjs-node-gpu'
+  entrypoint: 'yarn'
+  id: 'test'
+  args: ['test-ci']
+  waitFor: ['yarn']
+
+# Integration test.
+- name: 'node:10'
+  dir: 'tfjs-node-gpu'
+  entrypoint: 'yarn'
+  id: 'test-ts-integration'
+  args: ['test-ts-integration']
+  waitFor: ['yarn', 'test']
+
+# CPU / GPU package alignment.
+- name: 'node:10'
+  dir: 'tfjs-node-gpu'
+  entrypoint: 'yarn'
+  id: 'ensure-cpu-gpu-packages-align'
+  args: ['ensure-cpu-gpu-packages-align']
+  waitFor: ['yarn', 'test']
+
+timeout: 1800s
+logsBucket: 'gs://tfjs-build-logs'
+options:
+  logStreamingOption: 'STREAM_ON'
+  substitution_option: 'ALLOW_LOOSE'

tfjs-node-gpu/integration/typescript/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "typescript-integration-test",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "prep": "yalc link @tensorflow/tfjs-node-gpu && cd node_modules/@tensorflow/tfjs-node-gpu && yarn clean-deps && yarn install-gpu",
+    "test": "ts-node src/test.ts && tsc && node dist/test.js"
+  },
+  "dependencies": {
+    "ts-node": "^8.0.3",
+    "typescript": "3.5.3",
+    "yalc": "^1.0.0-pre.27"
+  }
+}

tfjs-node-gpu/integration/typescript/src/test.ts

@@ -0,0 +1,25 @@
+/**
+ * @license
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+Error.stackTraceLimit = Infinity;
+
+import * as tf from '@tensorflow/tfjs-node-gpu';
+
+const a = tf.tensor2d([1, 2, 3, 4], [2, 2], 'float32');
+const b = tf.tensor2d([5, 6, 7, 8], [2, 2], 'float32');
+const c = a.matMul(b);
+console.log(c.dataSync());

tfjs-node-gpu/integration/typescript/tsconfig.json

@@ -0,0 +1 @@
+../../../tfjs-node/integration/typescript/tsconfig.json

tfjs-node-gpu/integration/typescript/yarn.lock

@@ -0,0 +1 @@
+../../../tfjs-node/integration/typescript/yarn.lock

tfjs-node-gpu/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@tensorflow/tfjs-node-gpu",
+  "version": "1.2.9",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "gypfile": true,
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tensorflow/tfjs.git"
+  },
+  "engines": {
+    "node": ">=8.11.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "build-npm": "./scripts/build-npm.sh",
+    "build-addon": "./scripts/build-and-upload-addon.sh",
+    "build-addon-from-source": "node-pre-gyp install --build-from-source",
+    "clean-deps": "rm -rf deps && rm -rf lib",
+    "coverage": "nyc ts-node src/run_tests.ts",
+    "enable-gpu": "node scripts/install.js gpu download && yarn && yarn build-addon-from-source",
+    "ensure-cpu-gpu-packages-align": "node scripts/ensure-cpu-gpu-packages-align.js",
+    "format": "clang-format -i -style=Google binding/*.cc binding/*.h",
+    "install": "yarn install-gpu",
+    "install-gpu": "node scripts/install.js gpu download",
+    "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source",
+    "link-local": "yalc link",
+    "lint": "tslint -p . -t verbose",
+    "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build",
+    "preinstall": "./prep-gpu.sh",
+    "publish-local": "yarn prep && yalc push",
+    "test": "ts-node src/run_tests.ts",
+    "test-ci": "./scripts/test-ci.sh",
+    "test-ts-integration": "./scripts/test-ts-integration.sh",
+    "upload-windows-addon": "./scripts/build-and-upload-windows-addon-gpu.bat"
+  },
+  "devDependencies": {
+    "@types/jasmine": "~2.8.6",
+    "@types/node": "^10.5.1",
+    "@types/progress": "^2.0.1",
+    "@types/rimraf": "~2.0.2",
+    "clang-format": "~1.2.2",
+    "jasmine": "~3.1.0",
+    "node-fetch": "^2.3.0",
+    "node-gyp": "~5.0.3",
+    "nyc": "^13.3.0",
+    "tmp": "^0.0.33",
+    "ts-node": "^5.0.1",
+    "tslint": "~5.9.1",
+    "tslint-no-circular-imports": "^0.7.0",
+    "typescript": "3.5.3",
+    "yalc": "~1.0.0-pre.21"
+  },
+  "dependencies": {
+    "@tensorflow/tfjs": "1.2.9",
+    "adm-zip": "^0.4.11",
+    "https-proxy-agent": "^2.2.1",
+    "node-pre-gyp": "0.13.0",
+    "progress": "^2.0.0",
+    "rimraf": "^2.6.2",
+    "tar": "^4.4.6"
+  },
+  "binary": {
+    "module_name": "tfjs_binding",
+    "module_path": "./lib/napi-v{napi_build_version}",
+    "host": "https://storage.googleapis.com/tf-builds/pre-built-binary",
+    "remote_path": "./napi-v{napi_build_version}/{version}/",
+    "napi_versions": [
+      3,
+      4
+    ]
+  }
+}

tfjs-node/scripts/build-npm-gpu.sh renamed to tfjs-node-gpu/prep-gpu.sh

@@ -16,23 +16,13 @@
 
 set -e
 
-# The binding builds with a symlink by default, for NPM packages change the
-# download option to move libtensorflow next to the prebuilt binary.
-sed -i -e 's/symlink/move/' binding.gyp
+rm -rf ./src
+rm -rf ./binding
+rm -rf ./scripts
+rm -f binding.gyp
 
-# Build GPU:
-sed -i -e 's/tfjs-node"/tfjs-node-gpu"/' package.json
-sed -i -e 's/install.js"/install.js gpu download"/' package.json
-rimraf deps/
-rimraf dist/
-rimraf lib/
-yarn build-addon "$1"
-yarn prep
-tsc --sourceMap false
-# This produces a tarball that will later be used by `npm publish`.
-npm pack
+cp -R ../tfjs-node/src ./src
+cp -R ../tfjs-node/binding ./binding
+cp -R ../tfjs-node/scripts ./scripts
+cp -R ../tfjs-node/binding.gyp .
 
-# Revert GPU changes:
-sed -i -e 's/move/symlink/' binding.gyp
-sed -i -e 's/tfjs-node-gpu"/tfjs-node"/' package.json
-sed -i -e 's/install.js gpu download"/install.js"/' package.json

tfjs-node-gpu/test_images/gif_test.gif

@@ -0,0 +1 @@
+../../tfjs-node/test_images/gif_test.gif

tfjs-node-gpu/test_images/image_bmp_4_channel_test.bmp

@@ -0,0 +1 @@
+../../tfjs-node/test_images/image_bmp_4_channel_test.bmp

tfjs-node-gpu/test_images/image_bmp_test.bmp

@@ -0,0 +1 @@
+../../tfjs-node/test_images/image_bmp_test.bmp

tfjs-node-gpu/test_images/image_jpeg_test.jpeg

@@ -0,0 +1 @@
+../../tfjs-node/test_images/image_jpeg_test.jpeg

tfjs-node-gpu/test_images/image_png_4_channel_test.png

@@ -0,0 +1 @@
+../../tfjs-node/test_images/image_png_4_channel_test.png

tfjs-node-gpu/test_images/image_png_test.png

@@ -0,0 +1 @@
+../../tfjs-node/test_images/image_png_test.png

tfjs-node-gpu/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "sourceMap": true,
+    "removeComments": false,
+    "preserveConstEnums": true,
+    "declaration": true,
+    "target": "es5",
+    "lib": [
+      "es2015",
+      "dom"
+    ],
+    "outDir": "./dist",
+    "noUnusedLocals": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+    "alwaysStrict": true,
+    "noUnusedParameters": false,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "allowUnreachableCode": false
+  },
+  "include": [
+    "src/"
+  ],
+  "exclude": [
+    "node_modules/"
+  ]
+}

tfjs-node-gpu/tslint.json

@@ -0,0 +1 @@
+../tfjs-node/tslint.json