From 20ec2ec320d76112809c4c9082a67849005289e2 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Mon, 21 Mar 2022 23:42:24 +0900 Subject: [PATCH] feat: support import resolving --- .gitignore | 1 + README.md | 22 ++++ bin.js | 48 +++++-- index.js | 36 ++++- package.json | 2 +- test/imports.js | 132 +++++++++++++++++++ test/imports/folder-a/circular.proto | 5 + test/imports/folder-a/deepest.proto | 9 ++ test/imports/folder-a/folder-b/deeper.proto | 6 + test/imports/folder-a/folder-b/deeper2.proto | 7 + test/imports/invalid.proto | 5 + test/imports/valid.proto | 6 + 12 files changed, 261 insertions(+), 18 deletions(-) create mode 100644 test/imports.js create mode 100644 test/imports/folder-a/circular.proto create mode 100644 test/imports/folder-a/deepest.proto create mode 100644 test/imports/folder-a/folder-b/deeper.proto create mode 100644 test/imports/folder-a/folder-b/deeper2.proto create mode 100644 test/imports/invalid.proto create mode 100644 test/imports/valid.proto diff --git a/.gitignore b/.gitignore index cb9868d..9d7f6cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules test/helpers/compiled.js +test/helpers/imports.js diff --git a/README.md b/README.md index 9e05000..5150b8c 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,28 @@ var js = protobuf.toJS(fs.readFileSync('test.proto')) fs.writeFileSync('messages.js', js) ``` +## Imports + +The cli tool supports protocol buffer [imports][] by default. + +**Currently all imports are treated as public and the public/weak keywords +not supported.** + +To use it programmatically you need to pass-in a `filename` & a `resolveImport` +hooks: + +```js +var protobuf = require('protocol-buffers') +var messages = protobuf(null, { + filename: 'initial.proto', + resolveImport (filename) { + // can return a Buffer, String or Schema + } +}) +``` + +[imports]: https://developers.google.com/protocol-buffers/docs/proto3#importing_definitions + ## Performance This module is fast. diff --git a/bin.js b/bin.js index f5574e4..96f8dfc 100755 --- a/bin.js +++ b/bin.js @@ -1,37 +1,45 @@ #!/usr/bin/env node const protobuf = require('./') const fs = require('fs') +const path = require('path') let filename = null let output = null let watch = false let encodings = null +const importPaths = [] // handrolled parser to not introduce minimist as this is used a bunch of prod places // TODO: if this becomes more complicated / has bugs, move to minimist for (let i = 2; i < process.argv.length; i++) { - const v = process.argv[i] - const n = v.split('=')[0] - if (v[0] !== '-') { - filename = v - } else if (n === '--output' || n === '-o' || n === '-wo') { - if (n === '-wo') watch = true - output = v === n ? process.argv[++i] : v.split('=').slice(1).join('=') - } else if (n === '--watch' || n === '-w') { + const parts = process.argv[i].split('=') + const key = parts[0] + const value = parts.slice(1).join('=') + if (key[0] !== '-') { + filename = path.resolve(key) + } else if (key === '--output' || key === '-o' || key === '-wo') { + if (key === '-wo') watch = true + output = value || process.argv[++i] + } else if (key === '--watch' || key === '-w') { watch = true - } else if (n === '--encodings' || n === '-e') { - encodings = v === n ? process.argv[++i] : v.split('=').slice(1).join('=') + } else if (key === '--encodings' || key === '-e') { + encodings = value || process.argv[++i] + } else if (key === '--proto_path' || key === '-I') { + importPaths.push(path.resolve(value || process.argv[++i])) } } +importPaths.push(process.cwd()) if (!filename) { console.error('Usage: protocol-buffers [schema-file.proto] [options]') console.error() - console.error(' --output, -o [output-file.js]') - console.error(' --watch, -w (recompile on schema change)') + console.error(' --output, -o [output-file.js]') + console.error(' --watch, -w (recompile on schema change)') + console.error(' --proto_path, -I [path-root] # base to lookup imports, multiple supported') console.error() process.exit(1) } +filename = path.relative(process.cwd(), filename) if (watch && !output) { console.error('--watch requires --output') @@ -49,6 +57,20 @@ function write () { fs.writeFileSync(output, compile()) } +function resolveImport (filename) { + for (let i = 0; i < importPaths.length; i++) { + const importPath = importPaths[i] + try { + return fs.readFileSync(path.join(importPath, filename)) + } catch (err) {} + } + throw new Error('File "' + filename + '" not found in import path:\n - ' + importPaths.join('\n - ')) +} + function compile () { - return protobuf.toJS(fs.readFileSync(filename), { encodings }) + return protobuf.toJS(null, { + encodings: encodings, + filename: filename, + resolveImport: resolveImport + }) } diff --git a/index.js b/index.js index 4a0bc1f..b332049 100644 --- a/index.js +++ b/index.js @@ -11,11 +11,39 @@ const flatten = function (values) { return result } +function resolveImport (filename, opts, context) { + if (!opts.resolveImport) throw new Error('opts.resolveImport is required if opts.filename is given.') + return _resolveImport(filename, opts, context) +} + +function _resolveImport (filename, opts, context) { + if (context.stack.has(filename)) { + throw new Error('File recursively imports itself: ' + Array.from(context.stack).concat(filename).join(' -> ')) + } + context.stack.add(filename) + const importData = opts.resolveImport(filename) + const sch = (typeof importData === 'object' && !Buffer.isBuffer(importData)) ? importData : schema.parse(importData) + sch.imports.forEach(function (importDef) { + const imported = _resolveImport(importDef, opts, context) + sch.enums = sch.enums.concat(imported.enums) + sch.messages = sch.messages.concat(imported.messages) + }) + context.stack.delete(filename) + return sch +} + module.exports = function (proto, opts) { if (!opts) opts = {} - if (!proto) throw new Error('Pass in a .proto string or a protobuf-schema parsed object') - - const sch = (typeof proto === 'object' && !Buffer.isBuffer(proto)) ? proto : schema.parse(proto) + let sch + if (opts.filename) { + sch = resolveImport(opts.filename, opts, { + cache: {}, + stack: new Set() + }) + } else { + if (!proto) throw new Error('Pass in a .proto string or a protobuf-schema parsed object') + sch = (typeof proto === 'object' && !Buffer.isBuffer(proto)) ? proto : schema.parse(proto) + } // to not make toString,toJSON enumarable we make a fire-and-forget prototype const Messages = function () { @@ -38,5 +66,5 @@ module.exports = function (proto, opts) { } module.exports.toJS = function (proto, opts) { - return compileToJS(module.exports(proto, { inlineEnc: true }), opts) + return compileToJS(module.exports(proto, Object.assign({ inlineEnc: true }, opts)), opts) } diff --git a/package.json b/package.json index 0c5f41b..278ced2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "scripts": { "test": "standard && npm run test-generated && npm run test-compiled", "test-generated": "tape test/*.js", - "test-compiled": "./bin.js test/test.proto -o test/helpers/compiled.js && COMPILED=true tape test/*.js", + "test-compiled": "./bin.js test/test.proto -o test/helpers/compiled.js && ./bin.js test/imports/valid.proto -o test/helpers/imports.js && COMPILED=true tape test/*.js", "bench": "node bench" }, "bugs": { diff --git a/test/imports.js b/test/imports.js new file mode 100644 index 0000000..80cc448 --- /dev/null +++ b/test/imports.js @@ -0,0 +1,132 @@ +const tape = require('tape') +const fs = require('fs') +const path = require('path') +const compile = require('..') +const schema = require('protocol-buffers-schema') + +const projectBase = path.resolve(__dirname, '..') +function resolveImport (filename) { + const filepath = path.resolve(projectBase, filename) + return fs.readFileSync(filepath) +} + +function load (filename) { + return { + filename: filename, + raw: fs.readFileSync(path.join(projectBase, filename)) + } +} + +const valid = load('test/imports/valid.proto') + +function testCompiled (t, compiled) { + t.deepEqual(compiled.DeepestEnum, { + A: 1, + B: 2, + C: 3 + }) + + const encoded = { + deepestMessage: compiled.DeepestMessage.encode({ + field: 3 + }), + deeper: compiled.Deeper.encode({ + foo: { + field: 1 + }, + bar: 2 + }), + deeper2: compiled.Deeper2.encode({ + foo: { + field: 3 + } + }), + valid: compiled.Valid.encode({ + ext: { + foo: { + field: 3 + }, + bar: 1 + } + }) + } + + t.deepEqual(encoded.deeper, Buffer.from('0a0208011002', 'hex')) + + const decoded = { + deepestMessage: compiled.DeepestMessage.decode(encoded.deepestMessage), + deeper: compiled.Deeper.decode(encoded.deeper) + } + t.deepEqual(decoded, { + deepestMessage: { + field: 3 + }, + deeper: { + foo: { + field: 1 + }, + bar: 2 + } + }) + t.end() +} + +if (process.env.COMPILED) { + tape('validated compiled', function (t) { + testCompiled(t, require('./helpers/imports.js')) + }) +} + +tape('valid imports', function (t) { + testCompiled(t, compile(null, { + filename: valid.filename, + resolveImport: resolveImport + })) +}) + +const deeper = load('test/imports/folder-a/folder-b/deeper.proto') +const deeper2 = load('test/imports/folder-a/folder-b/deeper2.proto') +const deepest = load('test/imports/folder-a/deepest.proto') + +tape('valid import with pre-parsed schemas', function (t) { + const cache = {} + cache[valid.filename] = schema.parse(valid.raw) + cache[deeper.filename] = schema.parse(deeper.raw) + cache[deeper2.filename] = schema.parse(deeper2.raw) + cache[deepest.filename] = schema.parse(deepest.raw) + t.deepEquals(Object.keys(compile(null, { + filename: valid.filename, + resolveImport: function (filename) { + return cache[filename] + } + })), ['DeepestEnum', 'Valid', 'Deeper', 'DeepestMessage', 'Deeper2']) + t.end() +}) + +tape('valid import without resolving', function (t) { + t.throws(function () { + compile(valid.raw, {}) + }, /Could not resolve Deeper/) + t.end() +}) + +tape('import with .filename, without resolveImports', function (t) { + t.throws(function () { + compile(null, { + filename: 'test' + }) + }, /opts.resolveImport is required if opts.filename is given./) + t.end() +}) + +const invalid = load('test/imports/invalid.proto') + +tape('circular import', function (t) { + t.throws(function () { + compile(invalid.raw, { + filename: invalid.filename, + resolveImport: resolveImport + }) + }, /File recursively imports itself: test\/imports\/invalid.proto -> test\/imports\/folder-a\/circular.proto -> test\/imports\/invalid.proto/) + t.end() +}) diff --git a/test/imports/folder-a/circular.proto b/test/imports/folder-a/circular.proto new file mode 100644 index 0000000..fc291ed --- /dev/null +++ b/test/imports/folder-a/circular.proto @@ -0,0 +1,5 @@ +import "test/imports/invalid.proto"; + +message Circular { + required int32 num = 1; +} diff --git a/test/imports/folder-a/deepest.proto b/test/imports/folder-a/deepest.proto new file mode 100644 index 0000000..b1d2583 --- /dev/null +++ b/test/imports/folder-a/deepest.proto @@ -0,0 +1,9 @@ +enum DeepestEnum { + A=1; + B=2; + C=3; +} + +message DeepestMessage { + optional DeepestEnum field = 1 [default = B]; +} diff --git a/test/imports/folder-a/folder-b/deeper.proto b/test/imports/folder-a/folder-b/deeper.proto new file mode 100644 index 0000000..20710e7 --- /dev/null +++ b/test/imports/folder-a/folder-b/deeper.proto @@ -0,0 +1,6 @@ +import "test/imports/folder-a/deepest.proto"; + +message Deeper { + required DeepestMessage foo = 1; + required DeepestEnum bar = 2; +} diff --git a/test/imports/folder-a/folder-b/deeper2.proto b/test/imports/folder-a/folder-b/deeper2.proto new file mode 100644 index 0000000..986e125 --- /dev/null +++ b/test/imports/folder-a/folder-b/deeper2.proto @@ -0,0 +1,7 @@ +// This is added to check cache when two different files +// reference the same deep file. +import "test/imports/folder-a/deepest.proto"; + +message Deeper2 { + required DeepestMessage foo = 1; +} diff --git a/test/imports/invalid.proto b/test/imports/invalid.proto new file mode 100644 index 0000000..190366a --- /dev/null +++ b/test/imports/invalid.proto @@ -0,0 +1,5 @@ +import "test/imports/folder-a/circular.proto"; + +message Invalid { + required int32 num = 1; +} diff --git a/test/imports/valid.proto b/test/imports/valid.proto new file mode 100644 index 0000000..d7ae9a4 --- /dev/null +++ b/test/imports/valid.proto @@ -0,0 +1,6 @@ +import "test/imports/folder-a/folder-b/deeper.proto"; +import "test/imports/folder-a/folder-b/deeper2.proto"; + +message Valid { + required Deeper ext = 1; +}