diff --git a/.gitignore b/.gitignore index 9daa824..bead798 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .DS_Store node_modules +test/tests/*/output +index.js +!src/index.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 2132785..b507df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # changelog +## 0.5.0 + +* Ignore input sourcemaps + ## 0.4.0 * Handle sourcemaps @@ -18,4 +22,4 @@ ## 0.1.1 -* First release \ No newline at end of file +* First release diff --git a/README.md b/README.md index f904c50..3cddd35 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ var gobble = require( 'gobble' ); module.exports = gobble( 'js' ).transform( 'concat', { dest: 'bundle.js' }); ``` -The `dest` property is required. Other values - `files`, `sort`, `separator` and `writeSourcemap`, explained below - are optional. +The `dest` property is required. Other values - `files`, `sort` and `separator`, explained below - are optional. ### `files` @@ -64,12 +64,6 @@ module.exports = gobble( 'js' ) }); ``` -### `writeSourcemap` - -Concatenating javascript or CSS files requires some extra handling of their sourcemaps, specially in complex workflows. With this option set to `true`, the sourcemaps of the files to be concatenated will be parsed, files with no sourcemap will be assigned an identity (1:1) sourcemap, and a new sourcemap will be generated from all of them. - -The default value is `true` when `dest` is a file with a `.js` or `.css` extension, and `false` otherwise. - ## License diff --git a/index.js b/index.js index 25aad13..363f16d 100644 --- a/index.js +++ b/index.js @@ -1,27 +1,35 @@ -var sander = require( 'sander' ), - path = require( 'path' ), - mapSeries = require( 'promise-map-series' ), - minimatch = require( 'minimatch' ); -var SourceNode = require( 'source-map' ).SourceNode; -var SourceMapConsumer = require( 'source-map' ).SourceMapConsumer; - -var sourceMapRegExp = new RegExp(/(?:\/\/#|\/\/@|\/\*#)\s*sourceMappingURL=(.*)\s*(?:\*\/\s*)?$/); -var extensionsRegExp = new RegExp(/(\.js|\.css)$/); +var ref = require( 'path' ), basename = ref.basename, extname = ref.extname, join = ref.join; +var sander = require( 'sander' ); +var mapSeries = require( 'promise-map-series' ); +var minimatch = require( 'minimatch' ); +var ref$1 = require( 'vlq' ), encode = ref$1.encode; + +var getSourcemapComment = { + '.js': function ( file ) { return ("//# sourceMappingURL=" + file + ".map"); }, + '.css': function ( file ) { return ("/*# sourceMappingURL=" + file + ".map */"); } +}; module.exports = function concat ( inputdir, outputdir, options ) { + if ( !options.dest ) throw new Error( ("You must pass a 'dest' option to gobble-concat") ); - if ( !options.dest ) { - throw new Error( 'You must pass a \'dest\' option to gobble-concat' ); - } + var ext = extname( options.dest ); - if ( options.writeSourcemap === undefined ) { - options.writeSourcemap = !!options.dest.match( extensionsRegExp ); - } + var shouldCreateSourcemap = ( ext in getSourcemapComment ) && options.sourceMap !== false; + var separator = options.separator || '\n\n'; + var separatorSemis = separator.split( '\n' ).join( ';' ) return sander.lsr( inputdir ).then( function ( allFiles ) { - var patterns = options.files, - alreadySeen = {}, - nodes = []; + var patterns = options.files; + var alreadySeen = {}; + var fileContents = []; + + var sourceMap = shouldCreateSourcemap ? { + version: 3, + file: basename( options.dest ), + sources: [], + sourcesContent: [], + mappings: '' + } : null; if ( !patterns ) { // use all files @@ -36,73 +44,79 @@ module.exports = function concat ( inputdir, outputdir, options ) { var filtered = allFiles.filter( function ( filename ) { var shouldInclude = !alreadySeen[ filename ] && minimatch( filename, pattern ); - if ( shouldInclude ) { - alreadySeen[ filename ] = true; - } - + if ( shouldInclude ) alreadySeen[ filename ] = true; return shouldInclude; }); return processFiles( filtered.sort( options.sort ) ); }).then( writeResult ); - function processFiles ( filenames ) { + var sourceIndexOffset = 0; + var sourceLineOffset = 0; + var sourceColumnOffset = 0; + + var lineCount = 0; + return mapSeries( filenames.sort( options.sort ), function ( filename ) { - return sander.readFile( inputdir, filename ).then( function ( fileContents ) { - - /// Run a regexp against the code to check for source mappings. - var match = sourceMapRegExp.exec(fileContents); - - if (!match) { -// if (options.verbose) console.log('Creating ident sourcemap for ', filename); - var newNode = new SourceNode(1, 1, filename, fileContents.toString()); - newNode.setSourceContent(filename, fileContents.toString()); - nodes.push( newNode ); - } else { - var sourcemapFilename = match[1]; - - return sander.readFile( inputdir, path.dirname(filename), sourcemapFilename ).then( function ( mapContents ) { - // Sourcemap exists - var parsedMap = new SourceMapConsumer( mapContents.toString() ); - nodes.push( SourceNode.fromStringWithSourceMap( fileContents.toString(), parsedMap ) ); -// if (options.verbose) console.log('Loaded sourcemap for ', filename + ': ' + sourcemapFilename + "(" + mapContents.length + " bytes)"); - }, function(err) { - throw new Error('File ' + inputdir + ' / ' + filename + ' refers to a non-existing sourcemap at ' + sourcemapFilename + ' ' + err); - }); + return sander.readFile( inputdir, filename, { encoding: 'utf-8' }).then( function ( contents ) { + if ( sourceMap ) { + sourceMap.sources.push( join( inputdir, filename ) ); + sourceMap.sourcesContent.push( contents ); + + contents = contents.replace( /^(?:\/\/[@#]\s*sourceMappingURL=(\S+)|\/\*#?\s*sourceMappingURL=(\S+)\s?\*\/)$/gm, '' ); + var lines = contents.split( '\n' ); + + var encoded = lines + .map( function ( line ) { + var encodedLine = ''; + + if ( line.length ) { + encodedLine += encode([ 0, sourceIndexOffset, sourceLineOffset, sourceColumnOffset ]); + + for ( var i = 1; i <= line.length; i += 1 ) { + encodedLine += ',CAAC'; // equivalent to encode([ 1, 0, 0, 1 ]) + } + + sourceLineOffset = 1; + sourceIndexOffset = 0; + sourceColumnOffset = -( line.length ); + + lineCount += 1; + } + + return encodedLine; + }) + .join( ';' ); + + sourceLineOffset = -lineCount; + lineCount = 0; + + sourceMap.mappings += encoded + separatorSemis; } + + fileContents.push( contents ); + + sourceIndexOffset = 1; }); }); } function writeResult () { - if (!nodes[0]) { - // Degenerate case: no matched files - return sander.writeFile( outputdir, options.dest, '' ); - } - - var separatorNode = new SourceNode(null, null, null, options.separator || '\n\n'); + var code = fileContents.join( separator ); - var dest = new SourceNode(null, null, null, ''); - dest.add(nodes[0]); - - for (var i=1; i" ], "license": "MIT", "repository": "https://github.com/gobblejs/gobble-concat", + "scripts": { + "test": "mocha --compilers js:buble/register", + "build": "buble -i src/index.js -o index.js --target node:0.10", + "pretest": "npm run build" + }, "files": [ "index.js" ], @@ -18,6 +23,12 @@ "minimatch": "~1.0.0", "promise-map-series": "~0.2.0", "sander": "^0.1.6", - "source-map": "^0.5.3" + "vlq": "^0.2.1" + }, + "devDependencies": { + "buble": "^0.6.2", + "glob": "^7.0.3", + "mocha": "^2.4.5", + "sander": "^0.1.6" } } diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..a00bf80 --- /dev/null +++ b/src/index.js @@ -0,0 +1,122 @@ +const { basename, extname, join } = require( 'path' ); +const sander = require( 'sander' ); +const mapSeries = require( 'promise-map-series' ); +const minimatch = require( 'minimatch' ); +const { encode } = require( 'vlq' ); + +const getSourcemapComment = { + '.js': file => `//# sourceMappingURL=${file}.map`, + '.css': file => `/*# sourceMappingURL=${file}.map */` +}; + +module.exports = function concat ( inputdir, outputdir, options ) { + if ( !options.dest ) throw new Error( `You must pass a 'dest' option to gobble-concat` ); + + const ext = extname( options.dest ); + + const shouldCreateSourcemap = ( ext in getSourcemapComment ) && options.sourceMap !== false; + const separator = options.separator || '\n\n'; + const separatorSemis = separator.split( '\n' ).join( ';' ) + + return sander.lsr( inputdir ).then( allFiles => { + let patterns = options.files; + let alreadySeen = {}; + let fileContents = []; + + let sourceMap = shouldCreateSourcemap ? { + version: 3, + file: basename( options.dest ), + sources: [], + sourcesContent: [], + mappings: '' + } : null; + + if ( !patterns ) { + // use all files + return processFiles( allFiles.sort( options.sort ) ).then( writeResult ); + } + + if ( typeof patterns === 'string' ) { + patterns = [ patterns ]; + } + + return mapSeries( patterns, pattern => { + let filtered = allFiles.filter( filename => { + const shouldInclude = !alreadySeen[ filename ] && minimatch( filename, pattern ); + + if ( shouldInclude ) alreadySeen[ filename ] = true; + return shouldInclude; + }); + + return processFiles( filtered.sort( options.sort ) ); + }).then( writeResult ); + + function processFiles ( filenames ) { + let sourceIndexOffset = 0; + let sourceLineOffset = 0; + let sourceColumnOffset = 0; + + let lineCount = 0; + + return mapSeries( filenames.sort( options.sort ), filename => { + return sander.readFile( inputdir, filename, { encoding: 'utf-8' }).then( contents => { + if ( sourceMap ) { + sourceMap.sources.push( join( inputdir, filename ) ); + sourceMap.sourcesContent.push( contents ); + + contents = contents.replace( /^(?:\/\/[@#]\s*sourceMappingURL=(\S+)|\/\*#?\s*sourceMappingURL=(\S+)\s?\*\/)$/gm, '' ); + const lines = contents.split( '\n' ); + + const encoded = lines + .map( line => { + let encodedLine = ''; + + if ( line.length ) { + encodedLine += encode([ 0, sourceIndexOffset, sourceLineOffset, sourceColumnOffset ]); + + for ( let i = 1; i <= line.length; i += 1 ) { + encodedLine += ',CAAC'; // equivalent to encode([ 1, 0, 0, 1 ]) + } + + sourceLineOffset = 1; + sourceIndexOffset = 0; + sourceColumnOffset = -( line.length ); + + lineCount += 1; + } + + return encodedLine; + }) + .join( ';' ); + + sourceLineOffset = -lineCount; + lineCount = 0; + + sourceMap.mappings += encoded + separatorSemis; + } + + fileContents.push( contents ); + + sourceIndexOffset = 1; + }); + }); + } + + function writeResult () { + let code = fileContents.join( separator ); + + if ( shouldCreateSourcemap ) { + const comment = getSourcemapComment[ ext ]( basename( options.dest ) ); + + code += `\n${comment}`; + + return sander.Promise.all([ + sander.writeFile( outputdir, options.dest, code ), + sander.writeFile( outputdir, options.dest + '.map', JSON.stringify( sourceMap, null, ' ' ) ) + ]); + } + + return sander.writeFile( outputdir, options.dest, code ); + } + }); +}; diff --git a/test/test.js b/test/test.js new file mode 100644 index 0000000..033c546 --- /dev/null +++ b/test/test.js @@ -0,0 +1,51 @@ +const assert = require( 'assert' ); +const { extname, resolve } = require( 'path' ); +const { lsrSync, mkdir, mkdirSync, readdirSync, readFileSync, rimraf } = require( 'sander' ); +const { describe, it, afterEach } = require( 'mocha' ); +const glob = require( 'glob' ); +const concat = require( '..' ); + +const TMP = resolve( __dirname, 'tmp' ); + +describe( 'gobble-concat', () => { + const TESTS = resolve( __dirname, 'tests' ); + + readdirSync( TESTS ).forEach( dir => { + if ( dir[0] === '.' ) return; + + const input = resolve( TESTS, dir, 'input' ); + const expected = resolve( TESTS, dir, 'expected' ); + const output = resolve( TESTS, dir, 'output' ); + + it( dir, () => { + const options = require( resolve( TESTS, dir, 'options.js' ) ); + + mkdirSync( output ); + + function catalogue ( x ) { + return glob.sync( '**', { cwd: x }) + .map( file => { + let contents = readFileSync( TESTS, dir, x, file, { encoding: 'utf-8' }).trim(); + + if ( extname( file ) === '.map' ) { + contents = JSON.parse( contents ); + + if ( contents.sources ) { + contents.sources = contents.sources.map( source => resolve( dir, x, source ) ); + } + } + + return { file, contents }; + }); + } + + return concat( input, output, options ) + .then( () => { + assert.deepEqual( + catalogue( output ), + catalogue( expected ) + ); + }); + }); + }); +}); diff --git a/test/tests/concatenates-all-files-alphabetically/expected/bundle.js b/test/tests/concatenates-all-files-alphabetically/expected/bundle.js new file mode 100644 index 0000000..63ead00 --- /dev/null +++ b/test/tests/concatenates-all-files-alphabetically/expected/bundle.js @@ -0,0 +1,4 @@ +console.log( 'bar' ); + + +console.log( 'foo' ); diff --git a/test/tests/concatenates-all-files-alphabetically/input/bar.js b/test/tests/concatenates-all-files-alphabetically/input/bar.js new file mode 100644 index 0000000..366ece0 --- /dev/null +++ b/test/tests/concatenates-all-files-alphabetically/input/bar.js @@ -0,0 +1 @@ +console.log( 'bar' ); diff --git a/test/tests/concatenates-all-files-alphabetically/input/foo.js b/test/tests/concatenates-all-files-alphabetically/input/foo.js new file mode 100644 index 0000000..23a0735 --- /dev/null +++ b/test/tests/concatenates-all-files-alphabetically/input/foo.js @@ -0,0 +1 @@ +console.log( 'foo' ); diff --git a/test/tests/concatenates-all-files-alphabetically/options.js b/test/tests/concatenates-all-files-alphabetically/options.js new file mode 100644 index 0000000..39ffe26 --- /dev/null +++ b/test/tests/concatenates-all-files-alphabetically/options.js @@ -0,0 +1,4 @@ +module.exports = { + dest: 'bundle.js', + sourceMap: false +}; diff --git a/test/tests/concatenates-files-in-specified-order/expected/bundle.js b/test/tests/concatenates-files-in-specified-order/expected/bundle.js new file mode 100644 index 0000000..8d33f84 --- /dev/null +++ b/test/tests/concatenates-files-in-specified-order/expected/bundle.js @@ -0,0 +1,4 @@ +console.log( 'foo' ); + + +console.log( 'bar' ); diff --git a/test/tests/concatenates-files-in-specified-order/input/bar.js b/test/tests/concatenates-files-in-specified-order/input/bar.js new file mode 100644 index 0000000..366ece0 --- /dev/null +++ b/test/tests/concatenates-files-in-specified-order/input/bar.js @@ -0,0 +1 @@ +console.log( 'bar' ); diff --git a/test/tests/concatenates-files-in-specified-order/input/foo.js b/test/tests/concatenates-files-in-specified-order/input/foo.js new file mode 100644 index 0000000..23a0735 --- /dev/null +++ b/test/tests/concatenates-files-in-specified-order/input/foo.js @@ -0,0 +1 @@ +console.log( 'foo' ); diff --git a/test/tests/concatenates-files-in-specified-order/options.js b/test/tests/concatenates-files-in-specified-order/options.js new file mode 100644 index 0000000..46638b6 --- /dev/null +++ b/test/tests/concatenates-files-in-specified-order/options.js @@ -0,0 +1,5 @@ +module.exports = { + files: [ 'foo.js', 'bar.js' ], + dest: 'bundle.js', + sourceMap: false +}; diff --git a/test/tests/creates-sourcemap-for-js/expected/bundle.js b/test/tests/creates-sourcemap-for-js/expected/bundle.js new file mode 100644 index 0000000..546a958 --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/expected/bundle.js @@ -0,0 +1,9 @@ +console.log( 'bar' ); + + +console.log( 'baz' ); + + +console.log( 'foo' ); + +//# sourceMappingURL=bundle.js.map diff --git a/test/tests/creates-sourcemap-for-js/expected/bundle.js.map b/test/tests/creates-sourcemap-for-js/expected/bundle.js.map new file mode 100644 index 0000000..f1b3467 --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/expected/bundle.js.map @@ -0,0 +1,15 @@ +{ + "version": 3, + "file": "bundle.js", + "sources": [ + "../input/bar.js", + "../input/baz.js", + "../input/foo.js" + ], + "sourcesContent": [ + "console.log( 'bar' );\n", + "console.log( 'baz' );\n", + "console.log( 'foo' );\n" + ], + "mappings": "AAAA,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;ACDrB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;ACDrB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;" +} diff --git a/test/tests/creates-sourcemap-for-js/input/bar.js b/test/tests/creates-sourcemap-for-js/input/bar.js new file mode 100644 index 0000000..366ece0 --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/input/bar.js @@ -0,0 +1 @@ +console.log( 'bar' ); diff --git a/test/tests/creates-sourcemap-for-js/input/baz.js b/test/tests/creates-sourcemap-for-js/input/baz.js new file mode 100644 index 0000000..a3e6d7c --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/input/baz.js @@ -0,0 +1 @@ +console.log( 'baz' ); diff --git a/test/tests/creates-sourcemap-for-js/input/foo.js b/test/tests/creates-sourcemap-for-js/input/foo.js new file mode 100644 index 0000000..23a0735 --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/input/foo.js @@ -0,0 +1 @@ +console.log( 'foo' ); diff --git a/test/tests/creates-sourcemap-for-js/options.js b/test/tests/creates-sourcemap-for-js/options.js new file mode 100644 index 0000000..2f60be0 --- /dev/null +++ b/test/tests/creates-sourcemap-for-js/options.js @@ -0,0 +1,3 @@ +module.exports = { + dest: 'bundle.js' +}; diff --git a/test/tests/removes-existing-sourcemap-comments/expected/bundle.js b/test/tests/removes-existing-sourcemap-comments/expected/bundle.js new file mode 100644 index 0000000..3c827aa --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/expected/bundle.js @@ -0,0 +1,12 @@ +console.log( 'bar' ); + + + +console.log( 'baz' ); + + + +console.log( 'foo' ); + + +//# sourceMappingURL=bundle.js.map diff --git a/test/tests/removes-existing-sourcemap-comments/expected/bundle.js.map b/test/tests/removes-existing-sourcemap-comments/expected/bundle.js.map new file mode 100644 index 0000000..7a74391 --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/expected/bundle.js.map @@ -0,0 +1,15 @@ +{ + "version": 3, + "file": "bundle.js", + "sources": [ + "../input/bar.js", + "../input/baz.js", + "../input/foo.js" + ], + "sourcesContent": [ + "console.log( 'bar' );\n//# sourceMappingURL=one.js.map\n", + "console.log( 'baz' );\n//# sourceMappingURL=two.js.map\n", + "console.log( 'foo' );\n//# sourceMappingURL=three.js.map\n" + ], + "mappings": "AAAA,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;;ACDrB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;;ACDrB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;;" +} diff --git a/test/tests/removes-existing-sourcemap-comments/input/bar.js b/test/tests/removes-existing-sourcemap-comments/input/bar.js new file mode 100644 index 0000000..fca196f --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/input/bar.js @@ -0,0 +1,2 @@ +console.log( 'bar' ); +//# sourceMappingURL=one.js.map diff --git a/test/tests/removes-existing-sourcemap-comments/input/baz.js b/test/tests/removes-existing-sourcemap-comments/input/baz.js new file mode 100644 index 0000000..04395d4 --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/input/baz.js @@ -0,0 +1,2 @@ +console.log( 'baz' ); +//# sourceMappingURL=two.js.map diff --git a/test/tests/removes-existing-sourcemap-comments/input/foo.js b/test/tests/removes-existing-sourcemap-comments/input/foo.js new file mode 100644 index 0000000..d8ba68b --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/input/foo.js @@ -0,0 +1,2 @@ +console.log( 'foo' ); +//# sourceMappingURL=three.js.map diff --git a/test/tests/removes-existing-sourcemap-comments/options.js b/test/tests/removes-existing-sourcemap-comments/options.js new file mode 100644 index 0000000..2f60be0 --- /dev/null +++ b/test/tests/removes-existing-sourcemap-comments/options.js @@ -0,0 +1,3 @@ +module.exports = { + dest: 'bundle.js' +};