From cb3508da1429ff5dcf9196ef2c9dbc98020cd39a Mon Sep 17 00:00:00 2001 From: Eddie Leffler Date: Tue, 7 Jun 2016 17:38:49 -0700 Subject: [PATCH 1/2] Fix postProcessors race condition - Check for postProcessor bundles to be finished before writing assetsRequiredByEntryPoint to metaData.json - Test case for postProcessor race condition - Code consolidation for async parallel tasks --- index.js | 115 ++++++++++++++----------- test/css-post-processor-stream.js | 26 ++++++ test/example5/static/.gitignore | 1 + test/example5/views/common.css | 3 + test/example5/views/common.js | 2 + test/example5/views/package.json | 5 ++ test/example5/views/page1/package.json | 5 ++ test/example5/views/page1/page1.css | 3 + test/example5/views/page1/page1.jade | 0 test/example5/views/page1/page1.js | 3 + test/example5/views/page2/package.json | 5 ++ test/example5/views/page2/page2.css | 3 + test/example5/views/page2/page2.jade | 7 ++ test/example5/views/page2/page2.js | 3 + test/test.js | 64 +++++++++++++- 15 files changed, 193 insertions(+), 52 deletions(-) create mode 100644 test/css-post-processor-stream.js create mode 100644 test/example5/static/.gitignore create mode 100644 test/example5/views/common.css create mode 100644 test/example5/views/common.js create mode 100644 test/example5/views/package.json create mode 100644 test/example5/views/page1/package.json create mode 100644 test/example5/views/page1/page1.css create mode 100644 test/example5/views/page1/page1.jade create mode 100644 test/example5/views/page1/page1.js create mode 100644 test/example5/views/page2/package.json create mode 100644 test/example5/views/page2/page2.css create mode 100644 test/example5/views/page2/page2.jade create mode 100644 test/example5/views/page2/page2.js diff --git a/index.js b/index.js index c8ccc9d..6ea8663 100644 --- a/index.js +++ b/index.js @@ -104,6 +104,7 @@ function Cartero( entryPoints, outputDirPath, options ) { this.assetsRequiredByEntryPoint = {}; this.metaDataFileAlreadyWrited = false; + this.postProcessorTasks = []; this.watching = false; @@ -310,7 +311,7 @@ Cartero.prototype.processMains = function( callback ) { var tempJavascriptBundleEmitter = new EventEmitter(); tempJavascriptBundleEmitter.setMaxListeners( 0 ); // don't warn if we got lots of listeners, as we need 1 per entry point - + function createTempJsBundleStreamsByEntryPoint() { tempJsBundlesByEntryPoint = _.map( _this.mainPaths, function( thisEntryPoint ) { var thisJsBundlePath = _this.getTempBundlePath( 'js' ); @@ -336,32 +337,59 @@ Cartero.prototype.processMains = function( callback ) { } } ); - if( this.watch ) { - browserifyInstance.on( 'update', function() { - async.parallel( [ function( nextParallel ) { - browserifyInstance.bundle( function( err, buf ) { - if( err ) { - delete err.stream; // gets messy if we dump this to the console - log.error( '', err ); - return; - } + var parallelTasks = [ + function checkBrowserifyBundleDoneTask(nextParallel) { + browserifyInstance.bundle( function( err, buf ) { + if( err ) { + delete err.stream; // gets messy if we dump this to the console + log.error( '', err ); + return; + } + commonJsBundleContents = buf; + nextParallel(); + }); + }, - commonJsBundleContents = buf; - nextParallel(); - } ); - }, function( nextParallel ) { - var numberOfBundlesWritten = 0; + function checkParcelifyDoneTask(nextParallel) { + p.on('done', nextParallel); + }, - tempJavascriptBundleEmitter.on( 'tempBundleWritten', function( thisMainPath, tempBundlePath ) { - numberOfBundlesWritten++; + function checkEntryPointBundlesDoneTask(nextParallel) { + var numberOfBundlesWritten = 0; - // don't have to do anything here... we are just waiting until all of our - // temp bundles have been written before moving on. + tempJavascriptBundleEmitter.on('tempBundleWritten', function(thisMainPath, tempBundlePath) { + numberOfBundlesWritten++; + // don't have to do anything here... we are just waiting until all of our + // temp bundles have been written before moving on. see below comments + if (numberOfBundlesWritten === _this.mainPaths.length) { + nextParallel(); + } + }); + }, - if( numberOfBundlesWritten === _this.mainPaths.length ) nextParallel(); - } ); - } ], function( err ) { + // Make sure all postProcessing bundles have finished since it may complete + // after all entry point js bundles resolve. + function checkPostProcessingBundlesDoneTask(nextParallel) { + var counter = 0; + if (_this.postProcessors.length === 0) { + return nextParallel(); + } + function _postProcessedFinishedHandler(event) { + counter++; + if (counter === _this.postProcessorTasks.length) { + _this.removeListener('postProcessedBundleFinish', _postProcessedFinishedHandler); + return nextParallel(); + } + } + _this.removeListener('postProcessedBundleFinish', _postProcessedFinishedHandler); + _this.on('postProcessedBundleFinish', _postProcessedFinishedHandler); + } + ] + + if( this.watch ) { + browserifyInstance.on( 'update', function() { + async.parallel(parallelTasks, function( err ) { _this.writeAllFinalJavascriptBundles( tempJsBundlesByEntryPoint, needToWriteCommonJsBundle ? commonJsBundleContents : null, function() { // done } ); @@ -370,32 +398,7 @@ Cartero.prototype.processMains = function( callback ) { } // in parallel, let parcelify and browserify do their things - async.parallel( [ function( nextParallel ) { - browserifyInstance.bundle( function( err, buf ) { - if( err ) { - delete err.stream; // gets messy if we dump this to the console - log.error( '', err ); - _this.emit( 'error', err); - return; - } - - commonJsBundleContents = buf; - nextParallel(); - } ); - }, function( nextParallel ) { - p.on( 'done', nextParallel ); - }, function( nextParallel ) { - var numberOfBundlesWritten = 0; - - tempJavascriptBundleEmitter.on( 'tempBundleWritten', function( thisMainPath, tempBundlePath ) { - numberOfBundlesWritten++; - - // don't have to do anything here... we are just waiting until all of our - // temp bundles have been written before moving on. see below comments - - if( numberOfBundlesWritten === _this.mainPaths.length ) nextParallel(); - } ); - } ], function( err ) { + async.parallel(parallelTasks, function(err) { if( err ) return callback( err ); // we have to make sure that parcelify is done before executing this code, since we look up @@ -415,7 +418,7 @@ Cartero.prototype.processMains = function( callback ) { } ); } ); - p.on( 'packageCreated', function( newPackage ) { + p.on('packageCreated', function( newPackage ) { if( newPackage.isParcel ) { _this.parcelsByEntryPoint[ newPackage.mainPath ] = newPackage; } @@ -561,6 +564,12 @@ Cartero.prototype.copyTempBundleToFinalDestination = function( tempBundlePath, a } ); } ); if( postProcessorsToApply.length !== 0 ) { + // Keep track of all not script bundles if we have post processors to + // apply. This is so we know when these bundles are resolved. + if (assetType !== 'script') { + _this.postProcessorTasks.push(finalBundlePath); + } + // apply post processors bundleStream = bundleStream.pipe( combine.apply( null, postProcessorsToApply.map( function( thisPostProcessor ) { return thisPostProcessor( finalBundlePath ); @@ -569,6 +578,12 @@ Cartero.prototype.copyTempBundleToFinalDestination = function( tempBundlePath, a bundleStream.pipe( fs.createWriteStream( finalBundlePath ).on( 'close', function() { fs.unlink( tempBundlePath, function() {} ); + + // Notify that a postProcessed bundle has finished + if (_this.postProcessorTasks.indexOf(finalBundlePath) > -1) { + _this.emit('postProcessedBundleFinish'); + } + _this.emit( 'fileWritten', finalBundlePath, assetType, true, this.watching ); callback( null, finalBundlePath ); @@ -620,7 +635,7 @@ Cartero.prototype.writeCommonJavascriptBundle = function( buf, callback ) { if( this.watching && oldBundlePath ) { // if there is an old bundle that already exists, delete it. this - // happens in watch mode when a new bundle is generated. (note the old bundle + // happens in watch mode when a new bundle is generated. (note the old bundle // likely does not have the same path as the new bundle due to sha1) fs.unlinkSync( oldBundlePath ); } diff --git a/test/css-post-processor-stream.js b/test/css-post-processor-stream.js new file mode 100644 index 0000000..b6e7d4d --- /dev/null +++ b/test/css-post-processor-stream.js @@ -0,0 +1,26 @@ +var through = require('through'); +var path = require('path'); + +module.exports = function (file, options) { + var data = ''; + var options = options || {}; + if (file !== undefined && path.extname(file) !== '.css') { + return through(); + } else { + return through(write, end); + } + + function write(buffer) { + data += buffer; + } + + function end() { + var that = this; + // Simulate the stream to take some time to end so we can test for any race + // conditions. + setTimeout(function() { + that.queue(data); + that.queue(null); + }, 2000); + } +}; diff --git a/test/example5/static/.gitignore b/test/example5/static/.gitignore new file mode 100644 index 0000000..ee88966 --- /dev/null +++ b/test/example5/static/.gitignore @@ -0,0 +1 @@ +assets/ diff --git a/test/example5/views/common.css b/test/example5/views/common.css new file mode 100644 index 0000000..98e599d --- /dev/null +++ b/test/example5/views/common.css @@ -0,0 +1,3 @@ +.boop { + font-size: 24px; +} diff --git a/test/example5/views/common.js b/test/example5/views/common.js new file mode 100644 index 0000000..40ac67d --- /dev/null +++ b/test/example5/views/common.js @@ -0,0 +1,2 @@ + +console.log( 'got common' ); \ No newline at end of file diff --git a/test/example5/views/package.json b/test/example5/views/package.json new file mode 100644 index 0000000..dd9137c --- /dev/null +++ b/test/example5/views/package.json @@ -0,0 +1,5 @@ +{ + "name" : "common-js", + "main" : "common.js", + "style" : "*.css" +} diff --git a/test/example5/views/page1/package.json b/test/example5/views/page1/package.json new file mode 100644 index 0000000..52848ff --- /dev/null +++ b/test/example5/views/page1/package.json @@ -0,0 +1,5 @@ +{ + "view" : "page1.jade", + "main" : "page1.js", + "style" : "*.css" +} \ No newline at end of file diff --git a/test/example5/views/page1/page1.css b/test/example5/views/page1/page1.css new file mode 100644 index 0000000..07a2197 --- /dev/null +++ b/test/example5/views/page1/page1.css @@ -0,0 +1,3 @@ +body { + background : blue; +} diff --git a/test/example5/views/page1/page1.jade b/test/example5/views/page1/page1.jade new file mode 100644 index 0000000..e69de29 diff --git a/test/example5/views/page1/page1.js b/test/example5/views/page1/page1.js new file mode 100644 index 0000000..7508ff5 --- /dev/null +++ b/test/example5/views/page1/page1.js @@ -0,0 +1,3 @@ +require( "../common" ); + +console.log( 'hellooo dave' ); diff --git a/test/example5/views/page2/package.json b/test/example5/views/page2/package.json new file mode 100644 index 0000000..c4cbdff --- /dev/null +++ b/test/example5/views/page2/package.json @@ -0,0 +1,5 @@ +{ + "view" : "page2.jade", + "main" : "page2.js", + "style" : "*.css" +} \ No newline at end of file diff --git a/test/example5/views/page2/page2.css b/test/example5/views/page2/page2.css new file mode 100644 index 0000000..578b276 --- /dev/null +++ b/test/example5/views/page2/page2.css @@ -0,0 +1,3 @@ +body { + background : black; +} \ No newline at end of file diff --git a/test/example5/views/page2/page2.jade b/test/example5/views/page2/page2.jade new file mode 100644 index 0000000..d3ead21 --- /dev/null +++ b/test/example5/views/page2/page2.jade @@ -0,0 +1,7 @@ +doctype html +html(lang="en") + head + | !{cartero_js} + | !{cartero_css} + body + h1 Boop diff --git a/test/example5/views/page2/page2.js b/test/example5/views/page2/page2.js new file mode 100644 index 0000000..fff84d5 --- /dev/null +++ b/test/example5/views/page2/page2.js @@ -0,0 +1,3 @@ +require( "../common" ); +console.log( 'hellooo dave' ); + diff --git a/test/test.js b/test/test.js index 1309a25..790e6dd 100644 --- a/test/test.js +++ b/test/test.js @@ -4,6 +4,7 @@ var test = require( 'tape' ); var fs = require( 'fs' ); var crypto = require( 'crypto' ); var _ = require( 'underscore' ); +var cssPostProcessorStream = require('./css-post-processor-stream.js'); var outputDirFiles = [ 'metaData.json' ]; @@ -32,7 +33,7 @@ test( 'example1', function( t ) { ); t.deepEqual( JSON.parse( fs.readFileSync( path.join( outputDirPath, 'metaData.json' ), 'utf8' ) ).packageMap, packageMap ); - + t.deepEqual( fs.readdirSync( path.join( outputDirPath, packageId ) ).sort(), [ 'page1_bundle_9238125c90e5cfc790e8a5ac8926185dfb162b8c.css', 'page1_bundle_08786d2274344b392803ce9659e6d469ede96834.js' ].sort() @@ -140,7 +141,7 @@ test( 'example3', function( t ) { ); t.deepEqual( JSON.parse( fs.readFileSync( path.join( outputDirPath, 'metaData.json' ), 'utf8' ) ).packageMap, packageMap ); - + var page1PackageFiles = fs.readdirSync( path.join( outputDirPath, parcelIdsByPath[ 'page1' ] ) ).sort(); var page2PackageFiles = fs.readdirSync( path.join( outputDirPath, parcelIdsByPath[ 'page2' ] ) ).sort(); @@ -222,3 +223,62 @@ test( 'example4', function( t ) { } ); } ); + + +test( 'example5 – postProcessors', function( t ) { + t.plan( 4 ); + + var entryPoints = [ path.join( __dirname, 'example5/views/**/*' ) ]; + var entryPointFilter = function( fileName ) { + return path.extname( fileName ) === '.js'; + }; + var outputDirPath = path.join( __dirname, 'example5/static/assets' ); + var packageMap = {}; + var packageIds = []; + var parcelIdsByPath = {}; + var commonJsPackageId = ''; + var c = cartero(entryPoints, outputDirPath, { + entryPointFilter: entryPointFilter, + assetTypes: ['style'], + postProcessors: [function(file) { + return cssPostProcessorStream(file); + }] + }); + + c.on( 'packageCreated', function( newPackage ) { + if( newPackage.package.name === "common-js" ) + commonJsPackageId = newPackage.id; + + if( newPackage.isParcel ) { + var parcelId = newPackage.id; + parcelIdsByPath[ path.relative( path.join( __dirname, 'example5/views' ), newPackage.path ) ] = parcelId; + } + + packageMap[ newPackage.path.slice(1) ] = newPackage.id; + packageIds.push( newPackage.id ); + } ); + + c.on( 'done', function() { + t.deepEqual( + fs.readdirSync( outputDirPath ).sort(), + packageIds.concat( outputDirFiles ).concat( [ 'common_39ebe84e9d92379be5bdf9b8d938b38677a1d620.js' ] ).sort() + ); + + t.deepEqual( JSON.parse( fs.readFileSync( path.join( outputDirPath, 'metaData.json' ), 'utf8' ) ).packageMap, packageMap ); + + var page1PackageFiles = fs.readdirSync( path.join( outputDirPath, parcelIdsByPath[ 'page1' ] ) ).sort(); + var page2PackageFiles = fs.readdirSync( path.join( outputDirPath, parcelIdsByPath[ 'page2' ] ) ).sort(); + + var page1CssBundle = _.find( page1PackageFiles, function( thisFile ) { return path.extname( thisFile ) === '.css'; } ); + page1CssBundle = path.join( outputDirPath, parcelIdsByPath[ 'page1' ], page1CssBundle ); + + var page1CssContents = fs.readFileSync( page1CssBundle, 'utf8' ); + t.ok( page1CssContents.indexOf( 'background : blue' ) !== -1, 'page 1 has correct background color' ); + + var page2CssBundle = _.find( page2PackageFiles, function( thisFile ) { return path.extname( thisFile ) === '.css'; } ); + page2CssBundle = path.join( outputDirPath, parcelIdsByPath[ 'page2' ], page2CssBundle ); + + var page2CssContents = fs.readFileSync( page2CssBundle, 'utf8' ); + t.ok( page2CssContents.indexOf( 'background : black' ) !== -1, 'page 2 has correct background color' ); + } ); +} ); From 5cc33bb44774fbed2ca2b02104c8e995e1243700 Mon Sep 17 00:00:00 2001 From: David Beck Date: Tue, 14 Jun 2016 15:12:26 -0700 Subject: [PATCH 2/2] updates, with support for css common bundle in mind --- index.js | 359 ++++++++++++++++++++++++++----------------------------- 1 file changed, 168 insertions(+), 191 deletions(-) diff --git a/index.js b/index.js index 6ea8663..faf452e 100644 --- a/index.js +++ b/index.js @@ -31,7 +31,7 @@ var resolveTransform = require( './transforms/resolve' ); var kMetaDataFileName = 'metaData.json'; var kAssetsJsonName = 'assets.json'; -var kCommonJavascriptBundleName = 'common'; +var kCommonBundleName = 'common'; module.exports = Cartero; @@ -98,13 +98,13 @@ function Cartero( entryPoints, outputDirPath, options ) { this.packageManifest = {}; this.finalBundlesByParcelId = {}; + this.finalCommonBundles = {}; this.parcelsByEntryPoint = {}; this.packagePathsToIds = {}; this.assetsRequiredByEntryPoint = {}; this.metaDataFileAlreadyWrited = false; - this.postProcessorTasks = []; this.watching = false; @@ -307,28 +307,29 @@ Cartero.prototype.processMains = function( callback ) { var needToWriteCommonJsBundle = false; var commonJsBundleContents; - var tempJsBundlesByEntryPoint; var tempJavascriptBundleEmitter = new EventEmitter(); + var tempBundlesByEntryPoint = {}; // hash of entry points to asset types hashes e.g. { "" : { script : "" } } + var tempCommonBundles = {}; // hash of entry asset types { script : "" } } + tempJavascriptBundleEmitter.setMaxListeners( 0 ); // don't warn if we got lots of listeners, as we need 1 per entry point + + factor( browserifyInstance, { + outputs : function() { + var tempBundleOutputStreams = []; - function createTempJsBundleStreamsByEntryPoint() { - tempJsBundlesByEntryPoint = _.map( _this.mainPaths, function( thisEntryPoint ) { - var thisJsBundlePath = _this.getTempBundlePath( 'js' ); - var writeStream = fs.createWriteStream( thisJsBundlePath, { encoding : 'utf8' } ); + _.each( _this.mainPaths, function( thisEntryPoint ) { + var thisJsBundlePath = _this.getTempBundlePath( 'js' ); + var writeStream = fs.createWriteStream( thisJsBundlePath, { encoding : 'utf8' } ); - writeStream.on( 'finish', function() { - tempJavascriptBundleEmitter.emit( 'tempBundleWritten', thisEntryPoint, thisJsBundlePath ); - } ); + writeStream.on( 'finish', function() { + tempJavascriptBundleEmitter.emit( 'tempBundleWritten', thisEntryPoint, thisJsBundlePath ); + } ); - return { path : thisJsBundlePath, stream : writeStream }; - } ) - } + tempBundleOutputStreams.push( writeStream ); + } ); - factor( browserifyInstance, { - outputs : function() { - createTempJsBundleStreamsByEntryPoint(); - return _.pluck( tempJsBundlesByEntryPoint, 'stream' ); + return tempBundleOutputStreams; }, threshold : function( row, group ) { var putIntoCommonBundle = _this.factorThreshold( row, group ); @@ -337,68 +338,73 @@ Cartero.prototype.processMains = function( callback ) { } } ); - var parallelTasks = [ - function checkBrowserifyBundleDoneTask(nextParallel) { - browserifyInstance.bundle( function( err, buf ) { - if( err ) { - delete err.stream; // gets messy if we dump this to the console - log.error( '', err ); - return; - } - commonJsBundleContents = buf; - nextParallel(); - }); - }, - - function checkParcelifyDoneTask(nextParallel) { - p.on('done', nextParallel); - }, + function waitForAndRegisterBrowserifyBundles( nextParallel ) { + var numberOfBundlesWritten = 0; - function checkEntryPointBundlesDoneTask(nextParallel) { - var numberOfBundlesWritten = 0; + tempJavascriptBundleEmitter.on( 'tempBundleWritten', function( thisMainPath, tempBundlePath ) { + numberOfBundlesWritten++; - tempJavascriptBundleEmitter.on('tempBundleWritten', function(thisMainPath, tempBundlePath) { - numberOfBundlesWritten++; - // don't have to do anything here... we are just waiting until all of our - // temp bundles have been written before moving on. see below comments - if (numberOfBundlesWritten === _this.mainPaths.length) { - nextParallel(); - } - }); - }, + tempBundlesByEntryPoint[ thisMainPath ] = tempBundlesByEntryPoint[ thisMainPath ] || {}; + tempBundlesByEntryPoint[ thisMainPath ].script = tempBundlePath; - // Make sure all postProcessing bundles have finished since it may complete - // after all entry point js bundles resolve. - function checkPostProcessingBundlesDoneTask(nextParallel) { - var counter = 0; - if (_this.postProcessors.length === 0) { - return nextParallel(); - } + // don't have to do anything here... we are just waiting until all of our + // temp bundles have been written before moving on. see below comments - function _postProcessedFinishedHandler(event) { - counter++; - if (counter === _this.postProcessorTasks.length) { - _this.removeListener('postProcessedBundleFinish', _postProcessedFinishedHandler); - return nextParallel(); - } - } - _this.removeListener('postProcessedBundleFinish', _postProcessedFinishedHandler); - _this.on('postProcessedBundleFinish', _postProcessedFinishedHandler); - } - ] + if( numberOfBundlesWritten === _this.mainPaths.length ) nextParallel(); + } ); + } if( this.watch ) { browserifyInstance.on( 'update', function() { - async.parallel(parallelTasks, function( err ) { - _this.writeAllFinalJavascriptBundles( tempJsBundlesByEntryPoint, needToWriteCommonJsBundle ? commonJsBundleContents : null, function() { - // done + log.info( 'Javascript change detected; recreating javascript bundles...' ); + + async.parallel( [ function( nextParallel ) { + browserifyInstance.bundle( function( err, buf ) { + if( err ) { + delete err.stream; // gets messy if we dump this to the console + log.error( '', err ); + return; + } + + commonJsBundleContents = buf; + nextParallel(); + } ); + }, function( nextParallel ) { + waitForAndRegisterBrowserifyBundles( nextParallel ); + } ], function( err ) { + if( err ) return _this.emit( 'error', err ); + + _this.writeFinalBundles( tempBundlesByEntryPoint, tempCommonBundles, function( err ) { + if( err ) return _this.emit( 'error', err ); + + _this.writeMetaDataFile( function( err ) { + if( err ) return _this.emit( 'error', err ); + + // done + } ); } ); } ); } ); } // in parallel, let parcelify and browserify do their things - async.parallel(parallelTasks, function(err) { + async.parallel( [ function( nextParallel ) { + browserifyInstance.bundle( function( err, buf ) { + if( err ) { + delete err.stream; // gets messy if we dump this to the console + log.error( '', err ); + _this.emit( 'error', err ); + return; + } + + commonJsBundleContents = buf; + nextParallel(); + } ); + }, function( nextParallel ) { + p.on( 'done', nextParallel ); + }, function( nextParallel ) { + waitForAndRegisterBrowserifyBundles( nextParallel ); + } ], function( err ) { if( err ) return callback( err ); // we have to make sure that parcelify is done before executing this code, since we look up @@ -406,19 +412,26 @@ Cartero.prototype.processMains = function( callback ) { // that all our temp js bundles have been written, since otherwise we will have nothing to // copy. thus all the crazy async stuff involved. - _this.writeAllFinalJavascriptBundles( tempJsBundlesByEntryPoint, needToWriteCommonJsBundle ? commonJsBundleContents : null, function( err ) { - if( err ) return callback( err ); + async.series( [ function( nextSeries ) { + if( ! needToWriteCommonJsBundle ) return nextSeries(); + if( needToWriteCommonJsBundle ) { + tempCommonBundles.script = _this.getTempBundlePath( 'js' ); + fs.writeFile( tempCommonBundles.script, commonJsBundleContents, nextSeries ); + } + }, function( nextSeries ) { + _this.writeFinalBundles( tempBundlesByEntryPoint, tempCommonBundles, nextSeries ); + }, function( nextSeries ) { // finally, write the meta data file - _this.writeMetaDataFile( function( err ) { - if( err ) _this.emit( 'error', err ); + _this.writeMetaDataFile( nextSeries ); + } ], function( err ) { + if( err ) _this.emit( 'error', err ); - if( callback ) callback(); // and we're done - } ); + if( callback ) callback(); // and we're done } ); } ); - p.on('packageCreated', function( newPackage ) { + p.on( 'packageCreated', function( newPackage ) { if( newPackage.isParcel ) { _this.parcelsByEntryPoint[ newPackage.mainPath ] = newPackage; } @@ -485,16 +498,20 @@ Cartero.prototype.processMains = function( callback ) { } ); p.on( 'bundleWritten', function( bundlePath, assetType, thisParcel, watchModeUpdate ) { - _this.copyTempBundleToParcelDiretory( bundlePath, assetType, thisParcel, function( err ) { - if( err ) return _this.emit( 'error', err ); + tempBundlesByEntryPoint[ thisParcel.mainPath ] = tempBundlesByEntryPoint[ thisParcel.mainPath ] || {}; + tempBundlesByEntryPoint[ thisParcel.mainPath ][ assetType ] = bundlePath; + + if( watchModeUpdate ) { + _this.writeFinalBundles( tempBundlesByEntryPoint, tempCommonBundles, function( err ) { + if( err ) return _this.emit( 'error', err ); - if( watchModeUpdate ) { - _this.writeAssetsJsonForParcel( thisParcel, function( err ) { + _this.writeMetaDataFile( function( err ) { if( err ) return _this.emit( 'error', err ); + // done } ); - } - } ); + } ); + } } ); if( _this.watch ) { @@ -504,24 +521,24 @@ Cartero.prototype.processMains = function( callback ) { if( eventType === 'added' || eventType === 'changed' ) { _this.addAssetToAssetMap( thePackage, asset ); _this.writeIndividualAssetsToDisk( thePackage, [ asset.type ], nextSeries ); - } else - fs.unlink( asset.dstPath, function( err ) { - if( err ) return _this.emit( 'error', err ); - nextSeries(); - } ); + } else { + if( fs.existsSync( asset.dstPath ) ) fs.unlinkSync( asset.dstPath ); + nextSeries(); + } } }, function( nextSeries ) { async.each( thePackage.dependentParcels, function( thisParcel, nextParallel ) { - _this.writeAssetsJsonForParcel( thisParcel, function( err ) { - if( err ) return _this.emit( 'error', err ); - - nextParallel(); - } ); + _this.compileAssetsRequiredByParcel( thisParcel ); + nextParallel(); }, nextSeries ); } ], function( err ) { if( err ) return _this.emit( 'error', err ); - // done + _this.writeMetaDataFile( function( err ) { + if( err ) return _this.emit( 'error', err ); + + // done + } ); } ); } ); @@ -529,7 +546,11 @@ Cartero.prototype.processMains = function( callback ) { _this.writeIndividualAssetsToDisk( thePackage, assetTypesToWriteToDisk, function( err ) { if( err ) return _this.emit( 'error', err ); - // done + _this.writeMetaDataFile( function( err ) { + if( err ) return _this.emit( 'error', err ); + + // done + } ); } ); } ); } @@ -564,12 +585,6 @@ Cartero.prototype.copyTempBundleToFinalDestination = function( tempBundlePath, a } ); } ); if( postProcessorsToApply.length !== 0 ) { - // Keep track of all not script bundles if we have post processors to - // apply. This is so we know when these bundles are resolved. - if (assetType !== 'script') { - _this.postProcessorTasks.push(finalBundlePath); - } - // apply post processors bundleStream = bundleStream.pipe( combine.apply( null, postProcessorsToApply.map( function( thisPostProcessor ) { return thisPostProcessor( finalBundlePath ); @@ -577,13 +592,7 @@ Cartero.prototype.copyTempBundleToFinalDestination = function( tempBundlePath, a } bundleStream.pipe( fs.createWriteStream( finalBundlePath ).on( 'close', function() { - fs.unlink( tempBundlePath, function() {} ); - - // Notify that a postProcessed bundle has finished - if (_this.postProcessorTasks.indexOf(finalBundlePath) > -1) { - _this.emit('postProcessedBundleFinish'); - } - + if( fs.existsSync( tempBundlePath ) ) fs.unlinkSync( tempBundlePath ); _this.emit( 'fileWritten', finalBundlePath, assetType, true, this.watching ); callback( null, finalBundlePath ); @@ -592,111 +601,87 @@ Cartero.prototype.copyTempBundleToFinalDestination = function( tempBundlePath, a } ); }; -Cartero.prototype.copyTempBundleToParcelDiretory = function( tempBundlePath, assetType, parcel, callback ) { +Cartero.prototype.writeFinalBundles = function( tempBundlesByEntryPoint, tempCommonBundles, callback ) { var _this = this; - var outputDirPath = this.getPackageOutputDirectory( parcel ); - var parcelBaseName = path.basename( parcel.path ); - var finalBundlePathWithoutShasumAndExt = path.join( outputDirPath, parcelBaseName + '_bundle' ); - var oldBundlePath = _this.finalBundlesByParcelId[ parcel.id ] && _this.finalBundlesByParcelId[ parcel.id ][ assetType ]; - this.copyTempBundleToFinalDestination( tempBundlePath, assetType, finalBundlePathWithoutShasumAndExt, function( err, finalBundlePath ) { - if( err ) return callback( err ); - - if( ! _this.finalBundlesByParcelId[ parcel.id ] ) _this.finalBundlesByParcelId[ parcel.id ] = {}; - _this.finalBundlesByParcelId[ parcel.id ][ assetType ] = finalBundlePath; - - if( this.watching ) { - // if there is an old bundle that already exists for this asset type, delete it. this - // happens in watch mode when a new bundle is generated. (note the old bundle - // likely does not have the same path as the new bundle due to sha1) - if( oldBundlePath ) { - fs.unlinkSync( oldBundlePath ); - delete _this.finalBundlesByParcelId[ parcel.id ][ assetType ]; - } - } - - callback(); - } ); -}; - -Cartero.prototype.writeCommonJavascriptBundle = function( buf, callback ) { - var _this = this; - var tempBundlePath = this.getTempBundlePath( 'js' ); - var oldBundlePath = this.commonJsBundlePath; + async.series( [ function( nextSeries ) { + // need to write common bundle first, if there is one, so we know its path when writing parcel asset json files + async.forEachOf( tempCommonBundles, function( thisTempCommonBundlePath, assetType, nextEach ) { + var commonBundlePathWithoutShasumAndExt = path.join( _this.outputDirPath, kCommonBundleName ); + var oldBundlePath = _this.finalCommonBundles[ assetType ]; - fs.writeFile( tempBundlePath, buf, function( err ) { - if( err ) return callback( err ); + delete tempCommonBundles[ assetType ]; - var commonBundlePathWithoutShasumAndExt = path.join( _this.outputDirPath, kCommonJavascriptBundleName ); - _this.copyTempBundleToFinalDestination( tempBundlePath, 'script', commonBundlePathWithoutShasumAndExt, function( err, finalBundlePath ) { - if( err ) return callback( err ); + _this.copyTempBundleToFinalDestination( thisTempCommonBundlePath, assetType, commonBundlePathWithoutShasumAndExt, function( err, finalBundlePath ) { + if( err ) return nextEach( err ); - _this.commonJsBundlePath = finalBundlePath; + _this.finalCommonBundles[ assetType ] = finalBundlePath; - if( this.watching && oldBundlePath ) { - // if there is an old bundle that already exists, delete it. this - // happens in watch mode when a new bundle is generated. (note the old bundle - // likely does not have the same path as the new bundle due to sha1) - fs.unlinkSync( oldBundlePath ); - } + if( _this.watching ) { + // if there is an old bundle that already exists, delete it. this + // happens in watch mode when a new bundle is generated. (note the old bundle + // likely does not have the same path as the new bundle due to sha1) + if( oldBundlePath && fs.existsSync( oldBundlePath ) ) fs.unlinkSync( oldBundlePath ); + } - callback(); - } ); - } ); -}; + nextEach(); + } ); + }, nextSeries ); + }, function( nextSeries ) { + + async.forEachOf( tempBundlesByEntryPoint, function( thisParcelTempBundles, thisMainPath, nextEntryPoint ) { + var thisParcel = _this.parcelsByEntryPoint[ thisMainPath ]; -Cartero.prototype.writeAllFinalJavascriptBundles = function( tempJsBundlesByEntryPoint, commonJsBundleContents, callback ) { - var _this = this; + async.forEachOf( thisParcelTempBundles, function( thisTempBundlePath, assetType, nextAssetType ) { + var outputDirPath = _this.getPackageOutputDirectory( thisParcel ); + var parcelBaseName = path.basename( thisParcel.path ); + var finalBundlePathWithoutShasumAndExt = path.join( outputDirPath, parcelBaseName + '_bundle' ); + var oldBundlePath = _this.finalBundlesByParcelId[ thisParcel.id ] && _this.finalBundlesByParcelId[ thisParcel.id ][ assetType ]; - async.series( [ function( nextSeries ) { - // need to write common bundle first, if there is one, so we know its path when writing parcel asset json files - if( commonJsBundleContents ) _this.writeCommonJavascriptBundle( commonJsBundleContents, function( err ) { - if( err ) _this.emit( 'error', err ); + delete tempBundlesByEntryPoint[ thisMainPath ][ assetType ]; + + _this.copyTempBundleToFinalDestination( thisTempBundlePath, assetType, finalBundlePathWithoutShasumAndExt, function( err, finalBundlePath ) { + if( err ) return nextAssetType( err ); - nextSeries(); - } ); - else { - this.commonJsBundleContents = null; - nextSeries(); - } - }, function( nextSeries ) { - async.forEachOf( tempJsBundlesByEntryPoint, function( thisTempBundle, index, nextEach ) { - var thisMainPath = _this.mainPaths[ index ]; - var thisParcel = _this.parcelsByEntryPoint[ thisMainPath ]; + _this.finalBundlesByParcelId[ thisParcel.id ] = _this.finalBundlesByParcelId[ thisParcel.id ] || {}; + _this.finalBundlesByParcelId[ thisParcel.id ][ assetType ] = finalBundlePath; - _this.copyTempBundleToParcelDiretory( thisTempBundle.path, 'script', thisParcel, function( err ) { - if( err ) return callback( err ); + if( _this.watching ) { + // if there is an old bundle that already exists for this asset type, delete it. this + // happens in watch mode when a new bundle is generated. (note the old bundle + // likely does not have the same path as the new bundle due to sha1) - _this.writeAssetsJsonForParcel( thisParcel, function( err ) { - if( err ) return callback( err ); + if( oldBundlePath && fs.existsSync( oldBundlePath ) ) fs.unlinkSync( oldBundlePath ); + } - nextEach(); + nextAssetType(); } ); + }, function( err ) { + if( err ) return nextEntryPoint( err ); + + delete tempBundlesByEntryPoint[ thisMainPath ]; + _this.compileAssetsRequiredByParcel( thisParcel ); + + nextEntryPoint(); } ); }, nextSeries ); } ], callback ); }; -Cartero.prototype.writeAssetsJsonForParcel = function( parcel, callback ) { +Cartero.prototype.compileAssetsRequiredByParcel = function( parcel ) { var _this = this; var bundles = _this.finalBundlesByParcelId[ parcel.id ]; var content = {}; // if we have a common bundle, it needs to come before parcel specific bundle - if( this.commonJsBundlePath ) { - content.script = content.script || []; - content.script.push( path.relative( this.outputDirPath, this.commonJsBundlePath ) ); - } - - if( bundles && bundles.script ) { - content.script = content.script || []; - content.script.push( path.relative( this.outputDirPath, bundles.script ) ); - } - - _.without( _this.assetTypes, 'script' ).forEach( function( thisAssetType ) { - var concatenateThisAssetType = _.contains( _this.assetTypesToConcatenate, thisAssetType ); + _.each( this.finalCommonBundles, function( thisBundlePath, thisAssetType ) { + content[ thisAssetType ] = content[ thisAssetType ] || []; + content[ thisAssetType ].push( path.relative( _this.outputDirPath, thisBundlePath ) ); + } ); + _.each( _this.assetTypes.concat( [ 'script' ] ), function( thisAssetType ) { + var concatenateThisAssetType = thisAssetType === 'script' || _.contains( _this.assetTypesToConcatenate, thisAssetType ); var filesOfThisType; if( concatenateThisAssetType ) filesOfThisType = bundles && bundles[ thisAssetType ] ? [ bundles[ thisAssetType ] ] : []; @@ -708,14 +693,6 @@ Cartero.prototype.writeAssetsJsonForParcel = function( parcel, callback ) { } ); _this.assetsRequiredByEntryPoint[ _this.getPackageMapKeyFromPath( parcel.mainPath ) ] = content; - var packageDirPath = this.getPackageOutputDirectory( parcel ); - if( _this.watching && this.metaDataFileAlreadyWrited ) { - _this.writeMetaDataFile( function( err ) { - if( err ) _this.emit( 'error', err ); - - if( callback ) callback(); - } ); - } else callback(); }; Cartero.prototype.getPackageOutputDirectory = function( thePackage ) {