11var request = require ( 'request' )
22 , async = require ( 'async' )
3- , nfs = require ( 'node-fs ' )
3+ , mkdirp = require ( 'mkdirp ' )
44 , fs = require ( 'fs' )
55 , lockerUrl
66 , auth
@@ -10,8 +10,7 @@ exports.sync = function(processInfo, cb) {
1010 auth = processInfo . auth ;
1111 auth . headers = { "Authorization" :"token " + auth . accessToken , "Connection" :"keep-alive" } ;
1212 var cached = { } ;
13- if ( processInfo . config && processInfo . config . cached )
14- cached = processInfo . config . cached ;
13+ if ( processInfo . config && processInfo . config . cached ) cached = processInfo . config . cached ;
1514 lockerUrl = processInfo . lockerUrl ;
1615 exports . syncRepos ( cached , function ( err , repos ) {
1716 if ( err ) console . error ( err ) ;
@@ -33,23 +32,15 @@ exports.syncRepos = function(cached, callback) {
3332 if ( cached [ repo . id ] == ckey ) return cb ( ) ; // if it hasn't changed!
3433 cached [ repo . id ] = ckey ;
3534 // get the watchers, is nice
36- // logger.debug("checking "+repo.id);
37- request . get ( { url :"https://api.github.com/repos/" + repo . id + "/watchers" , json :true , headers :auth . headers } , function ( err , resp , watchers ) {
35+ getWatchers ( repo , function ( err , resp , watchers ) {
3836 repo . watchers = watchers ;
3937 // now see if it's an app special on isle 9
40- request . get ( { uri :'https://raw.github.com/' + repo . id + '/HEAD/package.json' , headers :auth . headers } , function ( err , resp , pkg ) {
41- if ( err || ! pkg ) return cb ( ) ;
42- try {
43- pkg = JSON . parse ( pkg ) ;
44- } catch ( E ) { return cb ( ) ; } // this is going to be really hard to catch, but what can we do from here to let someone know? console.error?
45- var repository = pkg . repository ;
46- if ( ! ( repository && ( repository . static == "true" || repository . static == true ) && repository . type == "app" ) ) return cb ( ) ;
38+ getPackage ( repo , cb , function ( pkg ) {
4739 // ok, we've got an app, make sure settings are valid
48- request . get ( { uri : "https://api.github.com/repos/" + repo . id + "/git/trees/HEAD?recursive=1" , headers : auth . headers , json : true } , function ( err , resp , tree ) {
40+ getTree ( repo , function ( err , resp , tree ) {
4941 if ( err || ! tree || ! tree . tree ) return cb ( ) ;
5042 syncRepo ( repo , tree . tree , function ( err ) {
51- if ( err )
52- {
43+ if ( err ) {
5344 delete cached [ repo . id ] ; // invalidate the cache, it never sync'd
5445 console . error ( err ) ;
5546 return cb ( ) ;
@@ -58,19 +49,25 @@ exports.syncRepos = function(cached, callback) {
5849 pkg . repository . handle = repo . id . replace ( "/" , "-" ) . toLowerCase ( ) ;
5950 pkg . name = pkg . repository . handle ;
6051 pkg . repository . url = 'https://github.com/' + repo . id ;
61- if ( ! pkg . author ) pkg . author = auth . profile . name + ' <' + auth . profile . email + '>' ;
52+ if ( ! pkg . author && auth . profile . name ) {
53+ pkg . author = { name :auth . profile . name } ;
54+ if ( auth . profile . email ) pkg . author . email = auth . profile . email ;
55+ }
6256 if ( ! pkg . version ) pkg . version = "0.0.0" ;
6357 if ( ! pkg . repository . title ) pkg . repository . title = repo . name ;
58+ // XXX: hmmm, these next two if statements should never be true since we only sync the repo
59+ // if static == true and type == "app" per line 46 (smurthas)
6460 if ( ! pkg . repository . hasOwnProperty ( 'static' ) ) pkg . repository . static = true ;
6561 if ( ! pkg . repository . hasOwnProperty ( 'type' ) ) pkg . repository . type = "app" ;
6662 if ( ! pkg . repository . hasOwnProperty ( 'update' ) ) pkg . repository . update = "auto" ;
6763 fs . writeFileSync ( repo . id + "/package.json" , JSON . stringify ( pkg ) ) ; // overwriting :/
6864 request . post ( { url :lockerUrl + '/map/upsert?manifest=Me/github/' + repo . id + '/package.json' } , function ( err , resp ) {
6965 if ( err ) console . error ( err ) ;
70- cb ( ) ;
66+ // XXX: shouldn't we handle this error!!?! (smurthas)
67+ return cb ( ) ;
7168 } ) ;
7269 } ) ;
73- } )
70+ } ) ;
7471 } ) ;
7572 } ) ;
7673 } , function ( err ) {
@@ -80,50 +77,84 @@ exports.syncRepos = function(cached, callback) {
8077}
8178
8279function getIDFromUrl ( url ) {
83- if ( typeof url !== 'string' )
84- return url ;
80+ if ( typeof url !== 'string' ) return url ;
8581 return url . substring ( url . lastIndexOf ( '/' , url . lastIndexOf ( '/' ) - 1 ) + 1 ) ;
8682}
8783
88- function syncRepo ( repo , tree , callback )
89- {
84+ function getWatchers ( repo , callback ) {
85+ request . get ( { url :"https://api.github.com/repos/" + repo . id + "/watchers" ,
86+ headers :auth . headers , json :true } , callback ) ;
87+ }
88+
89+ function getPackage ( repo , notAppCallback , isAppCallback ) {
90+ request . get ( { uri : 'https://raw.github.com/' + repo . id + '/HEAD/package.json' ,
91+ headers :auth . headers } , function ( err , resp , pkg ) {
92+ if ( resp . statusCode === 404 ) return notAppCallback ( ) ;
93+ try {
94+ pkg = JSON . parse ( pkg ) ;
95+ if ( ! isApp ( pkg ) ) return notAppCallback ( ) ;
96+ } catch ( E ) {
97+ console . error ( 'for repo ' + repo . id + ', found a package.json, but couldn\'t parse it, or it wasn\'t an app' , pkg ) ;
98+ return notAppCallback ( ) ;
99+ } // this is going to be really hard to catch, but what can we do from here to let someone know?
100+ return isAppCallback ( pkg ) ;
101+ } ) ;
102+ }
103+
104+ function getTree ( repo , callback ) {
105+ request . get ( { uri :"https://api.github.com/repos/" + repo . id + "/git/trees/HEAD?recursive=1" ,
106+ headers :auth . headers , json :true } , callback ) ;
107+ }
108+
109+ function isApp ( pkg ) {
110+ var repository = pkg . repository ;
111+ return repository && ( repository . static == "true" || repository . static == true ) && repository . type == "app" ;
112+ }
113+
114+ function syncRepo ( repo , tree , callback ) {
90115 var existing = { } ;
91116 try {
92117 var oldtree = JSON . parse ( fs . readFileSync ( repo . id + ".tree.json" ) ) ;
93- for ( var i = 0 ; i < oldtree . length ; i ++ )
94- {
95- existing [ oldtree [ i ] . path ] = oldtree [ i ] . sha ;
96- }
97- } catch ( e ) { } ;
118+ for ( var i in oldtree ) existing [ oldtree [ i ] . path ] = oldtree [ i ] . sha ;
119+ } catch ( e ) {
120+ if ( e . code !== 'ENOENT' ) return callback ( e ) ;
121+ }
98122 // make sure there's at least one tree entry for the repo dir itself
99123 tree . push ( { path :"." , sha :"na" , type :"tree" } ) ;
100- async . forEach ( tree , function ( t , cb ) {
101- if ( t . type != "tree" ) return cb ( ) ;
102- if ( existing [ t . path ] == t . sha ) return cb ( ) ; // no changes
103- nfs . mkdir ( repo . id + "/" + t . path , 0777 , true , cb ) ;
104- } , function ( ) {
124+ async . forEachSeries ( tree , function ( t , cbTree ) {
125+ if ( t . type !== "tree" ) return cbTree ( ) ;
126+ if ( existing [ t . path ] === t . sha ) return cbTree ( ) ; // no changes
127+ mkdirp ( repo . id + "/" + t . path , 0777 , function ( err ) {
128+ if ( err && err . code !== 'EEXIST' ) return cbTree ( err ) ;
129+ return cbTree ( ) ;
130+ } ) ;
131+ } , function ( err ) {
105132 // then the blobs
106133 var errors = [ ] ;
107- async . forEachSeries ( tree , function ( t , cb ) {
108- if ( t . type != "blob" ) return cb ( ) ;
109- if ( existing [ t . path ] == t . sha ) return cb ( ) ; // no changes
110- request . get ( { uri :'https://raw.github.com/' + repo . id + '/HEAD/' + t . path , encoding : 'binary' , headers :auth . headers } , function ( err , resp , body ) {
111- // logger.debug(resp.statusCode + " for "+ t.path);
112- if ( err || ! resp || resp . statusCode != 200 ) {
113- // don't save the sha so it gets retried again
114- errors . push ( repo . id + ": error fetching " + t . path + " status code " + ( resp ? resp . statusCode : 0 ) + " body " + body ) ;
115- t . sha = "" ;
116- return cb ( ) ;
117- }
118- if ( body ) {
119- fs . writeFile ( repo . id + "/" + t . path , body , 'binary' , cb ) ;
120- } else {
121- cb ( ) ;
122- }
134+ async . forEachSeries ( tree , function ( treeItem , cb ) {
135+ return saveBlob ( treeItem , repo , existing , cb ) ;
136+ } , function ( err ) {
137+ fs . writeFile ( repo . id + ".tree.json" , JSON . stringify ( tree ) , function ( err ) {
138+ if ( err ) errors . push ( err ) ;
139+ return callback ( errors . length > 0 ? errors : null ) ;
123140 } ) ;
124- } , function ( ) {
125- fs . writeFile ( repo . id + ".tree.json" , JSON . stringify ( tree ) ) ;
126- callback ( errors . length > 0 ? errors : null ) ;
127141 } ) ;
128142 } ) ;
129143}
144+
145+ function saveBlob ( treeItem , repo , existing , cb ) {
146+ if ( treeItem . type !== "blob" ) return cb ( ) ;
147+ if ( existing [ treeItem . path ] === treeItem . sha ) return cb ( ) ; // no changes
148+ request . get ( { uri :'https://raw.github.com/' + repo . id + '/HEAD/' + treeItem . path ,
149+ encoding : 'binary' ,
150+ headers :auth . headers } ,
151+ function ( err , resp , body ) {
152+ if ( err || ! resp || resp . statusCode !== 200 || body === undefined || body === null ) {
153+ // don't save the sha so it gets retried again
154+ errors . push ( repo . id + ": error fetching " + repo . id + "/" + treeItem . path + " status code " + ( resp ? resp . statusCode : 0 ) + " body " + body ) ;
155+ treeItem . sha = "" ;
156+ return cb ( ) ;
157+ }
158+ fs . writeFile ( repo . id + "/" + treeItem . path , body , 'binary' , cb ) ;
159+ } ) ;
160+ }
0 commit comments