Retrieve and respond on index using etags and handling 304's

Author: Chris Close

lib/api.js

@@ -23,13 +23,20 @@ api.configure = function(config) {
 // GET /package
 api.get(new RegExp('^/([^/]+)$'), function(req, res, match) {
   var name = match[1];
-  Package.getIndex(name, function(err, data) {
+  Package.getIndex(name, function(err, data, etag) {
+    if (req.headers['if-none-match'] === etag) {
+      res.statusCode = 304;
+      res.end();
+      return;
+    }
+
     if (err) {
       logger.error('[500] Error: ', err);
       res.statusCode = 500;
       res.end();
       return;
     }
+    res.setHeader('ETag', etag);
     res.end(JSON.stringify(data));
   });
 });
@@ -52,6 +59,7 @@ api.get(new RegExp('^/([^/]+)/(.+)/([^/]+)$'), function(req, res, match) {
               res.end();
               return;
             }
+
             res.setHeader('Content-type', 'application/octet-stream');
             fs.createReadStream(fullpath).pipe(res);
           });

lib/cache.js

@@ -69,7 +69,7 @@ Cache.prototype.filename = function() {
   return cacheName;
 };
 
-Cache.prototype.complete = function(itemHash, taskHash, cacheFilePath) {
+Cache.prototype.complete = function(itemHash, taskHash, cacheFilePath, etag) {
   if (arguments.length < 3) {
     throw new Error('Invalid call to Cache.complete()');
   }
@@ -83,7 +83,7 @@ Cache.prototype.complete = function(itemHash, taskHash, cacheFilePath) {
 
   // make pluggable: update the cache with the INPUT item stats
 
-  this.data[itemHash].taskResults[taskHash] = { path: cacheFilePath };
+  this.data[itemHash].taskResults[taskHash] = { path: cacheFilePath, etag: etag };
   // console.log('Complete', itemHash, taskHash);
   this.save();
 };
@@ -113,7 +113,7 @@ Cache.prototype.lookup = function(itemHash, taskHash) {
       !cacheMeta.taskResults[taskHash].path) {
     return false;
   }
-  return cacheMeta.taskResults[taskHash].path;
+  return cacheMeta.taskResults[taskHash];
 };
 
 Cache.hash = Cache.prototype.hash = function(method, str) {

lib/package.js

@@ -88,18 +88,15 @@ Package.getIndex = function(pname, onDone) {
     logger.log('[OK] Reusing cached result for ' + uri);
   });
 
-  r.getReadablePath(function(err, fullpath) {
+  r.getReadablePath(function(err, fullpath, etag) {
     if (err) {
       return onDone(err);
     }
 
     r.removeAllListeners('fetch-error');
     r.removeAllListeners('fetch-cached');
 
-    return onDone(err,
-      Package._rewriteLocation(
-        JSON.parse(fs.readFileSync(fullpath))
-      ));
+    return onDone(err, Package._rewriteLocation(JSON.parse(fs.readFileSync(fullpath))), etag);
   });
 };
 

lib/resource.js

@@ -77,6 +77,10 @@ Resource.configure = function(opts) {
 
 Resource.prototype.exists = function() {
   // logger.log('exists', this.url, 'GET', Cache.lookup(this.url, 'GET'));
+  return this.lookup().path;
+};
+
+Resource.prototype.lookup = function() {
   return Cache.lookup(this.url, 'GET');
 };
 
@@ -107,7 +111,7 @@ Resource.prototype.getReadablePath = function(onDone) {
         self.emit('fetch-cached');
         // is the index up to date?
         // yes: return readable stream
-        return onDone(null, self.exists());
+        return onDone(null, self.exists(), self.lookup().etag);
       }
     }
 
@@ -118,12 +122,12 @@ Resource.prototype.getReadablePath = function(onDone) {
       var Package = require('./package.js');
       return Package.getIndex(self.getPackageName(), function(err, data) {
         if (err) {
-          return onDone(err, null);
+          return onDone(err, null, null);
         }
         expectedHash = verify.getSha(self.basename, data);
         verify.check(self.exists(), function(err, actualHash) {
           if (err) {
-            return onDone(err, null);
+            return onDone(err, null, null);
           }
           if (actualHash === expectedHash) {
             return onDone(null, self.exists()); // return readable stream if file is good
@@ -149,9 +153,9 @@ Resource.prototype.getReadablePath = function(onDone) {
     }
     // return readable path
     if (self.err) {
-      return onDone(self.err, null);
+      return onDone(self.err, null, null);
     }
-    onDone(self.err, self.exists());
+    onDone(self.err, self.exists(), self.lookup().etag);
   });
 
   // are we blocking? => nothing more to do so return
@@ -198,6 +202,7 @@ Resource.prototype.retry = function() {
 
 Resource.prototype._afterFetch = function(err, readableStream) {
   var self = this;
+
   clearTimeout(self.fetchTimer);
   // queue returned:
 
@@ -209,6 +214,17 @@ Resource.prototype._afterFetch = function(err, readableStream) {
   }
   // resource fetch OK,
 
+  if (readableStream.statusCode === 304) {
+    // We can rely on the data already in the cache.
+    // No more operations required.
+    logger.log('304 - ' + self.url);
+    lastUpdated[self.url] = new Date();
+    guard.release(self.url);
+    return;
+  }
+
+  self.etag = readableStream.headers['etag'];
+
   // write to disk
   var cachename = Cache.filename(),
       out = fs.createWriteStream(cachename, {flags: 'w'});
@@ -234,7 +250,7 @@ Resource.prototype._afterFetch = function(err, readableStream) {
           return self.retry();
         }
         // mark as OK, return all pending callback
-        Cache.complete(self.url, 'GET', cachename);
+        Cache.complete(self.url, 'GET', cachename, self.etag);
         // set last updated
         lastUpdated[self.url] = new Date();
         guard.release(self.url);
@@ -319,6 +335,11 @@ Resource.prototype._fetchTask = function(onDone) {
     opts.strictSSL = false;
   }
 
+  if (self.lookup() && self.lookup().etag) {
+    opts.headers = opts.headers || {};
+    opts.headers['if-none-match'] = self.lookup().etag;
+  }
+
   logger.log('[GET] ' + this.url);
 
   req = request.get(opts);

util/ls.js

@@ -19,7 +19,7 @@ Object.keys(meta).forEach(function(uri) {
     files.splice(index, 1);
   }
 
-  console.log('Cache check', uri, cache.lookup(uri, 'GET'));
+  console.log('Cache check', uri, cache.lookup(uri, 'GET').path);
 
 });