honor etags when making requests

lib/local-storage.js

@@ -33,6 +33,7 @@ function get_boilerplate(name) {
 		// our own object
 		'_distfiles': {},
 		'_attachments': {},
+		'_uplinks': {},
 	};
 }
 
@@ -96,32 +97,32 @@ Storage.prototype._read_create_package = function(name, callback) {
 // synchronize remote package info with the local one
 // TODO: readfile called twice
 Storage.prototype.update_versions = function(name, newdata, callback) {
-	var self = this;
+	var self = this
 	self._read_create_package(name, function(err, data) {
-		if (err) return callback(err);
+		if (err) return callback(err)
 
-		var change = false;
+		var change = false
 		for (var ver in newdata.versions) {
 			if (data.versions[ver] == null) {
-				var verdata = newdata.versions[ver];
+				var verdata = newdata.versions[ver]
 
 				// why does anyone need to keep that in database?
-				delete verdata.readme;
+				delete verdata.readme
 
-				change = true;
+				change = true
-				data.versions[ver] = verdata;
+				data.versions[ver] = verdata
 				
 				if (verdata.dist && verdata.dist.tarball) {
 					var url = utils.parse_tarball_url(
 						verdata.dist.__sinopia_orig_tarball || verdata.dist.tarball
-					);
+					)
 
 					// we do NOT overwrite any existing records
 					if (url != null && data._distfiles[url.filename] == null) {
 						data._distfiles[url.filename] = {
 							url: verdata.dist.__sinopia_orig_tarball || verdata.dist.tarball,
 							sha: verdata.dist.shasum,
-						};
+						}
 					}
 				}
 			}
@@ -130,20 +131,30 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 			// if tag is updated to reference latter version, that's fine
 			var need_change = 
 				(data['dist-tags'][tag] == null) ||
-				(!semver.gte(newdata['dist-tags'][tag], data['dist-tags'][tag]));
+				(!semver.gte(newdata['dist-tags'][tag], data['dist-tags'][tag]))
 
 			if (need_change) {
-				change = true;
+				change = true
-				data['dist-tags'][tag] = newdata['dist-tags'][tag];
+				data['dist-tags'][tag] = newdata['dist-tags'][tag]
+			}
+		}
+		for (var up in newdata._uplinks) {
+			var need_change = 
+				!utils.is_object(data._uplinks[up]) || (newdata._uplinks[up].etag !== data._uplinks[up].etag)
+
+			if (need_change) {
+				change = true
+				data._uplinks[up] = newdata._uplinks[up]
 			}
 		}
 		
 		if (change) {
+			self.logger.debug('updating package info')
 			self._write_package(name, data, callback)
 		} else {
-			callback();
+			callback()
 		}
-	});
+	})
 }
 
 Storage.prototype.add_version = function(name, version, metadata, tag, callback) {
@@ -367,7 +378,7 @@ Storage.prototype.update_package = function(name, updateFn, _callback) {
 }
 
 Storage.prototype._normalize_package = function(pkg) {
-	['versions', 'dist-tags', '_distfiles', '_attachments'].forEach(function(key) {
+	['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks'].forEach(function(key) {
 		if (!utils.is_object(pkg[key])) pkg[key] = {}
 	});
 	if (typeof(pkg._rev) !== 'string') pkg._rev = '0-0000000000000000'

lib/storage.js

@@ -20,6 +20,7 @@ function Storage(config) {
 	this.uplinks = {};
 	for (var p in config.uplinks) {
 		this.uplinks[p] = new Proxy(config.uplinks[p], config);
+		this.uplinks[p].upname = p;
 	}
 	this.local = new Local(config);
 
@@ -52,7 +53,7 @@ Storage.prototype.add_package = function(name, metadata, callback) {
 	}
 
 	async.map(uplinks, function(up, cb) {
-		up.get_package(name, function(err, res) {
+		up.get_package(name, null, function(err, res) {
 			cb(null, [err, res]);
 		});
 	}, function(err, results) {
@@ -313,79 +314,88 @@ Storage.prototype.get_tarball = function(name, filename) {
 // Used storages: local && uplink (proxy_access)
 //
 Storage.prototype.get_package = function(name, callback) {
-	var self = this;
+	var self = this
-	var uplinks = [this.local];
+
-	for (var i in this.uplinks) {
+	self.local.get_package(name, function(err, data) {
-		if (this.config.proxy_access(name, i)) {
+		if (err && (!err.status || err.status >= 500)) {
-			uplinks.push(this.uplinks[i]);
+			// report internal errors right away
+			return cb(err)
 		}
-	}
 
-	var result = {
+		var uplinks = []
-		name: name,
+		for (var i in self.uplinks) {
-		versions: {},
+			if (self.config.proxy_access(name, i)) {
-		'dist-tags': {},
+				uplinks.push(self.uplinks[i])
-	};
+			}
-	var exists = false;
+		}
-	var latest;
 
-	async.map(uplinks, function(up, cb) {
+		var result = data || {
-		up.get_package(name, function(err, up_res) {
+			name: name,
-			if (err) {
+			versions: {},
-				if (up === self.local && (!err.status || err.status >= 500)) {
+			'dist-tags': {},
-					// report internal errors right away
+			_uplinks: {},
-					return cb(err)
+		}
-				} else {
+		var exists = !err
+		var latest = result['dist-tags'].latest
+
+		async.map(uplinks, function(up, cb) {
+			var oldetag = null
+			if (utils.is_object(result._uplinks[up.upname]))
+				oldetag = result._uplinks[up.upname].etag
+
+			up.get_package(name, oldetag, function(err, up_res, etag) {
+				if (err || !up_res) return cb()
+
+				try {
+					utils.validate_metadata(up_res, name)
+				} catch(err) {
 					return cb()
 				}
-			}
 
-			if (up === self.local) {
+				result._uplinks[up.upname] = {
-				// file exists in local repo
+					etag: etag
-				exists = true
-				result._rev = up_res._rev
-			}
-
-			try {
-				utils.validate_metadata(up_res, name);
-			} catch(err) {
-				return cb();
-			}
-
-			var this_version = up_res['dist-tags'].latest;
-			if (latest == null
-			|| (!semver.gt(latest, this_version) && this_version)) {
-				latest = this_version;
-				var is_latest = true;
-			}
-
-			['versions', 'dist-tags'].forEach(function(key) {
-				for (var i in up_res[key]) {
-					if (!result[key][i] || is_latest) {
-						result[key][i] = up_res[key][i];
-					}
 				}
-			});
 
-			// if we got to this point, assume that the correct package exists
+				var this_version = up_res['dist-tags'].latest
-			// on the uplink
+				if (latest == null
-			exists = true;
+				|| (!semver.gt(latest, this_version) && this_version)) {
-			cb();
+					latest = this_version
-		});
+					var is_latest = true
-	}, function(err) {
+				}
-		if (err) return callback(err);
-		if (!exists) {
-			return callback(new UError({
-				status: 404,
-				msg: 'no such package available'
-			}));
-		}
 
-		self.local.update_versions(name, result, function(err) {
+				['versions', 'dist-tags'].forEach(function(key) {
+					for (var i in up_res[key]) {
+						if (!result[key][i] || is_latest) {
+							result[key][i] = up_res[key][i]
+						}
+					}
+				})
+
+				// if we got to this point, assume that the correct package exists
+				// on the uplink
+				exists = true
+				cb()
+			})
+		}, function(err) {
 			if (err) return callback(err);
-			callback(null, result);
+			if (!exists) {
-		});
+				return callback(new UError({
-	});
+					status: 404,
+					msg: 'no such package available'
+				}))
+			}
+
+			self.local.update_versions(name, result, function(err) {
+				if (err) return callback(err)
+
+				var whitelist = ['_rev', 'name', 'versions', 'dist-tags']
+				for (var i in result) {
+					if (!~whitelist.indexOf(i)) delete result[i]
+				}
+				callback(null, result)
+			})
+		})
+	})
 }
 
 module.exports = Storage;

lib/up-storage.js

@@ -34,14 +34,13 @@ function Storage(config, mainconfig) {
 }
 
 Storage.prototype.request = function(options, cb) {
-	var self = this;
+	var self = this
-	var headers = options.headers || {};
+	var headers = options.headers || {}
-	headers.accept = headers.accept || 'application/json';
+	headers.accept = headers.accept || 'application/json'
-	headers['user-agent'] = headers['user-agent'] || this.userAgent;
+	headers['user-agent'] = headers['user-agent'] || this.userAgent
 
-	var method = options.method || 'GET';
+	var method = options.method || 'GET'
-	var uri = options.uri_full || (this.config.url + options.uri);
+	var uri = options.uri_full || (this.config.url + options.uri)
-	var method = options.method || 'GET';
 	self.logger.info({
 		method: method,
 		headers: headers,
@@ -60,24 +59,27 @@ Storage.prototype.request = function(options, cb) {
 		body: json,
 		ca: this.ca,
 	}, function(err, res, body) {
+		var error
 		if (!err) {
 			var res_length = body.length;
 
-			if (options.json) {
+			if (options.json && res.statusCode < 300) {
 				try {
 					body = JSON.parse(body);
-				} catch(err) {
+				} catch(_err) {
-					return cb(err);
+					body = {}
+					err = _err
+					error = err.message
 				}
 			}
 
-			if (utils.is_object(body)) {
+			if (!err && utils.is_object(body)) {
 				if (body.error) {
-					var error = body.error;
+					error = body.error;
 				}
 			}
 		} else {
-			var error = err.message;
+			error = err.message;
 		}
 
 		var msg = '@{!status}, req: \'@{request.method} @{request.url}\'';
@@ -196,10 +198,16 @@ Storage.prototype.add_tarball = function(name, filename) {
 	return stream;
 }
 
-Storage.prototype.get_package = function(name, callback) {
+Storage.prototype.get_package = function(name, etag, callback) {
+	if (etag) {
+		var headers = {
+			'if-none-match': etag
+		}
+	}
 	this.request({
 		uri: '/' + escape(name),
 		json: true,
+		headers: headers,
 	}, function(err, res, body) {
 		if (err) return callback(err);
 		if (res.statusCode === 404) {
@@ -211,7 +219,7 @@ Storage.prototype.get_package = function(name, callback) {
 		if (!(res.statusCode >= 200 && res.statusCode < 300)) {
 			return callback(new Error('bad status code: ' + res.statusCode));
 		}
-		callback(null, body);
+		callback(null, body, res.headers.etag);
 	});
 }