turning uplink requests into streams

lib/index.js

@@ -83,7 +83,6 @@ module.exports = function(config_hash) {
 			return next(err);
 		});
 		res.header('content-type', 'application/octet-stream');
-		stream.on('data', console.log);//	rstream.pipe(stream);
 		stream.pipe(res);
 	});
 	

lib/st-local.js

@@ -81,8 +81,7 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 					// we do NOT overwrite any existing records
 					if (url != null && data._distfiles[url.filename] == null) {
 						data._distfiles[url.filename] = {
-							url: verdata.dist.tarball,
-							prefix: url.protocol + '//' + url.host + url.prepath,
+							url: verdata.dist.__sinopia_orig_tarball || verdata.dist.tarball,
 							sha: verdata.dist.shasum,
 						};
 					}

lib/st-proxy.js

@@ -1,4 +1,5 @@
 var request = require('request');
+var through = require('through');
 var UError = require('./error').UserError;
 var URL = require('url');
 
@@ -11,11 +12,13 @@ function Storage(config, mainconfig) {
 	this.url = URL.parse(this.config.url);
 	if (this.url.hostname === 'registry.npmjs.org') {
 		this.ca = require('./npmsslkeys');
-		if (this.config._autogenerated) {
+
+		// npm registry is too slow working with ssl :(
+		/*if (this.config._autogenerated) {
 			// encrypt all the things!
 			this.url.protocol = 'https';
 			this.config.url = URL.format(this.url);
-		}
+		}*/
 	}
 
 	this.config.url = this.config.url.replace(/\/$/, '');
@@ -27,7 +30,7 @@ Storage.prototype.can_fetch_url = function(url) {
 
 	return url.protocol === this.url.protocol
 	    && url.host === this.url.host
-	    && url.path === this.url.path
+	    && url.path.indexOf(this.url.path) === 0
 }
 
 Storage.prototype.get_package = function(name, callback) {
@@ -53,27 +56,47 @@ Storage.prototype.get_package = function(name, callback) {
 	});
 }
 
-Storage.prototype.get_tarball = function(name, filename, callback) {
-	request({
-		url: this.config.url + '/' + name + '/-/' + filename,
+Storage.prototype.get_tarball = function(name, filename) {
+	return this.get_url(this.config.url + '/' + name + '/-/' + filename);
+}
+
+Storage.prototype.get_url = function(url) {
+	var stream = through(function(data) {
+		this.queue(data);
+	}, function() {
+		this.queue(null);
+	});
+
+	var rstream = request({
+		url: url,
 		headers: {
 			'User-Agent': this.ua,
 		},
 		ca: this.ca,
 		encoding: null,
-	}, function(err, res, body) {
-		if (err) return callback(err);
+	});
+
+	rstream.on('response', function(res) {
 		if (res.statusCode === 404) {
-			return callback(new UError({
+			return stream.emit('error', new UError({
 				msg: 'file doesn\'t exist on uplink',
 				status: 404,
 			}));
 		}
 		if (!(res.statusCode >= 200 && res.statusCode < 300)) {
-			return callback(new Error('bad status code: ' + res.statusCode));
+			return stream.emit('error', new UError({
+				msg: 'bad uplink status code: ' + res.statusCode,
+				status: 500,
+			}));
 		}
-		callback(null, body);
+
+		rstream.pipe(stream);
 	});
+
+	rstream.on('error', function(err) {
+		stream.emit('error', err);
+	});
+	return stream;
 }
 
 module.exports = Storage;

lib/storage.js

@@ -108,12 +108,12 @@ Storage.prototype.get_tarball = function(name, filename, callback) {
 			}
 			if (uplink == null) {
 				uplink = new Proxy({
-					url: file.prefix,
+					url: file.url,
 					_autogenerated: true,
 				}, self.config);
 			}
 
-			var rstream2 = uplink.get_tarball(name, filename);
+			var rstream2 = uplink.get_url(file.url);
 			rstream2.on('error', function(err) {
 				stream.emit('error', err);
 			});