name change + a lot of work...

bin/sinopia

@@ -5,6 +5,7 @@ var yaml = require('js-yaml');
 var commander = require('commander');
 var pkg = yaml.safeLoad(fs.readFileSync('../package.yaml', 'utf8'));
 var server = require('../lib/index');
+var crypto = require('crypto');
 
 commander
 	.option('-l, --listen <[host:]port>', 'host:port number to listen on (default: localhost:4873)', '4873')
@@ -15,10 +16,38 @@ commander
 	.version(pkg.version)
 	.parse(process.argv);
 
+if (commander.config) {
+	var config = yaml.safeLoad(fs.readFileSync(commander.config, 'utf8'));
+} else {
+	var pass = crypto.randomBytes(8).toString('base64').replace(/[=+\/]/g, '');
+	var config = {
+		users: {
+			admin: {
+				password: crypto.createHash('sha1').update(pass).digest('hex')
+			},
+		},
+		uplinks: {
+			npmjs: {
+				url: 'https://registry.npmjs.org/'
+			},
+		},
+		packages: {
+			'/.*/': {
+				publish: ['admin'],
+				access: ['all'],
+				proxy: ['npmjs'],
+			}
+		}
+	}
+	console.log('starting with default config, use user: "admin", pass: "%s" to authenticate', pass);
+}
+
+if (!config.user_agent) config.user_agent = 'Sinopia/'+pkg.version;
+
 var hostport = commander.listen.split(':');
 if (hostport.length < 2) {
 	hostport = [undefined, hostport[0]];
 }
-server({}).listen(hostport[1], hostport[0]);
+server(config).listen(hostport[1], hostport[0]);
 console.log('Server is listening on http://%s:%s/', hostport[0] || 'localhost', hostport[1]);
 

config.example.yaml

@@ -0,0 +1,20 @@
+
+users:
+  user1:
+    # require('crypto').createHash('sha1').update('test').digest('hex')
+    password: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+
+GGusers: &GG user1 user2
+
+packages:
+  /^local-/:
+    #wg1: read write
+    #npmjs: read
+    access: *GG
+    publish: *GG
+    proxy: npmjs owner all
+

lib/config.js

@@ -0,0 +1,121 @@
+var assert = require('assert');
+var crypto = require('crypto');
+
+// [[a, [b, c]], d] -> [a, b, c, d]
+function flatten(array) {
+	var result = [];
+	for (var i=0; i<array.length; i++) {
+		if (Array.isArray(array[i])) {
+			result.push.apply(result, flatten(array[i]));
+		} else {
+			result.push(array[i]);
+		}
+	}
+	return result;
+}
+
+function Config(config) {
+	if (!(this instanceof Config)) return new Config(config);
+	for (var i in config) {
+		if (this[i] == null) this[i] = config[i];
+	}	
+
+	var users = {all:true};
+	
+	var check_user_or_uplink = function(arg) {
+		assert(arg !== 'all' || arg !== 'owner', 'CONFIG: reserved user/uplink name: ' + arg);
+		assert(users[arg] == null, 'CONFIG: duplicate user/uplink name: ' + arg);
+		users[arg] = true;
+	};
+
+	['users', 'uplinks', 'packages'].forEach(function(x) {
+		if (this[x] == null) this[x] = {};
+		assert(
+			typeof(this[x]) === 'object' &&
+			!Array.isArray(this[x])
+		, 'CONFIG: bad "'+x+'" value (object expected)');
+	});
+
+	for (var i in this.users) check_user_or_uplink(i);
+	for (var i in this.uplinks) check_user_or_uplink(i);
+
+	for (var i in this.users) {
+		assert(this.users[i].password, 'CONFIG: no password for user: ' + i);
+		assert(
+			typeof(this.users[i].password) === 'string' && 
+			this.users[i].password.match(/^[a-f0-9]{40}$/)
+		, 'CONFIG: wrong password format for user: ' + i + ', sha1 expected');
+	}
+	
+	for (var i in this.uplinks) {
+		assert(this.uplinks[i].url, 'CONFIG: no url for uplink: ' + i);
+		assert(
+			typeof(this.uplinks[i].url) === 'string'
+		, 'CONFIG: wrong url format for uplink: ' + i);
+		this.uplinks[i].url = this.uplinks[i].url.replace(/\/$/, '');
+	}
+	
+	for (var i in this.packages) {
+		var check_userlist = function(i, hash, action) {
+			if (hash[action] == null) hash[action] = [];
+			assert(
+				typeof(hash[action]) === 'object' &&
+				Array.isArray(hash[action])
+			, 'CONFIG: bad "'+i+'" package '+action+' description (array expected)');
+			hash[action] = flatten(hash[action]);
+			hash[action].forEach(function(user) {
+				assert(
+					users[user] != null
+				, 'CONFIG: "'+i+'" package: user "'+user+'" doesn\'t exist');
+			});
+		}
+
+		assert(
+			typeof(this.packages[i]) === 'object' &&
+			!Array.isArray(this.packages[i])
+		, 'CONFIG: bad "'+i+'" package description (object expected)');
+		check_userlist(i, this.packages[i], 'read');
+		check_userlist(i, this.packages[i], 'proxy');
+		check_userlist(i, this.packages[i], 'publish');
+	}
+
+	return this;
+}
+
+function allow_action(package, who, action) {
+	for (var i in this.packages) {
+		var match_package = i == package;
+		var m = i.match(/^\/(.*)\/$/);
+		if (m && (new RegExp(m[1])).exec(package)) {
+			match_package = true;
+		}
+
+		if (match_package) {
+			return this.packages[i][action].reduce(function(prev, curr) {
+				if (curr === who || curr === 'all') return true;
+				return prev;
+			}, false);
+		}
+	}
+	return false;
+}
+
+Config.prototype.allow_access = function(package, user) {
+	return allow_action.call(this, package, user, 'access');
+}
+
+Config.prototype.allow_publish = function(package, user) {
+	return allow_action.call(this, package, user, 'publish');
+}
+
+Config.prototype.allow_proxy = function(package, uplink) {
+	return allow_action.call(this, package, uplink, 'proxy');
+}
+
+Config.prototype.authenticate = function(user, password) {
+	if (this.users[user] == null) return false;
+	return crypto.createHash('sha1').update(password).digest('hex') === this.users[user].password;
+}
+
+module.exports = Config;
+

lib/index.js

@@ -1,48 +1,38 @@
 var express = require('express');
 var cookies = require('cookies');
-var proxy = require('./proxy');
 var utils = require('./utils');
-var storage = require('./storage');
+var Storage = require('./storage');
+var Config = require('./config');
+var UError = require('./error').UserError;
+var basic_auth = require('./middleware').basic_auth;
+var validate_name = require('./middleware').validate_name;
+var media = require('./middleware').media;
+var expect_json = require('./middleware').expect_json;
 
-function validate_name(req, res, next, value, name) {
-	if (utils.validate_name(req.params.package)) {
-		req.params.package = String(req.params.package);
-		next();
-	} else {
-		next(new Error({
-			status: 403,
-			msg: 'invalid package name',
-		}));
-	}
-};
+module.exports = function(config_hash) {
+	var config = new Config(config_hash);
+	var storage = new Storage(config);
+	var auth = basic_auth(function(user, pass) {
+		return config.authenticate(user, pass);
+   });
 
-function media(expect) {
-	return function(req, res, next) {
-		if (req.headers['content-type'] !== expect) {
-			next(new Error({
-				status: 415,
-				msg: 'wrong content-type, we expect '+expect,
-			}));
-		} else {
-			next();
-		}
-	}
-}
+	var can = function(action) {
+		return [auth, function(req, res, next) {
+			if (config['allow_'+action](req.params.package, req.remoteUser)) {
+				next();
+			} else {
+				next(new UError({
+					status: 403,
+					msg: 'user '+req.remoteUser+' not allowed to '+action+' it'
+				}));
+			}
+		}];
+	};
 
-function expect_json(req, res, next) {
-	if (typeof(req.body) !== 'object') {
-		return next({
-			status: 400,
-			msg: 'can\'t parse incoming json',
-		});
-	}
-	next();
-}
-
-module.exports = function(settings) {
 	var app = express();
 	app.use(express.logger());
 	app.use(express.bodyParser());
+
 	app.param('package', validate_name);
 	app.param('filename', validate_name);
 
@@ -52,27 +42,34 @@ module.exports = function(settings) {
 		});
 	});*/
 
-	app.get('/:package', function(req, res, next) {
+/*	app.get('/-/all', function(req, res) {
+		var https = require('https');
+		var JSONStream = require('JSONStream');
+		var request = require('request')({
+			url: 'https://registry.npmjs.org/-/all',
+			ca: require('./npmsslkeys'),
+		})
+		.pipe(JSONStream.parse('*'))
+		.on('data', function(d) {
+			console.log(d);
+		});
+	});*/
+
+	app.get('/:package', can('access'), function(req, res, next) {
 		storage.get_package(req.params.package, function(err, info) {
-			if (err) {
-				if (err.status === 404) {
-					return proxy.request(req, res);
-				} else {
-					return next(err);
-				}
-			}
+			if (err) return next(err);
 			res.send(info);
 		});
 	});
 
-	app.get('/:package/-/:filename', function(req, res, next) {
+	app.get('/:package/-/:filename', can('access'), function(req, res, next) {
 		storage.get_tarball(req.params.package, req.params.filename, function(err, stream) {
 			if (err) return next(err);
 			if (!stream) {
-				return next({
+				return next(new UError({
 					status: 404,
 					msg: 'package not found'
-				});
+				}));
 			}
 			res.header('content-type', 'application/octet-stream');
 			res.send(stream);
@@ -90,19 +87,37 @@ module.exports = function(settings) {
 			// npmjs.org sets 10h expire
 			expires: new Date(Date.now() + 10*60*60*1000)
 		});
-		res.send({"ok":true,"name":"anonymous","roles":[]});
+		res.send({"ok":true,"name":"somebody","roles":[]});
+	});
+
+	app.get('/-/user/:argument', function(req, res, next) {
+		// can't put 'org.couchdb.user' in route address for some reason
+		if (req.params.argument.split(':')[0] !== 'org.couchdb.user') return next('route');
+		res.status(200);
+		return res.send({
+			ok: 'hello there'
+		});
+	});
+
+	app.put('/-/user/:argument', function(req, res, next) {
+		// can't put 'org.couchdb.user' in route address for some reason
+		if (req.params.argument.split(':')[0] !== 'org.couchdb.user') return next('route');
+		res.status(201);
+		return res.send({
+			ok: 'we don\'t accept new users, but pretend that we do...',
+		});
 	});
 
 	// publishing a package
-	app.put('/:package', media('application/json'), expect_json, function(req, res, next) {
+	app.put('/:package', can('publish'), media('application/json'), expect_json, function(req, res, next) {
 		var name = req.params.package;
 		try {
 			var metadata = utils.validate_metadata(req.body, name);
 		} catch(err) {
-			return next({
+			return next(new UError({
 				status: 422,
 				msg: 'bad incoming package data',
-			});
+			}));
 		}
 
 		storage.add_package(name, metadata, function(err) {
@@ -115,7 +130,7 @@ module.exports = function(settings) {
 	});
 
 	// uploading package tarball
-	app.put('/:package/-/:filename/*', media('application/octet-stream'), function(req, res, next) {
+	app.put('/:package/-/:filename/*', can('publish'), media('application/octet-stream'), function(req, res, next) {
 		var name = req.params.package;
 
 		storage.add_tarball(name, req.params.filename, req, function(err) {
@@ -128,7 +143,7 @@ module.exports = function(settings) {
 	});
 	
 	// adding a version
-	app.put('/:package/:version/-tag/:tag', media('application/json'), expect_json, function(req, res, next) {
+	app.put('/:package/:version/-tag/:tag', can('publish'), media('application/json'), expect_json, function(req, res, next) {
 		var name = req.params.package;
 		var version = req.params.version;
 		var tag = req.params.tag;

lib/middleware.js

@@ -0,0 +1,78 @@
+var utils = require('./utils');
+
+module.exports.validate_name = function validate_name(req, res, next, value, name) {
+	if (utils.validate_name(req.params.package)) {
+		req.params.package = String(req.params.package);
+		next();
+	} else {
+		next(new Error({
+			status: 403,
+			msg: 'invalid package name',
+		}));
+	}
+};
+
+module.exports.media = function media(expect) {
+	return function(req, res, next) {
+		if (req.headers['content-type'] !== expect) {
+			next(new Error({
+				status: 415,
+				msg: 'wrong content-type, we expect '+expect,
+			}));
+		} else {
+			next();
+		}
+	}
+}
+
+module.exports.expect_json = function expect_json(req, res, next) {
+	if (typeof(req.body) !== 'object') {
+		return next({
+			status: 400,
+			msg: 'can\'t parse incoming json',
+		});
+	}
+	next();
+}
+
+module.exports.basic_auth = function basic_auth(callback) {
+	return function(req, res, next) {
+		var authorization = req.headers.authorization;
+
+		if (req.user) return next();
+		if (!authorization) return next({
+			status: 403,
+			msg: 'authorization required',
+		});
+
+		var parts = authorization.split(' ');
+
+		if (parts.length !== 2) return next({
+			status: 400,
+			msg: 'bad authorization header',
+		});
+
+		var scheme = parts[0]
+		, credentials = new Buffer(parts[1], 'base64').toString()
+		, index = credentials.indexOf(':');
+
+		if ('Basic' != scheme || index < 0) return next({
+			status: 400,
+			msg: 'bad authorization header',
+		});
+    
+		var user = credentials.slice(0, index)
+		, pass = credentials.slice(index + 1);
+
+		if (callback(user, pass)) {
+			req.user = req.remoteUser = user;
+			next();
+		} else {
+			next({
+				status: 403,
+				msg: 'bad username/password, access denied',
+			});
+		}
+	}
+};
+

lib/proxy.js

@@ -6,7 +6,7 @@ module.exports.request = function(req, resp) {
 		path: req.url,
 		ca: require('./npmsslkeys'),
 		headers: {
-			'User-Agent': 'npmrepod/0.0.0',
+			'User-Agent': 'sinopia/0.0.0',
 		},
 	}, function(res) {
 		resp.writeHead(res.statusCode, res.headers);

lib/st-local.js

@@ -0,0 +1,107 @@
+var storage = wrap(require('./drivers/fs'));
+var UError = require('./error').UserError;
+var info_file = 'package.json';
+
+function wrap(driver) {
+	if (typeof(driver.create_json) !== 'function') {
+		driver.create_json = function(name, value, cb) {
+			driver.create(name, JSON.stringify(value), cb);
+		};
+	}
+	if (typeof(driver.update_json) !== 'function') {
+		driver.update_json = function(name, value, cb) {
+			driver.update(name, JSON.stringify(value), cb);
+		};
+	}
+	if (typeof(driver.read_json) !== 'function') {
+		driver.read_json = function(name, cb) {
+			driver.read(name, function(err, res) {
+				if (err) return cb(err);
+				cb(null, JSON.parse(res));
+			});
+		};
+	}
+	return driver;
+}
+
+module.exports.add_package = function(name, metadata, callback) {
+	storage.create_json(name + '/' + info_file, metadata, function(err) {
+		if (err && err.code === 'EEXISTS') {
+			return callback(new UError({
+				status: 409,
+				msg: 'this package is already present'
+			}));
+		}
+		callback();
+	});
+}
+
+module.exports.add_version = function(name, version, metadata, tag, callback) {
+	storage.read_json(name + '/' + info_file, function(err, data) {
+		// TODO: race condition
+		if (err) return callback(err);
+
+		if (data.versions[version] != null) {
+			return callback(new UError({
+				status: 409,
+				msg: 'this version already present'
+			}));
+		}
+		data.versions[version] = metadata;
+		data['dist-tags'][tag] = version;
+		storage.update_json(name + '/' + info_file, data, callback);
+	});
+}
+
+module.exports.add_tarball = function(name, filename, stream, callback) {
+	if (name === info_file) {
+		return callback(new UError({
+			status: 403,
+			msg: 'can\'t use this filename'
+		}));
+	}
+
+	var data = new Buffer(0);
+	stream.on('data', function(d) {
+		var tmp = data;
+		data = new Buffer(tmp.length+d.length);
+		tmp.copy(data, 0);
+		d.copy(data, tmp.length);
+	});
+	stream.on('end', function(d) {
+		storage.create(name + '/' + filename, data, function(err) {
+			if (err && err.code === 'EEXISTS') {
+				return callback(new UError({
+					status: 409,
+					msg: 'this tarball is already present'
+				}));
+			}
+			callback.apply(null, arguments);
+		});
+	});
+}
+
+module.exports.get_tarball = function(name, filename, callback) {
+	storage.read(name + '/' + filename, function(err) {
+		if (err && err.code === 'ENOENT') {
+			return callback(new UError({
+				status: 404,
+				msg: 'no such package available'
+			}));
+		}
+		callback.apply(null, arguments);
+	});
+}
+
+module.exports.get_package = function(name, callback) {
+	storage.read_json(name + '/' + info_file, function(err) {
+		if (err && err.code === 'ENOENT') {
+			return callback(new UError({
+				status: 404,
+				msg: 'no such package available'
+			}));
+		}
+		callback.apply(null, arguments);
+	});
+}
+

lib/st-proxy.js

@@ -0,0 +1,31 @@
+var request = require('request');
+var URL = require('url');
+
+function Storage(name, config) {
+	if (!(this instanceof Storage)) return new Storage(config);
+	this.config = config;
+	this.name = name;
+	this.ca;
+
+	if (URL.parse(this.config.uplinks[this.name].url).hostname === 'registry.npmjs.org') {
+		this.ca = require('./npmsslkeys');
+	}
+	return this;
+}
+
+Storage.prototype.get_package = function(name, callback) {
+	request({
+		url: this.config.uplinks[this.name].url + '/' + name,
+		json: true,
+		headers: {
+			'User-Agent': this.config.user_agent,
+		},
+		ca: this.ca,
+	}, function(err, res, body) {
+		if (err) return callback(err);
+		callback(null, body);
+	});
+}
+
+module.exports = Storage;
+

lib/storage.js

@@ -1,100 +1,89 @@
-var storage = wrap(require('./drivers/fs'));
+var async = require('async');
+var semver = require('semver');
 var UError = require('./error').UserError;
-var info_file = '.package.json';
+var local = require('./st-local');
+var Proxy = require('./st-proxy');
+var utils = require('./utils');
 
-function wrap(driver) {
-	if (typeof(driver.create_json) !== 'function') {
-		driver.create_json = function(name, value, cb) {
-			driver.create(name, JSON.stringify(value), cb);
-		};
+function Storage(config) {
+	if (!(this instanceof Storage)) return new Storage(config);
+
+	this.config = config;
+	this.uplinks = {};
+	for (var p in config.uplinks) {
+		this.uplinks[p] = new Proxy(p, config);
 	}
-	if (typeof(driver.update_json) !== 'function') {
-		driver.update_json = function(name, value, cb) {
-			driver.update(name, JSON.stringify(value), cb);
-		};
-	}
-	if (typeof(driver.read_json) !== 'function') {
-		driver.read_json = function(name, cb) {
-			driver.read(name, function(err, res) {
-				if (err) return cb(err);
-				cb(null, JSON.parse(res));
-			});
-		};
-	}
-	return driver;
+
+	return this;
 }
 
-module.exports.add_package = function(name, metadata, callback) {
-	storage.create_json(name + '/' + info_file, metadata, function(err) {
-		if (err && err.code === 'EEXISTS') {
-			return callback(new UError({
-				status: 409,
-				msg: 'this package is already present'
-			}));
+Storage.prototype.add_package = function(name, metadata, callback) {
+	local.add_package(name, metadata, callback);
+}
+
+Storage.prototype.add_version = function(name, version, metadata, tag, callback) {
+	local.add_version(name, version, metadata, tag, callback);
+}
+
+Storage.prototype.add_tarball = function(name, filename, stream, callback) {
+	local.add_tarball(name, filename, stream, callback);
+}
+
+Storage.prototype.get_tarball = function(name, filename, callback) {
+	local.get_tarball(name, filename, callback);
+}
+
+Storage.prototype.get_package = function(name, callback) {
+	var uplinks = [local];
+	for (var i in this.uplinks) {
+		if (this.config.allow_proxy(name, i)) {
+			uplinks.push(this.uplinks[i]);
 		}
-		callback();
-	});
-}
+	}
 
-module.exports.add_version = function(name, version, metadata, tag, callback) {
-	storage.read_json(name + '/' + info_file, function(err, data) {
-		// TODO: race condition
-		if (err) return callback(err);
+	var result = {
+		name: name,
+		versions: {},
+		'dist-tags': {},
+	};
+	var latest;
 
-		if (data.versions[version] != null) {
-			return callback(new UError({
-				status: 409,
-				msg: 'this version already present'
-			}));
-		}
-		data.versions[version] = metadata;
-		data['dist-tags'][tag] = version;
-		storage.update_json(name + '/' + info_file, data, callback);
-	});
-}
+	async.map(uplinks, function(up, cb) {
+		up.get_package(name, function(err, up_res) {
+			if (err) return cb();
 
-module.exports.add_tarball = function(name, filename, stream, callback) {
-	var data = new Buffer(0);
-	stream.on('data', function(d) {
-		var tmp = data;
-		data = new Buffer(tmp.length+d.length);
-		tmp.copy(data, 0);
-		d.copy(data, tmp.length);
-	});
-	stream.on('end', function(d) {
-		storage.create(name + '/' + filename, data, function(err) {
-			if (err && err.code === 'EEXISTS') {
-				return callback(new UError({
-					status: 409,
-					msg: 'this tarball is already present'
-				}));
+			var this_version = up_res['dist-tags'].latest;
+			if (!semver.gt(latest, this_version) && this_version) {
+				latest = this_version;
+				var is_latest = true;
 			}
-			callback.apply(null, arguments);
+
+			try {
+				utils.validate_metadata(up_res, name);
+			} catch(err) {
+				return cb();
+			}
+
+			['versions', 'dist-tags'].forEach(function(key) {
+				for (var i in up_res[key]) {
+					if (!result[key][i] || is_latest) {
+						result[key][i] = up_res[key][i];
+					}
+				}
+			});
+			cb();
 		});
-	});
-}
-
-module.exports.get_tarball = function(name, filename, callback) {
-	storage.read(name + '/' + filename, function(err) {
-		if (err && err.code === 'ENOENT') {
+	}, function(err) {
+		if (err) return callback(err);
+		if (Object.keys(result.versions).length === 0) {
 			return callback(new UError({
 				status: 404,
 				msg: 'no such package available'
 			}));
 		}
-		callback.apply(null, arguments);
+		callback(null, result);
 	});
 }
 
-module.exports.get_package = function(name, callback) {
-	storage.read_json(name + '/' + info_file, function(err) {
-		if (err && err.code === 'ENOENT') {
-			return callback(new UError({
-				status: 404,
-				msg: 'no such package available'
-			}));
-		}
-		callback.apply(null, arguments);
-	});
-}
+module.exports = Storage;
 

package.yaml

@@ -1,5 +1,5 @@
 
-name: npmrepod
+name: sinopia
 version: 0.0.1
 description: Private npm repository server
 
@@ -10,13 +10,15 @@ author:
 main: index.js
 
 bin:
-  npmrepod: ./bin/npmrepod
+  sinopia: ./bin/sinopia
 
 dependencies:
   express: '>= 3.2.5'
   commander: '>= 1.1.1'
   js-yaml: '>= 2.0.5'
   cookies: '>= 0.3.6'
+  async: '*'
+  semver: '*'
 
 preferGlobal: true
 license: BSD