Infrastructure/verdaccio
0
Fork 0
mirror of https://github.com/verdaccio/verdaccio.git synced 2025-01-27 22:59:51 -05:00
Code Issues Activity

refactor: storage classes clean up

Browse source
This commit is contained in:
Juan Picado @jotadeveloper 2017-07-29 19:11:52 +02:00
parent 14bbd93722
commit c4555cd64e
No known key found for this signature in database
GPG key ID: 18AC54485952D158
3 changed files with 124 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

136
src/lib/storage.js
View file

@ -14,6 +14,16 @@ const MyStreams = require('./storage/streams');
const Proxy = require('./storage/up-storage'); const Proxy = require('./storage/up-storage');
const Utils = require('./utils'); const Utils = require('./utils');
const WHITELIST = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time'];
const getDefaultMetadata = (name) => {
return {
'name': name,
'versions': {},
'dist-tags': {},
'_uplinks': {},
};
};
/** /**
* Implements Storage interface * Implements Storage interface
* (same for storage.js, local-storage.js, up-storage.js). * (same for storage.js, local-storage.js, up-storage.js).
@ -51,7 +61,7 @@ class Storage {
*/ */
const checkPackageLocal = () => { const checkPackageLocal = () => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.localStorage.getPackage(name, {}, (err, results) => { this.localStorage.getPackageMetadata(name, {}, (err, results) => {
if (!_.isNil(err) && err.status !== 404) { if (!_.isNil(err) && err.status !== 404) {
return reject(err); return reject(err);
} }
@ -69,7 +79,7 @@ class Storage {
*/ */
const checkPackageRemote = () => { const checkPackageRemote = () => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
self._sync_package_with_uplinks(name, null, {}, (err, results, err_results) => { self._syncUplinksMetadata(name, null, {}, (err, results, err_results) => {
// something weird // something weird
if (err && err.status !== 404) { if (err && err.status !== 404) {
return reject(err); return reject(err);
@ -248,20 +258,20 @@ class Storage {
let err404 = err; let err404 = err;
rstream.abort(); rstream.abort();
rstream = null; // gc rstream = null; // gc
self.localStorage.getPackage(name, function(err, info) { self.localStorage.getPackageMetadata(name, (err, info) => {
if (!err && info._distfiles && _.isNil(info._distfiles[filename]) === false) { if (_.isNil(err) && info._distfiles && _.isNil(info._distfiles[filename]) === false) {
// information about this file exists locally // information about this file exists locally
serve_file(info._distfiles[filename]); serveFile(info._distfiles[filename]);
} else { } else {
// we know nothing about this file, trying to get information elsewhere // we know nothing about this file, trying to get information elsewhere
self._sync_package_with_uplinks(name, info, {}, function(err, info) { self._syncUplinksMetadata(name, info, {}, (err, info) => {
if (err) { if (_.isNil(err) === false) {
return readStream.emit('error', err); return readStream.emit('error', err);
} }
if (!info._distfiles || _.isNil(info._distfiles[filename])) { if (_.isNil(info._distfiles) || _.isNil(info._distfiles[filename])) {
return readStream.emit('error', err404); return readStream.emit('error', err404);
} }
serve_file(info._distfiles[filename]); serveFile(info._distfiles[filename]);
}); });
} }
}); });
@ -279,7 +289,7 @@ class Storage {
* Fetch and cache local/remote packages. * Fetch and cache local/remote packages.
* @param {Object} file define the package shape * @param {Object} file define the package shape
*/ */
function serve_file(file) { function serveFile(file) {
let uplink = null; let uplink = null;
for (let p in self.uplinks) { for (let p in self.uplinks) {
if (self.uplinks[p].isUplinkValid(file.url)) { if (self.uplinks[p].isUplinkValid(file.url)) {
@ -300,7 +310,7 @@ class Storage {
let on_open = function() { let on_open = function() {
// prevent it from being called twice // prevent it from being called twice
on_open = function() {}; on_open = function() {};
let rstream2 = uplink.get_url(file.url); let rstream2 = uplink.fetchTarball(file.url);
rstream2.on('error', function(err) { rstream2.on('error', function(err) {
if (savestream) { if (savestream) {
savestream.abort(); savestream.abort();
@ -357,21 +367,23 @@ class Storage {
* @param {*} callback * @param {*} callback
*/ */
get_package(name, options, callback) { get_package(name, options, callback) {
if (typeof(options) === 'function') { if (_.isFunction(options)) {
callback = options, options = {}; callback = options, options = {};
} }
this.localStorage.getPackage(name, options, (err, data) => { this.localStorage.getPackageMetadata(name, options, (err, data) => {
if (err && (!err.status || err.status >= 500)) { if (err && (!err.status || err.status >= 500)) {
// report internal errors right away // report internal errors right away
return callback(err); return callback(err);
} }
this._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) { this._syncUplinksMetadata(name, data, options, function(err, result, uplink_errors) {
if (err) return callback(err); if (err) {
const whitelist = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time']; return callback(err);
}
for (let i in result) { for (let i in result) {
if (whitelist.indexOf(i) === -1) { if (WHITELIST.indexOf(i) === -1) {
delete result[i]; delete result[i];
} }
} }
@ -454,9 +466,10 @@ class Storage {
let packages = []; let packages = [];
const getPackage = function(i) { const getPackage = function(i) {
self.localStorage.getPackage(locals[i], function(err, info) { self.localStorage.getPackageMetadata(locals[i], function(err, info) {
if (!err) { if (_.isNil(err) === false) {
let latest = info['dist-tags'].latest; const latest = info['dist-tags'].latest;
if (latest && info.versions[latest]) { if (latest && info.versions[latest]) {
packages.push(info.versions[latest]); packages.push(info.versions[latest]);
} else { } else {
@ -480,7 +493,7 @@ class Storage {
} }
/** /**
* Function fetches package information from uplinks and synchronizes it with local data * Function fetches package metadata from uplinks and synchronizes it with local data
if package is available locally, it MUST be provided in pkginfo if package is available locally, it MUST be provided in pkginfo
returns callback(err, result, uplink_errors) returns callback(err, result, uplink_errors)
* @param {*} name * @param {*} name
@ -488,43 +501,44 @@ class Storage {
* @param {*} options * @param {*} options
* @param {*} callback * @param {*} callback
*/ */
_sync_package_with_uplinks(name, packageInfo, options, callback) { _syncUplinksMetadata(name, packageInfo, options, callback) {
let self = this;
let exists = false; let exists = false;
if (!packageInfo) { const self = this;
exists = false; const upLinks = [];
packageInfo = { if (_.isNil(packageInfo)) {
'name': name, exists = false;
'versions': {}, packageInfo = getDefaultMetadata(name);
'dist-tags': {},
'_uplinks': {},
};
} else { } else {
exists = true; exists = true;
} }
let upLinks = [];
for (let i in self.uplinks) { for (let up in this.uplinks) {
if (self.config.hasProxyTo(name, i)) { if (this.config.hasProxyTo(name, up)) {
upLinks.push(self.uplinks[i]); upLinks.push(this.uplinks[up]);
} }
} }
async.map(upLinks, function(up, cb) { async.map(upLinks, (upLink, cb) => {
let _options = Object.assign({}, options);
if (Utils.is_object(packageInfo._uplinks[up.upname])) { const _options = Object.assign({}, options);
let fetched = packageInfo._uplinks[up.upname].fetched; let upLinkMeta = packageInfo._uplinks[upLink.upname];
if (fetched && fetched > (Date.now() - up.maxage)) {
if (Utils.is_object(upLinkMeta)) {
const fetched = upLinkMeta.fetched;
if (fetched && fetched > (Date.now() - upLink.maxage)) {
return cb(); return cb();
} }
_options.etag = packageInfo._uplinks[up.upname].etag; _options.etag = upLinkMeta.etag;
} }
up.getRemotePackage(name, _options, function(err, upLinkResponse, etag) { upLink.getRemoteMetadata(name, _options, (err, upLinkResponse, eTag) => {
if (err && err.status === 304) { if (err && err.status === 304) {
packageInfo._uplinks[up.upname].fetched = Date.now(); upLinkMeta.fetched = Date.now();
} }
if (err || !upLinkResponse) { if (err || !upLinkResponse) {
@ -541,8 +555,8 @@ class Storage {
return cb(null, [err]); return cb(null, [err]);
} }
packageInfo._uplinks[up.upname] = { packageInfo._uplinks[upLink.upname] = {
etag: etag, etag: eTag,
fetched: Date.now(), fetched: Date.now(),
}; };
@ -551,18 +565,7 @@ class Storage {
packageInfo['time'] = upLinkResponse.time; packageInfo['time'] = upLinkResponse.time;
} }
for (let i in upLinkResponse.versions) { this._updateVersionsHiddenUpLink(upLinkResponse.versions, upLink);
if (Object.prototype.hasOwnProperty.call(upLinkResponse.versions, i)) {
// this won't be serialized to json,
// kinda like an ES6 Symbol
Object.defineProperty(upLinkResponse.versions[i], '_verdaccio_uplink', {
value: up.upname,
enumerable: false,
configurable: false,
writable: true,
});
}
}
try { try {
Storage._merge_versions(packageInfo, upLinkResponse, self.config); Storage._merge_versions(packageInfo, upLinkResponse, self.config);
@ -579,7 +582,7 @@ class Storage {
exists = true; exists = true;
cb(); cb();
}); });
}, function(err, upLinksErrors) { }, (err, upLinksErrors) => {
assert(!err && Array.isArray(upLinksErrors)); assert(!err && Array.isArray(upLinksErrors));
if (!exists) { if (!exists) {
@ -597,6 +600,23 @@ class Storage {
}); });
} }
/**
* Set a hidden value for each version.
* @param {Array} versions list of version
* @param {String} upLink uplink name
* @private
*/
_updateVersionsHiddenUpLink(versions, upLink) {
for (let i in versions) {
if (Object.prototype.hasOwnProperty.call(versions, i)) {
const version = versions[i];
// holds a "hidden" value to be used by the package storage.
version[Symbol.for('__verdaccio_uplink')] = upLink.upname;
}
}
}
/** /**
* Set up the Up Storage for each link. * Set up the Up Storage for each link.
* @param {Object} config * @param {Object} config

49
src/lib/storage/local/local-storage.js
View file

@ -23,6 +23,7 @@ const fileExist = 'EEXISTS';
const noSuchFile = 'ENOENT'; const noSuchFile = 'ENOENT';
const resourceNotAvailable = 'EAGAIN'; const resourceNotAvailable = 'EAGAIN';
const generatePackageTemplate = function(name) { const generatePackageTemplate = function(name) {
return { return {
// standard things // standard things
@ -178,19 +179,11 @@ class LocalStorage {
url: version.dist.tarball, url: version.dist.tarball,
sha: version.dist.shasum, sha: version.dist.shasum,
}; };
// if (verdata[Symbol('_verdaccio_uplink')]) {
if (version._verdaccio_uplink) { const upLink = version[Symbol.for('__verdaccio_uplink')];
// if we got this information from a known registry,
// use the same protocol for the tarball if (_.isNil(upLink) === false) {
// hash = this._updateUplinkToRemoteProtocol(hash, upLink);
// see https://github.com/rlidwka/sinopia/issues/166
const tarballUrl = URL.parse(hash.url);
const uplinkUrl = URL.parse(this.config.uplinks[version._verdaccio_uplink].url);
if (uplinkUrl.host === tarballUrl.host) {
tarballUrl.protocol = uplinkUrl.protocol;
hash.registry = version._verdaccio_uplink;
hash.url = URL.format(tarballUrl);
}
} }
} }
} }
@ -214,6 +207,7 @@ class LocalStorage {
} }
} }
} }
if (packageInfo.readme !== packageLocalJson.readme) { if (packageInfo.readme !== packageLocalJson.readme) {
packageLocalJson.readme = packageInfo.readme; packageLocalJson.readme = packageInfo.readme;
change = true; change = true;
@ -235,6 +229,27 @@ class LocalStorage {
}); });
} }
/**
* Ensure the dist file remains as the same protocol
* @param {Object} hash metadata
* @param {String} upLink registry key
* @private
*/
_updateUplinkToRemoteProtocol(hash, upLink) {
// if we got this information from a known registry,
// use the same protocol for the tarball
//
// see https://github.com/rlidwka/sinopia/issues/166
const tarballUrl = URL.parse(hash.url);
const uplinkUrl = URL.parse(this.config.uplinks[upLink].url);
if (uplinkUrl.host === tarballUrl.host) {
tarballUrl.protocol = uplinkUrl.protocol;
hash.registry = upLink;
hash.url = URL.format(tarballUrl);
}
}
/** /**
* Add a new version to a previous local package. * Add a new version to a previous local package.
* @param {*} name * @param {*} name
@ -441,7 +456,7 @@ class LocalStorage {
uploadStream.emit('error', createError[409]('this tarball is already present')); uploadStream.emit('error', createError[409]('this tarball is already present'));
} else if (err.code === noSuchFile) { } else if (err.code === noSuchFile) {
// check if package exists to throw an appropriate message // check if package exists to throw an appropriate message
this.getPackage(name, function(_err, res) { this.getPackageMetadata(name, function(_err, res) {
if (_err) { if (_err) {
uploadStream.emit('error', _err); uploadStream.emit('error', _err);
} else { } else {
@ -541,12 +556,12 @@ class LocalStorage {
* @param {*} callback * @param {*} callback
* @return {Function} * @return {Function}
*/ */
getPackage(name, options, callback) { getPackageMetadata(name, options, callback) {
if (_.isFunction(options)) { if (_.isFunction(options)) {
callback = options || {}; callback = options || {};
} }
let storage = this.storage(name); const storage = this.storage(name);
if (!storage) { if (!storage) {
return callback( createError[404]('no such package available') ); return callback( createError[404]('no such package available') );
} }
@ -580,7 +595,7 @@ class LocalStorage {
} }
if (stats.mtime > startKey) { if (stats.mtime > startKey) {
this.getPackage(item.name, options, function(err, data) { this.getPackageMetadata(item.name, options, function(err, data) {
if (err) { if (err) {
return cb(err); return cb(err);
} }

25
src/lib/storage/up-storage.js
View file

@ -14,10 +14,12 @@ const encode = function(thing) {
return encodeURIComponent(thing).replace(/^%40/, '@'); return encodeURIComponent(thing).replace(/^%40/, '@');
}; };
const jsonContentType = 'application/json';
const contenTypeAccept = [ const contenTypeAccept = [
'application/octet-stream', 'application/octet-stream',
'application/vnd.npm.install-v1+json; q=1.0', // 'application/vnd.npm.install-v1+json; q=1.0',
'application/json; q=0.8, */*', `${jsonContentType} q=0.8, */*`,
].join(', '); ].join(', ');
/** /**
@ -94,7 +96,7 @@ class ProxyStorage {
let self = this; let self = this;
let headers = options.headers || {}; let headers = options.headers || {};
headers['Accept'] = headers['Accept'] || 'application/json'; headers['Accept'] = headers['Accept'] || contenTypeAccept;
headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip'; headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip';
// registry.npmjs.org will only return search result if user-agent include string 'npm' // registry.npmjs.org will only return search result if user-agent include string 'npm'
headers['User-Agent'] = headers['User-Agent'] || `npm (${this.userAgent})`; headers['User-Agent'] = headers['User-Agent'] || `npm (${this.userAgent})`;
@ -229,12 +231,12 @@ class ProxyStorage {
} }
/** /**
* Get a remote package. * Get a remote package metadata
* @param {*} name * @param {*} name package name
* @param {*} options * @param {*} options request options, eg: eTag.
* @param {*} callback * @param {*} callback
*/ */
getRemotePackage(name, options, callback) { getRemoteMetadata(name, options, callback) {
const headers = {}; const headers = {};
if (_.isNil(options.etag) === false) { if (_.isNil(options.etag) === false) {
headers['If-None-Match'] = options.etag; headers['If-None-Match'] = options.etag;
@ -263,16 +265,17 @@ class ProxyStorage {
} }
/** /**
* Get an url. * Fetch a tarball from the uplink.
* @param {String} url * @param {String} url
* @return {Stream} * @return {Stream}
*/ */
get_url(url) { fetchTarball(url) {
const stream = new MyStreams.ReadTarball({}); const stream = new MyStreams.ReadTarball({});
stream.abort = function() {};
let current_length = 0; let current_length = 0;
let expected_length; let expected_length;
let readStream = this.request({
stream.abort = () => {};
const readStream = this.request({
uri_full: url, uri_full: url,
encoding: null, encoding: null,
headers: { headers: {