mirror of
https://github.com/verdaccio/verdaccio.git
synced 2025-01-13 22:48:31 -05:00
refactor: storage classes clean up
This commit is contained in:
parent
14bbd93722
commit
c4555cd64e
3 changed files with 124 additions and 86 deletions
src/lib
|
@ -14,6 +14,16 @@ const MyStreams = require('./storage/streams');
|
|||
const Proxy = require('./storage/up-storage');
|
||||
const Utils = require('./utils');
|
||||
|
||||
const WHITELIST = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time'];
|
||||
const getDefaultMetadata = (name) => {
|
||||
return {
|
||||
'name': name,
|
||||
'versions': {},
|
||||
'dist-tags': {},
|
||||
'_uplinks': {},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Implements Storage interface
|
||||
* (same for storage.js, local-storage.js, up-storage.js).
|
||||
|
@ -51,7 +61,7 @@ class Storage {
|
|||
*/
|
||||
const checkPackageLocal = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.localStorage.getPackage(name, {}, (err, results) => {
|
||||
this.localStorage.getPackageMetadata(name, {}, (err, results) => {
|
||||
if (!_.isNil(err) && err.status !== 404) {
|
||||
return reject(err);
|
||||
}
|
||||
|
@ -69,7 +79,7 @@ class Storage {
|
|||
*/
|
||||
const checkPackageRemote = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
self._sync_package_with_uplinks(name, null, {}, (err, results, err_results) => {
|
||||
self._syncUplinksMetadata(name, null, {}, (err, results, err_results) => {
|
||||
// something weird
|
||||
if (err && err.status !== 404) {
|
||||
return reject(err);
|
||||
|
@ -248,20 +258,20 @@ class Storage {
|
|||
let err404 = err;
|
||||
rstream.abort();
|
||||
rstream = null; // gc
|
||||
self.localStorage.getPackage(name, function(err, info) {
|
||||
if (!err && info._distfiles && _.isNil(info._distfiles[filename]) === false) {
|
||||
self.localStorage.getPackageMetadata(name, (err, info) => {
|
||||
if (_.isNil(err) && info._distfiles && _.isNil(info._distfiles[filename]) === false) {
|
||||
// information about this file exists locally
|
||||
serve_file(info._distfiles[filename]);
|
||||
serveFile(info._distfiles[filename]);
|
||||
} else {
|
||||
// we know nothing about this file, trying to get information elsewhere
|
||||
self._sync_package_with_uplinks(name, info, {}, function(err, info) {
|
||||
if (err) {
|
||||
self._syncUplinksMetadata(name, info, {}, (err, info) => {
|
||||
if (_.isNil(err) === false) {
|
||||
return readStream.emit('error', err);
|
||||
}
|
||||
if (!info._distfiles || _.isNil(info._distfiles[filename])) {
|
||||
if (_.isNil(info._distfiles) || _.isNil(info._distfiles[filename])) {
|
||||
return readStream.emit('error', err404);
|
||||
}
|
||||
serve_file(info._distfiles[filename]);
|
||||
serveFile(info._distfiles[filename]);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
@ -279,7 +289,7 @@ class Storage {
|
|||
* Fetch and cache local/remote packages.
|
||||
* @param {Object} file define the package shape
|
||||
*/
|
||||
function serve_file(file) {
|
||||
function serveFile(file) {
|
||||
let uplink = null;
|
||||
for (let p in self.uplinks) {
|
||||
if (self.uplinks[p].isUplinkValid(file.url)) {
|
||||
|
@ -300,7 +310,7 @@ class Storage {
|
|||
let on_open = function() {
|
||||
// prevent it from being called twice
|
||||
on_open = function() {};
|
||||
let rstream2 = uplink.get_url(file.url);
|
||||
let rstream2 = uplink.fetchTarball(file.url);
|
||||
rstream2.on('error', function(err) {
|
||||
if (savestream) {
|
||||
savestream.abort();
|
||||
|
@ -357,21 +367,23 @@ class Storage {
|
|||
* @param {*} callback
|
||||
*/
|
||||
get_package(name, options, callback) {
|
||||
if (typeof(options) === 'function') {
|
||||
if (_.isFunction(options)) {
|
||||
callback = options, options = {};
|
||||
}
|
||||
|
||||
this.localStorage.getPackage(name, options, (err, data) => {
|
||||
this.localStorage.getPackageMetadata(name, options, (err, data) => {
|
||||
if (err && (!err.status || err.status >= 500)) {
|
||||
// report internal errors right away
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
this._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) {
|
||||
if (err) return callback(err);
|
||||
const whitelist = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time'];
|
||||
this._syncUplinksMetadata(name, data, options, function(err, result, uplink_errors) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
for (let i in result) {
|
||||
if (whitelist.indexOf(i) === -1) {
|
||||
if (WHITELIST.indexOf(i) === -1) {
|
||||
delete result[i];
|
||||
}
|
||||
}
|
||||
|
@ -454,9 +466,10 @@ class Storage {
|
|||
let packages = [];
|
||||
|
||||
const getPackage = function(i) {
|
||||
self.localStorage.getPackage(locals[i], function(err, info) {
|
||||
if (!err) {
|
||||
let latest = info['dist-tags'].latest;
|
||||
self.localStorage.getPackageMetadata(locals[i], function(err, info) {
|
||||
if (_.isNil(err) === false) {
|
||||
const latest = info['dist-tags'].latest;
|
||||
|
||||
if (latest && info.versions[latest]) {
|
||||
packages.push(info.versions[latest]);
|
||||
} else {
|
||||
|
@ -480,7 +493,7 @@ class Storage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Function fetches package information from uplinks and synchronizes it with local data
|
||||
* Function fetches package metadata from uplinks and synchronizes it with local data
|
||||
if package is available locally, it MUST be provided in pkginfo
|
||||
returns callback(err, result, uplink_errors)
|
||||
* @param {*} name
|
||||
|
@ -488,43 +501,44 @@ class Storage {
|
|||
* @param {*} options
|
||||
* @param {*} callback
|
||||
*/
|
||||
_sync_package_with_uplinks(name, packageInfo, options, callback) {
|
||||
let self = this;
|
||||
_syncUplinksMetadata(name, packageInfo, options, callback) {
|
||||
let exists = false;
|
||||
if (!packageInfo) {
|
||||
exists = false;
|
||||
const self = this;
|
||||
const upLinks = [];
|
||||
|
||||
packageInfo = {
|
||||
'name': name,
|
||||
'versions': {},
|
||||
'dist-tags': {},
|
||||
'_uplinks': {},
|
||||
};
|
||||
if (_.isNil(packageInfo)) {
|
||||
exists = false;
|
||||
packageInfo = getDefaultMetadata(name);
|
||||
} else {
|
||||
exists = true;
|
||||
}
|
||||
|
||||
let upLinks = [];
|
||||
for (let i in self.uplinks) {
|
||||
if (self.config.hasProxyTo(name, i)) {
|
||||
upLinks.push(self.uplinks[i]);
|
||||
|
||||
for (let up in this.uplinks) {
|
||||
if (this.config.hasProxyTo(name, up)) {
|
||||
upLinks.push(this.uplinks[up]);
|
||||
}
|
||||
}
|
||||
|
||||
async.map(upLinks, function(up, cb) {
|
||||
let _options = Object.assign({}, options);
|
||||
if (Utils.is_object(packageInfo._uplinks[up.upname])) {
|
||||
let fetched = packageInfo._uplinks[up.upname].fetched;
|
||||
if (fetched && fetched > (Date.now() - up.maxage)) {
|
||||
async.map(upLinks, (upLink, cb) => {
|
||||
|
||||
const _options = Object.assign({}, options);
|
||||
let upLinkMeta = packageInfo._uplinks[upLink.upname];
|
||||
|
||||
if (Utils.is_object(upLinkMeta)) {
|
||||
|
||||
const fetched = upLinkMeta.fetched;
|
||||
|
||||
if (fetched && fetched > (Date.now() - upLink.maxage)) {
|
||||
return cb();
|
||||
}
|
||||
|
||||
_options.etag = packageInfo._uplinks[up.upname].etag;
|
||||
_options.etag = upLinkMeta.etag;
|
||||
}
|
||||
|
||||
up.getRemotePackage(name, _options, function(err, upLinkResponse, etag) {
|
||||
upLink.getRemoteMetadata(name, _options, (err, upLinkResponse, eTag) => {
|
||||
if (err && err.status === 304) {
|
||||
packageInfo._uplinks[up.upname].fetched = Date.now();
|
||||
upLinkMeta.fetched = Date.now();
|
||||
}
|
||||
|
||||
if (err || !upLinkResponse) {
|
||||
|
@ -541,8 +555,8 @@ class Storage {
|
|||
return cb(null, [err]);
|
||||
}
|
||||
|
||||
packageInfo._uplinks[up.upname] = {
|
||||
etag: etag,
|
||||
packageInfo._uplinks[upLink.upname] = {
|
||||
etag: eTag,
|
||||
fetched: Date.now(),
|
||||
};
|
||||
|
||||
|
@ -551,18 +565,7 @@ class Storage {
|
|||
packageInfo['time'] = upLinkResponse.time;
|
||||
}
|
||||
|
||||
for (let i in upLinkResponse.versions) {
|
||||
if (Object.prototype.hasOwnProperty.call(upLinkResponse.versions, i)) {
|
||||
// this won't be serialized to json,
|
||||
// kinda like an ES6 Symbol
|
||||
Object.defineProperty(upLinkResponse.versions[i], '_verdaccio_uplink', {
|
||||
value: up.upname,
|
||||
enumerable: false,
|
||||
configurable: false,
|
||||
writable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
this._updateVersionsHiddenUpLink(upLinkResponse.versions, upLink);
|
||||
|
||||
try {
|
||||
Storage._merge_versions(packageInfo, upLinkResponse, self.config);
|
||||
|
@ -579,7 +582,7 @@ class Storage {
|
|||
exists = true;
|
||||
cb();
|
||||
});
|
||||
}, function(err, upLinksErrors) {
|
||||
}, (err, upLinksErrors) => {
|
||||
assert(!err && Array.isArray(upLinksErrors));
|
||||
|
||||
if (!exists) {
|
||||
|
@ -597,6 +600,23 @@ class Storage {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a hidden value for each version.
|
||||
* @param {Array} versions list of version
|
||||
* @param {String} upLink uplink name
|
||||
* @private
|
||||
*/
|
||||
_updateVersionsHiddenUpLink(versions, upLink) {
|
||||
for (let i in versions) {
|
||||
if (Object.prototype.hasOwnProperty.call(versions, i)) {
|
||||
const version = versions[i];
|
||||
|
||||
// holds a "hidden" value to be used by the package storage.
|
||||
version[Symbol.for('__verdaccio_uplink')] = upLink.upname;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the Up Storage for each link.
|
||||
* @param {Object} config
|
||||
|
|
|
@ -23,6 +23,7 @@ const fileExist = 'EEXISTS';
|
|||
const noSuchFile = 'ENOENT';
|
||||
const resourceNotAvailable = 'EAGAIN';
|
||||
|
||||
|
||||
const generatePackageTemplate = function(name) {
|
||||
return {
|
||||
// standard things
|
||||
|
@ -178,19 +179,11 @@ class LocalStorage {
|
|||
url: version.dist.tarball,
|
||||
sha: version.dist.shasum,
|
||||
};
|
||||
// if (verdata[Symbol('_verdaccio_uplink')]) {
|
||||
if (version._verdaccio_uplink) {
|
||||
// if we got this information from a known registry,
|
||||
// use the same protocol for the tarball
|
||||
//
|
||||
// see https://github.com/rlidwka/sinopia/issues/166
|
||||
const tarballUrl = URL.parse(hash.url);
|
||||
const uplinkUrl = URL.parse(this.config.uplinks[version._verdaccio_uplink].url);
|
||||
if (uplinkUrl.host === tarballUrl.host) {
|
||||
tarballUrl.protocol = uplinkUrl.protocol;
|
||||
hash.registry = version._verdaccio_uplink;
|
||||
hash.url = URL.format(tarballUrl);
|
||||
}
|
||||
|
||||
const upLink = version[Symbol.for('__verdaccio_uplink')];
|
||||
|
||||
if (_.isNil(upLink) === false) {
|
||||
hash = this._updateUplinkToRemoteProtocol(hash, upLink);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -214,6 +207,7 @@ class LocalStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (packageInfo.readme !== packageLocalJson.readme) {
|
||||
packageLocalJson.readme = packageInfo.readme;
|
||||
change = true;
|
||||
|
@ -235,6 +229,27 @@ class LocalStorage {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the dist file remains as the same protocol
|
||||
* @param {Object} hash metadata
|
||||
* @param {String} upLink registry key
|
||||
* @private
|
||||
*/
|
||||
_updateUplinkToRemoteProtocol(hash, upLink) {
|
||||
// if we got this information from a known registry,
|
||||
// use the same protocol for the tarball
|
||||
//
|
||||
// see https://github.com/rlidwka/sinopia/issues/166
|
||||
const tarballUrl = URL.parse(hash.url);
|
||||
const uplinkUrl = URL.parse(this.config.uplinks[upLink].url);
|
||||
|
||||
if (uplinkUrl.host === tarballUrl.host) {
|
||||
tarballUrl.protocol = uplinkUrl.protocol;
|
||||
hash.registry = upLink;
|
||||
hash.url = URL.format(tarballUrl);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new version to a previous local package.
|
||||
* @param {*} name
|
||||
|
@ -441,7 +456,7 @@ class LocalStorage {
|
|||
uploadStream.emit('error', createError[409]('this tarball is already present'));
|
||||
} else if (err.code === noSuchFile) {
|
||||
// check if package exists to throw an appropriate message
|
||||
this.getPackage(name, function(_err, res) {
|
||||
this.getPackageMetadata(name, function(_err, res) {
|
||||
if (_err) {
|
||||
uploadStream.emit('error', _err);
|
||||
} else {
|
||||
|
@ -541,12 +556,12 @@ class LocalStorage {
|
|||
* @param {*} callback
|
||||
* @return {Function}
|
||||
*/
|
||||
getPackage(name, options, callback) {
|
||||
getPackageMetadata(name, options, callback) {
|
||||
if (_.isFunction(options)) {
|
||||
callback = options || {};
|
||||
}
|
||||
|
||||
let storage = this.storage(name);
|
||||
const storage = this.storage(name);
|
||||
if (!storage) {
|
||||
return callback( createError[404]('no such package available') );
|
||||
}
|
||||
|
@ -580,7 +595,7 @@ class LocalStorage {
|
|||
}
|
||||
|
||||
if (stats.mtime > startKey) {
|
||||
this.getPackage(item.name, options, function(err, data) {
|
||||
this.getPackageMetadata(item.name, options, function(err, data) {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
|
|
@ -14,10 +14,12 @@ const encode = function(thing) {
|
|||
return encodeURIComponent(thing).replace(/^%40/, '@');
|
||||
};
|
||||
|
||||
const jsonContentType = 'application/json';
|
||||
|
||||
const contenTypeAccept = [
|
||||
'application/octet-stream',
|
||||
'application/vnd.npm.install-v1+json; q=1.0',
|
||||
'application/json; q=0.8, */*',
|
||||
// 'application/vnd.npm.install-v1+json; q=1.0',
|
||||
`${jsonContentType} q=0.8, */*`,
|
||||
].join(', ');
|
||||
|
||||
/**
|
||||
|
@ -94,7 +96,7 @@ class ProxyStorage {
|
|||
|
||||
let self = this;
|
||||
let headers = options.headers || {};
|
||||
headers['Accept'] = headers['Accept'] || 'application/json';
|
||||
headers['Accept'] = headers['Accept'] || contenTypeAccept;
|
||||
headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip';
|
||||
// registry.npmjs.org will only return search result if user-agent include string 'npm'
|
||||
headers['User-Agent'] = headers['User-Agent'] || `npm (${this.userAgent})`;
|
||||
|
@ -229,12 +231,12 @@ class ProxyStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get a remote package.
|
||||
* @param {*} name
|
||||
* @param {*} options
|
||||
* Get a remote package metadata
|
||||
* @param {*} name package name
|
||||
* @param {*} options request options, eg: eTag.
|
||||
* @param {*} callback
|
||||
*/
|
||||
getRemotePackage(name, options, callback) {
|
||||
getRemoteMetadata(name, options, callback) {
|
||||
const headers = {};
|
||||
if (_.isNil(options.etag) === false) {
|
||||
headers['If-None-Match'] = options.etag;
|
||||
|
@ -263,16 +265,17 @@ class ProxyStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get an url.
|
||||
* Fetch a tarball from the uplink.
|
||||
* @param {String} url
|
||||
* @return {Stream}
|
||||
*/
|
||||
get_url(url) {
|
||||
fetchTarball(url) {
|
||||
const stream = new MyStreams.ReadTarball({});
|
||||
stream.abort = function() {};
|
||||
let current_length = 0;
|
||||
let expected_length;
|
||||
let readStream = this.request({
|
||||
|
||||
stream.abort = () => {};
|
||||
const readStream = this.request({
|
||||
uri_full: url,
|
||||
encoding: null,
|
||||
headers: {
|
||||
|
|
Loading…
Reference in a new issue