diff --git a/src/lib/storage.js b/src/lib/storage.js index 9c1be72ca..24306817c 100644 --- a/src/lib/storage.js +++ b/src/lib/storage.js @@ -14,6 +14,16 @@ const MyStreams = require('./storage/streams'); const Proxy = require('./storage/up-storage'); const Utils = require('./utils'); +const WHITELIST = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time']; +const getDefaultMetadata = (name) => { + return { + 'name': name, + 'versions': {}, + 'dist-tags': {}, + '_uplinks': {}, + }; +}; + /** * Implements Storage interface * (same for storage.js, local-storage.js, up-storage.js). @@ -51,7 +61,7 @@ class Storage { */ const checkPackageLocal = () => { return new Promise((resolve, reject) => { - this.localStorage.getPackage(name, {}, (err, results) => { + this.localStorage.getPackageMetadata(name, {}, (err, results) => { if (!_.isNil(err) && err.status !== 404) { return reject(err); } @@ -69,7 +79,7 @@ class Storage { */ const checkPackageRemote = () => { return new Promise((resolve, reject) => { - self._sync_package_with_uplinks(name, null, {}, (err, results, err_results) => { + self._syncUplinksMetadata(name, null, {}, (err, results, err_results) => { // something weird if (err && err.status !== 404) { return reject(err); @@ -248,20 +258,20 @@ class Storage { let err404 = err; rstream.abort(); rstream = null; // gc - self.localStorage.getPackage(name, function(err, info) { - if (!err && info._distfiles && _.isNil(info._distfiles[filename]) === false) { + self.localStorage.getPackageMetadata(name, (err, info) => { + if (_.isNil(err) && info._distfiles && _.isNil(info._distfiles[filename]) === false) { // information about this file exists locally - serve_file(info._distfiles[filename]); + serveFile(info._distfiles[filename]); } else { // we know nothing about this file, trying to get information elsewhere - self._sync_package_with_uplinks(name, info, {}, function(err, info) { - if (err) { + self._syncUplinksMetadata(name, info, {}, (err, info) => { + if (_.isNil(err) === false) { return readStream.emit('error', err); } - if (!info._distfiles || _.isNil(info._distfiles[filename])) { + if (_.isNil(info._distfiles) || _.isNil(info._distfiles[filename])) { return readStream.emit('error', err404); } - serve_file(info._distfiles[filename]); + serveFile(info._distfiles[filename]); }); } }); @@ -279,7 +289,7 @@ class Storage { * Fetch and cache local/remote packages. * @param {Object} file define the package shape */ - function serve_file(file) { + function serveFile(file) { let uplink = null; for (let p in self.uplinks) { if (self.uplinks[p].isUplinkValid(file.url)) { @@ -300,7 +310,7 @@ class Storage { let on_open = function() { // prevent it from being called twice on_open = function() {}; - let rstream2 = uplink.get_url(file.url); + let rstream2 = uplink.fetchTarball(file.url); rstream2.on('error', function(err) { if (savestream) { savestream.abort(); @@ -357,21 +367,23 @@ class Storage { * @param {*} callback */ get_package(name, options, callback) { - if (typeof(options) === 'function') { + if (_.isFunction(options)) { callback = options, options = {}; } - this.localStorage.getPackage(name, options, (err, data) => { + this.localStorage.getPackageMetadata(name, options, (err, data) => { if (err && (!err.status || err.status >= 500)) { // report internal errors right away return callback(err); } - this._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) { - if (err) return callback(err); - const whitelist = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time']; + this._syncUplinksMetadata(name, data, options, function(err, result, uplink_errors) { + if (err) { + return callback(err); + } + for (let i in result) { - if (whitelist.indexOf(i) === -1) { + if (WHITELIST.indexOf(i) === -1) { delete result[i]; } } @@ -454,9 +466,10 @@ class Storage { let packages = []; const getPackage = function(i) { - self.localStorage.getPackage(locals[i], function(err, info) { - if (!err) { - let latest = info['dist-tags'].latest; + self.localStorage.getPackageMetadata(locals[i], function(err, info) { + if (_.isNil(err) === false) { + const latest = info['dist-tags'].latest; + if (latest && info.versions[latest]) { packages.push(info.versions[latest]); } else { @@ -480,7 +493,7 @@ class Storage { } /** - * Function fetches package information from uplinks and synchronizes it with local data + * Function fetches package metadata from uplinks and synchronizes it with local data if package is available locally, it MUST be provided in pkginfo returns callback(err, result, uplink_errors) * @param {*} name @@ -488,43 +501,44 @@ class Storage { * @param {*} options * @param {*} callback */ - _sync_package_with_uplinks(name, packageInfo, options, callback) { - let self = this; + _syncUplinksMetadata(name, packageInfo, options, callback) { let exists = false; - if (!packageInfo) { - exists = false; + const self = this; + const upLinks = []; - packageInfo = { - 'name': name, - 'versions': {}, - 'dist-tags': {}, - '_uplinks': {}, - }; + if (_.isNil(packageInfo)) { + exists = false; + packageInfo = getDefaultMetadata(name); } else { exists = true; } - let upLinks = []; - for (let i in self.uplinks) { - if (self.config.hasProxyTo(name, i)) { - upLinks.push(self.uplinks[i]); + + for (let up in this.uplinks) { + if (this.config.hasProxyTo(name, up)) { + upLinks.push(this.uplinks[up]); } } - async.map(upLinks, function(up, cb) { - let _options = Object.assign({}, options); - if (Utils.is_object(packageInfo._uplinks[up.upname])) { - let fetched = packageInfo._uplinks[up.upname].fetched; - if (fetched && fetched > (Date.now() - up.maxage)) { + async.map(upLinks, (upLink, cb) => { + + const _options = Object.assign({}, options); + let upLinkMeta = packageInfo._uplinks[upLink.upname]; + + if (Utils.is_object(upLinkMeta)) { + + const fetched = upLinkMeta.fetched; + + if (fetched && fetched > (Date.now() - upLink.maxage)) { return cb(); } - _options.etag = packageInfo._uplinks[up.upname].etag; + _options.etag = upLinkMeta.etag; } - up.getRemotePackage(name, _options, function(err, upLinkResponse, etag) { + upLink.getRemoteMetadata(name, _options, (err, upLinkResponse, eTag) => { if (err && err.status === 304) { - packageInfo._uplinks[up.upname].fetched = Date.now(); + upLinkMeta.fetched = Date.now(); } if (err || !upLinkResponse) { @@ -541,8 +555,8 @@ class Storage { return cb(null, [err]); } - packageInfo._uplinks[up.upname] = { - etag: etag, + packageInfo._uplinks[upLink.upname] = { + etag: eTag, fetched: Date.now(), }; @@ -551,18 +565,7 @@ class Storage { packageInfo['time'] = upLinkResponse.time; } - for (let i in upLinkResponse.versions) { - if (Object.prototype.hasOwnProperty.call(upLinkResponse.versions, i)) { - // this won't be serialized to json, - // kinda like an ES6 Symbol - Object.defineProperty(upLinkResponse.versions[i], '_verdaccio_uplink', { - value: up.upname, - enumerable: false, - configurable: false, - writable: true, - }); - } - } + this._updateVersionsHiddenUpLink(upLinkResponse.versions, upLink); try { Storage._merge_versions(packageInfo, upLinkResponse, self.config); @@ -579,7 +582,7 @@ class Storage { exists = true; cb(); }); - }, function(err, upLinksErrors) { + }, (err, upLinksErrors) => { assert(!err && Array.isArray(upLinksErrors)); if (!exists) { @@ -597,6 +600,23 @@ class Storage { }); } + /** + * Set a hidden value for each version. + * @param {Array} versions list of version + * @param {String} upLink uplink name + * @private + */ + _updateVersionsHiddenUpLink(versions, upLink) { + for (let i in versions) { + if (Object.prototype.hasOwnProperty.call(versions, i)) { + const version = versions[i]; + + // holds a "hidden" value to be used by the package storage. + version[Symbol.for('__verdaccio_uplink')] = upLink.upname; + } + } + } + /** * Set up the Up Storage for each link. * @param {Object} config diff --git a/src/lib/storage/local/local-storage.js b/src/lib/storage/local/local-storage.js index 446a3a7a3..e0dbffba5 100644 --- a/src/lib/storage/local/local-storage.js +++ b/src/lib/storage/local/local-storage.js @@ -23,6 +23,7 @@ const fileExist = 'EEXISTS'; const noSuchFile = 'ENOENT'; const resourceNotAvailable = 'EAGAIN'; + const generatePackageTemplate = function(name) { return { // standard things @@ -178,19 +179,11 @@ class LocalStorage { url: version.dist.tarball, sha: version.dist.shasum, }; - // if (verdata[Symbol('_verdaccio_uplink')]) { - if (version._verdaccio_uplink) { - // if we got this information from a known registry, - // use the same protocol for the tarball - // - // see https://github.com/rlidwka/sinopia/issues/166 - const tarballUrl = URL.parse(hash.url); - const uplinkUrl = URL.parse(this.config.uplinks[version._verdaccio_uplink].url); - if (uplinkUrl.host === tarballUrl.host) { - tarballUrl.protocol = uplinkUrl.protocol; - hash.registry = version._verdaccio_uplink; - hash.url = URL.format(tarballUrl); - } + + const upLink = version[Symbol.for('__verdaccio_uplink')]; + + if (_.isNil(upLink) === false) { + hash = this._updateUplinkToRemoteProtocol(hash, upLink); } } } @@ -214,6 +207,7 @@ class LocalStorage { } } } + if (packageInfo.readme !== packageLocalJson.readme) { packageLocalJson.readme = packageInfo.readme; change = true; @@ -235,6 +229,27 @@ class LocalStorage { }); } + /** + * Ensure the dist file remains as the same protocol + * @param {Object} hash metadata + * @param {String} upLink registry key + * @private + */ + _updateUplinkToRemoteProtocol(hash, upLink) { + // if we got this information from a known registry, + // use the same protocol for the tarball + // + // see https://github.com/rlidwka/sinopia/issues/166 + const tarballUrl = URL.parse(hash.url); + const uplinkUrl = URL.parse(this.config.uplinks[upLink].url); + + if (uplinkUrl.host === tarballUrl.host) { + tarballUrl.protocol = uplinkUrl.protocol; + hash.registry = upLink; + hash.url = URL.format(tarballUrl); + } + } + /** * Add a new version to a previous local package. * @param {*} name @@ -441,7 +456,7 @@ class LocalStorage { uploadStream.emit('error', createError[409]('this tarball is already present')); } else if (err.code === noSuchFile) { // check if package exists to throw an appropriate message - this.getPackage(name, function(_err, res) { + this.getPackageMetadata(name, function(_err, res) { if (_err) { uploadStream.emit('error', _err); } else { @@ -541,12 +556,12 @@ class LocalStorage { * @param {*} callback * @return {Function} */ - getPackage(name, options, callback) { + getPackageMetadata(name, options, callback) { if (_.isFunction(options)) { callback = options || {}; } - let storage = this.storage(name); + const storage = this.storage(name); if (!storage) { return callback( createError[404]('no such package available') ); } @@ -580,7 +595,7 @@ class LocalStorage { } if (stats.mtime > startKey) { - this.getPackage(item.name, options, function(err, data) { + this.getPackageMetadata(item.name, options, function(err, data) { if (err) { return cb(err); } diff --git a/src/lib/storage/up-storage.js b/src/lib/storage/up-storage.js index 6df0ac25f..673c029f4 100644 --- a/src/lib/storage/up-storage.js +++ b/src/lib/storage/up-storage.js @@ -14,10 +14,12 @@ const encode = function(thing) { return encodeURIComponent(thing).replace(/^%40/, '@'); }; +const jsonContentType = 'application/json'; + const contenTypeAccept = [ 'application/octet-stream', - 'application/vnd.npm.install-v1+json; q=1.0', - 'application/json; q=0.8, */*', + // 'application/vnd.npm.install-v1+json; q=1.0', + `${jsonContentType} q=0.8, */*`, ].join(', '); /** @@ -94,7 +96,7 @@ class ProxyStorage { let self = this; let headers = options.headers || {}; - headers['Accept'] = headers['Accept'] || 'application/json'; + headers['Accept'] = headers['Accept'] || contenTypeAccept; headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip'; // registry.npmjs.org will only return search result if user-agent include string 'npm' headers['User-Agent'] = headers['User-Agent'] || `npm (${this.userAgent})`; @@ -229,12 +231,12 @@ class ProxyStorage { } /** - * Get a remote package. - * @param {*} name - * @param {*} options + * Get a remote package metadata + * @param {*} name package name + * @param {*} options request options, eg: eTag. * @param {*} callback */ - getRemotePackage(name, options, callback) { + getRemoteMetadata(name, options, callback) { const headers = {}; if (_.isNil(options.etag) === false) { headers['If-None-Match'] = options.etag; @@ -263,16 +265,17 @@ class ProxyStorage { } /** - * Get an url. + * Fetch a tarball from the uplink. * @param {String} url * @return {Stream} */ - get_url(url) { + fetchTarball(url) { const stream = new MyStreams.ReadTarball({}); - stream.abort = function() {}; let current_length = 0; let expected_length; - let readStream = this.request({ + + stream.abort = () => {}; + const readStream = this.request({ uri_full: url, encoding: null, headers: {