Infrastructure/verdaccio
0
Fork 0
mirror of https://github.com/verdaccio/verdaccio.git synced 2025-02-17 23:45:29 -05:00
Code Issues Activity

Migrate storages to classes

Browse source
This commit is contained in:
Juan Picado 2017-04-19 20:45:48 +02:00
parent e267ef1f54
commit 20f9436672
No known key found for this signature in database
GPG key ID: 18AC54485952D158
4 changed files with 1224 additions and 1081 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
lib/index.js
View file

@ -11,10 +11,10 @@ var Storage = require('./storage')
module.exports = function(config_hash) { module.exports = function(config_hash) {
Logger.setup(config_hash.logs) Logger.setup(config_hash.logs)
var config = Config(config_hash) var config = Config(config_hash);
var storage = Storage(config) var storage = new Storage(config);
var auth = Auth(config) var auth = Auth(config);
var app = express() var app = express();
// run in production mode by default, just in case // run in production mode by default, just in case
// it shouldn't make any difference anyway // it shouldn't make any difference anyway

288
lib/local-storage.js
View file

@ -1,30 +1,19 @@
"use strict"; "use strict";
var assert = require('assert') const assert = require('assert');
var async = require('async') const async = require('async');
var Crypto = require('crypto') const Crypto = require('crypto');
var fs = require('fs') const fs = require('fs');
var Error = require('http-errors') const Error = require('http-errors');
var Path = require('path') const Path = require('path');
var Stream = require('readable-stream') const Stream = require('readable-stream');
var URL = require('url') const URL = require('url');
var fs_storage = require('./local-fs') const fs_storage = require('./local-fs');
var Logger = require('./logger') const Logger = require('./logger');
var Search = require('./search') const Search = require('./search');
var MyStreams = require('./streams') const MyStreams = require('./streams');
var Utils = require('./utils') const Utils = require('./utils');
var info_file = 'package.json' const info_file = 'package.json';
//
// Implements Storage interface
// (same for storage.js, local-storage.js, up-storage.js)
//
class Storage {
constructor(config) {
this.config = config
this.logger = Logger.logger.child({ sub: 'fs' })
}
}
// returns the minimal package file // returns the minimal package file
function get_boilerplate(name) { function get_boilerplate(name) {
@ -39,15 +28,37 @@ function get_boilerplate(name) {
'_attachments': {}, '_attachments': {},
'_uplinks': {}, '_uplinks': {},
} }
} };
Storage.prototype._internal_error = function(err, file, message) { //
// Implements Storage interface
// (same for storage.js, local-storage.js, up-storage.js)
//
class Storage {
constructor(config) {
this.config = config
this.logger = Logger.logger.child({ sub: 'fs' })
}
/**
*
* @param {*} err
* @param {*} file
* @param {*} message
*/
_internal_error(err, file, message) {
this.logger.error( { err: err, file: file } this.logger.error( { err: err, file: file }
, message + ' @{file}: @{!err.message}' ) , message + ' @{file}: @{!err.message}' )
return Error[500]() return Error[500]()
} }
Storage.prototype.add_package = function(name, info, callback) { /**
*
* @param {*} name
* @param {*} info
* @param {*} callback
*/
add_package(name, info, callback) {
var storage = this.storage(name) var storage = this.storage(name)
if (!storage) return callback( Error[404]('this package cannot be added') ) if (!storage) return callback( Error[404]('this package cannot be added') )
@ -62,9 +73,14 @@ Storage.prototype.add_package = function(name, info, callback) {
} }
callback() callback()
}) })
} }
Storage.prototype.remove_package = function(name, callback) { /**
*
* @param {*} name
* @param {*} callback
*/
remove_package(name, callback) {
this.logger.info( { name: name } this.logger.info( { name: name }
, 'unpublishing @{name} (all)') , 'unpublishing @{name} (all)')
@ -106,9 +122,14 @@ Storage.prototype.remove_package = function(name, callback) {
Search.remove(name) Search.remove(name)
this.config.localList.remove(name) this.config.localList.remove(name)
} }
Storage.prototype._read_create_package = function(name, callback) { /**
*
* @param {*} name
* @param {*} callback
*/
_read_create_package(name, callback) {
var storage = this.storage(name) var storage = this.storage(name)
if (!storage) { if (!storage) {
var data = get_boilerplate(name) var data = get_boilerplate(name)
@ -128,11 +149,15 @@ Storage.prototype._read_create_package = function(name, callback) {
this._normalize_package(data) this._normalize_package(data)
callback(null, data) callback(null, data)
}) })
} }
// synchronize remote package info with the local one /**
// TODO: readfile called twice * Synchronize remote package info with the local one
Storage.prototype.update_versions = function(name, newdata, callback) { * @param {*} name
* @param {*} newdata
* @param {*} callback
*/
update_versions(name, newdata, callback) {
this._read_create_package(name, (err, data) => { this._read_create_package(name, (err, data) => {
if (err) return callback(err) if (err) return callback(err)
@ -156,7 +181,7 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
url: verdata.dist.tarball, url: verdata.dist.tarball,
sha: verdata.dist.shasum, sha: verdata.dist.shasum,
} }
// if (verdata[Symbol('_verdaccio_uplink')]) {
if (verdata._verdaccio_uplink) { if (verdata._verdaccio_uplink) {
// if we got this information from a known registry, // if we got this information from a known registry,
// use the same protocol for the tarball // use the same protocol for the tarball
@ -204,9 +229,17 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
callback(null, data) callback(null, data)
} }
}) })
} }
Storage.prototype.add_version = function(name, version, metadata, tag, callback) { /**
*
* @param {*} name
* @param {*} version
* @param {*} metadata
* @param {*} tag
* @param {*} callback
*/
add_version(name, version, metadata, tag, callback) {
this.update_package(name, (data, cb) => { this.update_package(name, (data, cb) => {
// keep only one readme per package // keep only one readme per package
data.readme = metadata.readme data.readme = metadata.readme
@ -227,7 +260,6 @@ Storage.prototype.add_version = function(name, version, metadata, tag, callback)
+ ' != ' + metadata.dist.shasum) ) + ' != ' + metadata.dist.shasum) )
} }
} }
data._attachments[tarball].version = version data._attachments[tarball].version = version
} }
} }
@ -237,9 +269,15 @@ Storage.prototype.add_version = function(name, version, metadata, tag, callback)
this.config.localList.add(name) this.config.localList.add(name)
cb() cb()
}, callback) }, callback)
} }
Storage.prototype.merge_tags = function(name, tags, callback) { /**
*
* @param {*} name
* @param {*} tags
* @param {*} callback
*/
merge_tags(name, tags, callback) {
this.update_package(name, function updater(data, cb) { this.update_package(name, function updater(data, cb) {
for (let t in tags) { for (let t in tags) {
if (tags[t] === null) { if (tags[t] === null) {
@ -255,9 +293,15 @@ Storage.prototype.merge_tags = function(name, tags, callback) {
} }
cb() cb()
}, callback) }, callback)
} }
Storage.prototype.replace_tags = function(name, tags, callback) { /**
*
* @param {*} name
* @param {*} tags
* @param {*} callback
*/
replace_tags(name, tags, callback) {
this.update_package(name, function updater(data, cb) { this.update_package(name, function updater(data, cb) {
data['dist-tags'] = {} data['dist-tags'] = {}
@ -275,10 +319,16 @@ Storage.prototype.replace_tags = function(name, tags, callback) {
} }
cb() cb()
}, callback) }, callback)
} }
// currently supports unpublishing only /**
Storage.prototype.change_package = function(name, metadata, revision, callback) { * Currently supports unpublishing only
* @param {*} name
* @param {*} metadata
* @param {*} revision
* @param {*} callback
*/
change_package(name, metadata, revision, callback) {
if (!Utils.is_object(metadata.versions) || !Utils.is_object(metadata['dist-tags'])) { if (!Utils.is_object(metadata.versions) || !Utils.is_object(metadata['dist-tags'])) {
return callback( Error[422]('bad data') ) return callback( Error[422]('bad data') )
@ -304,9 +354,16 @@ Storage.prototype.change_package = function(name, metadata, revision, callback)
if (err) return callback(err) if (err) return callback(err)
callback() callback()
}) })
} }
Storage.prototype.remove_tarball = function(name, filename, revision, callback) { /**
*
* @param {*} name
* @param {*} filename
* @param {*} revision
* @param {*} callback
*/
remove_tarball(name, filename, revision, callback) {
assert(Utils.validate_name(filename)) assert(Utils.validate_name(filename))
this.update_package(name, (data, cb) => { this.update_package(name, (data, cb) => {
@ -321,9 +378,14 @@ Storage.prototype.remove_tarball = function(name, filename, revision, callback)
var storage = this.storage(name) var storage = this.storage(name)
if (storage) storage.unlink(filename, callback) if (storage) storage.unlink(filename, callback)
}) })
} }
Storage.prototype.add_tarball = function(name, filename) { /**
*
* @param {*} name
* @param {*} filename
*/
add_tarball(name, filename) {
assert(Utils.validate_name(filename)) assert(Utils.validate_name(filename))
var stream = MyStreams.UploadTarballStream() var stream = MyStreams.UploadTarballStream()
@ -405,9 +467,15 @@ Storage.prototype.add_tarball = function(name, filename) {
stream.pipe(wstream) stream.pipe(wstream)
return stream return stream
} }
Storage.prototype.get_tarball = function(name, filename, callback) { /**
*
* @param {*} name
* @param {*} filename
* @param {*} callback
*/
get_tarball(name, filename, callback) {
assert(Utils.validate_name(filename)) assert(Utils.validate_name(filename))
var self = this var self = this
@ -441,9 +509,15 @@ Storage.prototype.get_tarball = function(name, filename, callback) {
rstream.pipe(stream) rstream.pipe(stream)
}) })
return stream return stream
} }
Storage.prototype.get_package = function(name, options, callback) { /**
*
* @param {*} name
* @param {*} options
* @param {*} callback
*/
get_package(name, options, callback) {
if (typeof(options) === 'function') { if (typeof(options) === 'function') {
callback = options, options = {}; callback = options, options = {};
} }
@ -462,10 +536,14 @@ Storage.prototype.get_package = function(name, options, callback) {
this._normalize_package(result) this._normalize_package(result)
callback(err, result) callback(err, result)
}) })
} }
// walks through each package and calls `on_package` on them /**
Storage.prototype._each_package = function (on_package, on_end) { * Walks through each package and calls `on_package` on them
* @param {*} on_package
* @param {*} on_end
*/
_each_package(on_package, on_end) {
var storages = {} var storages = {}
storages[this.config.storage] = true; storages[this.config.storage] = true;
@ -477,7 +555,6 @@ Storage.prototype._each_package = function (on_package, on_end) {
} }
}) })
} }
const base = Path.dirname(this.config.self_path); const base = Path.dirname(this.config.self_path);
async.eachSeries(Object.keys(storages), function (storage, cb) { async.eachSeries(Object.keys(storages), function (storage, cb) {
@ -511,26 +588,23 @@ Storage.prototype._each_package = function (on_package, on_end) {
} }
}, cb) }, cb)
}) })
}, on_end) }, on_end);
} }
// /**
// This function allows to update the package thread-safely * This function allows to update the package thread-safely
// Algorithm:
// Arguments: 1. lock package.json for writing
// - name - package name 2. read package.json
// - updateFn - function(package, cb) - update function 3. updateFn(pkg, cb), and wait for cb
// - callback - callback that gets invoked after it's all updated 4. write package.json.tmp
// 5. move package.json.tmp package.json
// Algorithm: 6. callback(err?)
// 1. lock package.json for writing * @param {*} name package name
// 2. read package.json * @param {*} updateFn function(package, cb) - update function
// 3. updateFn(pkg, cb), and wait for cb * @param {*} _callback callback that gets invoked after it's all updated
// 4. write package.json.tmp */
// 5. move package.json.tmp package.json update_package(name, updateFn, _callback) {
// 6. callback(err?)
//
Storage.prototype.update_package = function(name, updateFn, _callback) {
var self = this var self = this
var storage = self.storage(name) var storage = self.storage(name)
if (!storage) return _callback( Error[404]('no such package available') ) if (!storage) return _callback( Error[404]('no such package available') )
@ -571,18 +645,27 @@ Storage.prototype.update_package = function(name, updateFn, _callback) {
self._write_package(name, json, callback) self._write_package(name, json, callback)
}) })
}) })
} }
Storage.prototype.search = function(startkey, options) { /**
var stream = new Stream.PassThrough({ objectMode: true }) *
* @param {*} startkey
* @param {*} options
*/
search(startkey, options) {
const stream = new Stream.PassThrough({ objectMode: true });
this._each_package((item, cb) => { this._each_package((item, cb) => {
fs.stat(item.path, (err, stats) => { fs.stat(item.path, (err, stats) => {
if (err) return cb(err) if (err) {
return cb(err);
}
if (stats.mtime > startkey) { if (stats.mtime > startkey) {
this.get_package(item.name, options, function(err, data) { this.get_package(item.name, options, function(err, data) {
if (err) return cb(err) if (err) {
return cb(err);
}
var versions = Utils.semver_sort(Object.keys(data.versions)) var versions = Utils.semver_sort(Object.keys(data.versions))
var latest = data['dist-tags'] && data['dist-tags'].latest ? data['dist-tags'].latest : versions.pop() var latest = data['dist-tags'] && data['dist-tags'].latest ? data['dist-tags'].latest : versions.pop()
@ -601,7 +684,9 @@ Storage.prototype.search = function(startkey, options) {
keywords : data.versions[latest].keywords, keywords : data.versions[latest].keywords,
bugs : data.versions[latest].bugs, bugs : data.versions[latest].bugs,
license : data.versions[latest].license, license : data.versions[latest].license,
time : { modified: item.time ? new Date(item.time).toISOString() : undefined }, time : {
modified: item.time ? new Date(item.time).toISOString() : undefined
},
versions : {}, versions : {},
}) })
} }
@ -618,9 +703,13 @@ Storage.prototype.search = function(startkey, options) {
}) })
return stream return stream
} }
Storage.prototype._normalize_package = function(pkg) { /**
*
* @param {*} pkg
*/
_normalize_package(pkg) {
;['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks'].forEach(function(key) { ;['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks'].forEach(function(key) {
if (!Utils.is_object(pkg[key])) pkg[key] = {} if (!Utils.is_object(pkg[key])) pkg[key] = {}
}) })
@ -629,9 +718,15 @@ Storage.prototype._normalize_package = function(pkg) {
} }
// normalize dist-tags // normalize dist-tags
Utils.normalize_dist_tags(pkg) Utils.normalize_dist_tags(pkg)
} }
Storage.prototype._write_package = function(name, json, callback) { /**
*
* @param {*} name
* @param {*} json
* @param {*} callback
*/
_write_package(name, json, callback) {
// calculate revision a la couchdb // calculate revision a la couchdb
if (typeof(json._rev) !== 'string') { if (typeof(json._rev) !== 'string') {
@ -643,11 +738,17 @@ Storage.prototype._write_package = function(name, json, callback) {
var storage = this.storage(name) var storage = this.storage(name)
if (!storage) return callback() if (!storage) return callback()
storage.write_json(info_file, json, callback) storage.write_json(info_file, json, callback)
} }
Storage.prototype.storage = function(pkg) { /**
var path = this.config.get_package_spec(pkg).storage *
if (path == null) path = this.config.storage * @param {*} pkg
*/
storage(pkg) {
let path = this.config.get_package_spec(pkg).storage
if (path == null) {
path = this.config.storage
}
if (path == null || path === false) { if (path == null || path === false) {
this.logger.debug( { name: pkg } this.logger.debug( { name: pkg }
, 'this package has no storage defined: @{name}' ) , 'this package has no storage defined: @{name}' )
@ -659,6 +760,7 @@ Storage.prototype.storage = function(pkg) {
pkg pkg
) )
) )
}
} }
var Path_Wrapper = (function() { var Path_Wrapper = (function() {

344
lib/storage.js
View file

@ -1,47 +1,50 @@
"use strict"; "use strict";
var assert = require('assert') const assert = require('assert')
var async = require('async') const async = require('async')
var Error = require('http-errors') const Error = require('http-errors')
var Stream = require('stream') const Stream = require('stream')
var Local = require('./local-storage') const Local = require('./local-storage')
var Logger = require('./logger') const Logger = require('./logger')
var MyStreams = require('./streams') const MyStreams = require('./streams')
var Proxy = require('./up-storage') const Proxy = require('./up-storage')
var Utils = require('./utils') const Utils = require('./utils')
module.exports = Storage
// //
// Implements Storage interface // Implements Storage interface
// (same for storage.js, local-storage.js, up-storage.js) // (same for storage.js, local-storage.js, up-storage.js)
// //
function Storage(config) { class Storage {
var self = Object.create(Storage.prototype)
self.config = config
/**
*
* @param {*} config
*/
constructor(config) {
this.config = config
// we support a number of uplinks, but only one local storage // we support a number of uplinks, but only one local storage
// Proxy and Local classes should have similar API interfaces // Proxy and Local classes should have similar API interfaces
self.uplinks = {} this.uplinks = {}
for (var p in config.uplinks) { for (let p in config.uplinks) {
self.uplinks[p] = Proxy(config.uplinks[p], config) // instance for each up-link definition
self.uplinks[p].upname = p this.uplinks[p] = new Proxy(config.uplinks[p], config)
this.uplinks[p].upname = p
}
// an instance for local storage
this.local = new Local(config)
this.logger = Logger.logger.child();
} }
self.local = new Local(config)
self.logger = Logger.logger.child()
return self /**
} * Add a {name} package to a system
Function checks if package with the same name is available from uplinks.
// If it isn't, we create package locally
// Add a {name} package to a system Used storages: local (write) && uplinks
// * @param {*} name
// Function checks if package with the same name is available from uplinks. * @param {*} metadata
// If it isn't, we create package locally * @param {*} callback
// */
// Used storages: local (write) && uplinks add_package(name, metadata, callback) {
//
Storage.prototype.add_package = function(name, metadata, callback) {
var self = this var self = this
// NOTE: // NOTE:
@ -97,93 +100,103 @@ Storage.prototype.add_package = function(name, metadata, callback) {
function publish_package(cb) { function publish_package(cb) {
self.local.add_package(name, metadata, callback) self.local.add_package(name, metadata, callback)
} }
} }
// /**
// Add a new version of package {name} to a system * Add a new version of package {name} to a system
// Used storages: local (write)
// Used storages: local (write) * @param {*} name
// * @param {*} version
Storage.prototype.add_version = function(name, version, metadata, tag, callback) { * @param {*} metadata
* @param {*} tag
* @param {*} callback
*/
add_version(name, version, metadata, tag, callback) {
return this.local.add_version(name, version, metadata, tag, callback) return this.local.add_version(name, version, metadata, tag, callback)
} }
// /**
// Tags a package version with a provided tag * Tags a package version with a provided tag
// Used storages: local (write)
// Used storages: local (write) * @param {*} name
// * @param {*} tag_hash
Storage.prototype.merge_tags = function(name, tag_hash, callback) { * @param {*} callback
*/
merge_tags(name, tag_hash, callback) {
return this.local.merge_tags(name, tag_hash, callback) return this.local.merge_tags(name, tag_hash, callback)
} }
// /**
// Tags a package version with a provided tag * Tags a package version with a provided tag
// Used storages: local (write)
// Used storages: local (write) * @param {*} name
// * @param {*} tag_hash
Storage.prototype.replace_tags = function(name, tag_hash, callback) { * @param {*} callback
*/
replace_tags(name, tag_hash, callback) {
return this.local.replace_tags(name, tag_hash, callback) return this.local.replace_tags(name, tag_hash, callback)
} }
// /**
// Change an existing package (i.e. unpublish one version) * Change an existing package (i.e. unpublish one version)
// Function changes a package info from local storage and all uplinks with write access./
// Function changes a package info from local storage and all uplinks with Used storages: local (write)
// write access. * @param {*} name
// * @param {*} metadata
// Used storages: local (write) * @param {*} revision
// * @param {*} callback
Storage.prototype.change_package = function(name, metadata, revision, callback) { */
change_package(name, metadata, revision, callback) {
return this.local.change_package(name, metadata, revision, callback) return this.local.change_package(name, metadata, revision, callback)
} }
// /**
// Remove a package from a system * Remove a package from a system
// Function removes a package from local storage
// Function removes a package from local storage Used storages: local (write)
// * @param {*} name
// Used storages: local (write) * @param {*} callback
// */
Storage.prototype.remove_package = function(name, callback) { remove_package(name, callback) {
return this.local.remove_package(name, callback) return this.local.remove_package(name, callback)
} }
// /**
// Remove a tarball from a system Remove a tarball from a system
// Function removes a tarball from local storage.
// Function removes a tarball from local storage. Tarball in question should not be linked to in any existing
// Tarball in question should not be linked to in any existing versions, i.e. package version should be unpublished first.
// versions, i.e. package version should be unpublished first. Used storages: local (write)
// * @param {*} name
// Used storages: local (write) * @param {*} filename
// * @param {*} revision
Storage.prototype.remove_tarball = function(name, filename, revision, callback) { * @param {*} callback
*/
remove_tarball(name, filename, revision, callback) {
return this.local.remove_tarball(name, filename, revision, callback) return this.local.remove_tarball(name, filename, revision, callback)
} }
// /**
// Upload a tarball for {name} package * Upload a tarball for {name} package
// Function is syncronous and returns a WritableStream
// Function is syncronous and returns a WritableStream Used storages: local (write)
// * @param {*} name
// Used storages: local (write) * @param {*} filename
// */
Storage.prototype.add_tarball = function(name, filename) { add_tarball(name, filename) {
return this.local.add_tarball(name, filename) return this.local.add_tarball(name, filename)
} }
// /**
// Get a tarball from a storage for {name} package Get a tarball from a storage for {name} package
// Function is syncronous and returns a ReadableStream
// Function is syncronous and returns a ReadableStream Function tries to read tarball locally, if it fails then it reads package
// information in order to figure out where we can get this tarball from
// Function tries to read tarball locally, if it fails then it reads package Used storages: local || uplink (just one)
// information in order to figure out where we can get this tarball from * @param {*} name
// * @param {*} filename
// Used storages: local || uplink (just one) */
// get_tarball(name, filename) {
Storage.prototype.get_tarball = function(name, filename) {
var stream = MyStreams.ReadTarballStream() var stream = MyStreams.ReadTarballStream()
stream.abort = function() {} stream.abort = function() {}
@ -280,31 +293,32 @@ Storage.prototype.get_tarball = function(name, filename) {
on_open() on_open()
}) })
} }
} }
// /**
// Retrieve a package metadata for {name} package Retrieve a package metadata for {name} package
// Function invokes local.get_package and uplink.get_package for every
// Function invokes local.get_package and uplink.get_package for every uplink with proxy_access rights against {name} and combines results
// uplink with proxy_access rights against {name} and combines results into one json object
// into one json object Used storages: local && uplink (proxy_access)
// * @param {*} name
// Used storages: local && uplink (proxy_access) * @param {*} options
// * @param {*} callback
Storage.prototype.get_package = function(name, options, callback) { */
if (typeof(options) === 'function') callback = options, options = {} get_package(name, options, callback) {
if (typeof(options) === 'function') {
callback = options, options = {};
}
var self = this this.local.get_package(name, options, (err, data) => {
self.local.get_package(name, options, function(err, data) {
if (err && (!err.status || err.status >= 500)) { if (err && (!err.status || err.status >= 500)) {
// report internal errors right away // report internal errors right away
return callback(err) return callback(err)
} }
self._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) { this._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) {
if (err) return callback(err) if (err) return callback(err)
var whitelist = [ '_rev', 'name', 'versions', 'dist-tags', 'readme' ] const whitelist = [ '_rev', 'name', 'versions', 'dist-tags', 'readme' ]
for (var i in result) { for (var i in result) {
if (whitelist.indexOf(i) === -1) delete result[i] if (whitelist.indexOf(i) === -1) delete result[i]
} }
@ -317,26 +331,25 @@ Storage.prototype.get_package = function(name, options, callback) {
callback(null, result, uplink_errors) callback(null, result, uplink_errors)
}) })
}) })
} }
// /**
// Retrieve remote and local packages more recent than {startkey} Retrieve remote and local packages more recent than {startkey}
// Function streams all packages from all uplinks first, and then
// Function streams all packages from all uplinks first, and then local packages.
// local packages. Note that local packages could override registry ones just because
// they appear in JSON last. That's a trade-off we make to avoid
// Note that local packages could override registry ones just because memory issues.
// they appear in JSON last. That's a trade-off we make to avoid Used storages: local && uplink (proxy_access)
// memory issues. * @param {*} startkey
// * @param {*} options
// Used storages: local && uplink (proxy_access) */
// search(startkey, options) {
Storage.prototype.search = function(startkey, options) {
var self = this var self = this
var stream = new Stream.PassThrough({ objectMode: true }) var stream = new Stream.PassThrough({ objectMode: true })
async.eachSeries(Object.keys(self.uplinks), function(up_name, cb) { async.eachSeries(Object.keys(this.uplinks), function(up_name, cb) {
// shortcut: if `local=1` is supplied, don't call uplinks // shortcut: if `local=1` is supplied, don't call uplinks
if (options.req.query.local !== undefined) return cb() if (options.req.query.local !== undefined) return cb()
@ -364,10 +377,14 @@ Storage.prototype.search = function(startkey, options) {
}) })
}) })
return stream return stream;
} }
Storage.prototype.get_local = function(callback) { /**
*
* @param {*} callback
*/
get_local(callback) {
var self = this var self = this
var locals = this.config.localList.get() var locals = this.config.localList.get()
var packages = [] var packages = []
@ -397,16 +414,22 @@ Storage.prototype.get_local = function(callback) {
} else { } else {
callback(null, []) callback(null, [])
} }
} }
// function fetches package information from uplinks and synchronizes it with local data /**
// if package is available locally, it MUST be provided in pkginfo * Function fetches package information from uplinks and synchronizes it with local data
// returns callback(err, result, uplink_errors) if package is available locally, it MUST be provided in pkginfo
Storage.prototype._sync_package_with_uplinks = function(name, pkginfo, options, callback) { returns callback(err, result, uplink_errors)
* @param {*} name
* @param {*} pkginfo
* @param {*} options
* @param {*} callback
*/
_sync_package_with_uplinks(name, pkginfo, options, callback) {
var self = this var self = this
let exists = false;
if (!pkginfo) { if (!pkginfo) {
var exists = false exists = false
pkginfo = { pkginfo = {
name : name, name : name,
@ -415,11 +438,11 @@ Storage.prototype._sync_package_with_uplinks = function(name, pkginfo, options,
_uplinks : {}, _uplinks : {},
} }
} else { } else {
var exists = true exists = true
} }
var uplinks = [] var uplinks = []
for (var i in self.uplinks) { for (let i in self.uplinks) {
if (self.config.can_proxy_to(name, i)) { if (self.config.can_proxy_to(name, i)) {
uplinks.push(self.uplinks[i]) uplinks.push(self.uplinks[i])
} }
@ -456,20 +479,20 @@ Storage.prototype._sync_package_with_uplinks = function(name, pkginfo, options,
etag: etag, etag: etag,
fetched: Date.now() fetched: Date.now()
} }
for (let i in up_res.versions) {
for (var i in up_res.versions) {
// this won't be serialized to json, // this won't be serialized to json,
// kinda like an ES6 Symbol // kinda like an ES6 Symbol
//FIXME: perhaps Symbol('_verdaccio_uplink') here?
Object.defineProperty(up_res.versions[i], '_verdaccio_uplink', { Object.defineProperty(up_res.versions[i], '_verdaccio_uplink', {
value : up.upname, value : up.upname,
enumerable : false, enumerable : false,
configurable : false, configurable : false,
writable : true, writable : true,
}) });
} }
try { try {
Storage._merge_versions(pkginfo, up_res, self.config) Storage._merge_versions(pkginfo, up_res, self.config);
} catch(err) { } catch(err) {
self.logger.error({ self.logger.error({
sub: 'out', sub: 'out',
@ -497,11 +520,16 @@ Storage.prototype._sync_package_with_uplinks = function(name, pkginfo, options,
return callback(null, pkginfo, uplink_errors) return callback(null, pkginfo, uplink_errors)
}) })
}) })
} }
// function gets a local info and an info from uplinks and tries to merge it /**
// exported for unit tests only * Function gets a local info and an info from uplinks and tries to merge it
Storage._merge_versions = function(local, up, config) { exported for unit tests only.
* @param {*} local
* @param {*} up
* @param {*} config
*/
static _merge_versions(local, up, config) {
// copy new versions to a cache // copy new versions to a cache
// NOTE: if a certain version was updated, we can't refresh it reliably // NOTE: if a certain version was updated, we can't refresh it reliably
for (var i in up.versions) { for (var i in up.versions) {
@ -520,4 +548,8 @@ Storage._merge_versions = function(local, up, config) {
} }
} }
} }
}
} }
module.exports = Storage;

111
lib/up-storage.js
View file

@ -1,59 +1,19 @@
"use strict"; "use strict";
var JSONStream = require('JSONStream') const JSONStream = require('JSONStream')
var Error = require('http-errors') const Error = require('http-errors')
var request = require('request') const request = require('request')
var Stream = require('readable-stream') const Stream = require('readable-stream')
var URL = require('url') const URL = require('url')
var parse_interval = require('./config').parse_interval const parse_interval = require('./config').parse_interval
var Logger = require('./logger') const Logger = require('./logger')
var MyStreams = require('./streams') const MyStreams = require('./streams')
var Utils = require('./utils') const Utils = require('./utils')
var encode = function(thing) { const encode = function(thing) {
return encodeURIComponent(thing).replace(/^%40/, '@'); return encodeURIComponent(thing).replace(/^%40/, '@');
}; };
module.exports = Storage const _setupProxy = function(hostname, config, mainconfig, isHTTPS) {
//
// Implements Storage interface
// (same for storage.js, local-storage.js, up-storage.js)
//
function Storage(config, mainconfig) {
var self = Object.create(Storage.prototype)
self.config = config
self.failed_requests = 0
self.userAgent = mainconfig.user_agent
self.ca = config.ca
self.logger = Logger.logger.child({sub: 'out'})
self.server_id = mainconfig.server_id
self.url = URL.parse(self.config.url)
_setupProxy.call(self, self.url.hostname, config, mainconfig, self.url.protocol === 'https:')
self.config.url = self.config.url.replace(/\/$/, '')
if (Number(self.config.timeout) >= 1000) {
self.logger.warn([ 'Too big timeout value: ' + self.config.timeout,
'We changed time format to nginx-like one',
'(see http://wiki.nginx.org/ConfigNotation)',
'so please update your config accordingly' ].join('\n'))
}
// a bunch of different configurable timers
self.maxage = parse_interval(config_get('maxage' , '2m' ))
self.timeout = parse_interval(config_get('timeout' , '30s'))
self.max_fails = Number(config_get('max_fails' , 2 ))
self.fail_timeout = parse_interval(config_get('fail_timeout', '5m' ))
return self
// just a helper (`config[key] || default` doesn't work because of zeroes)
function config_get(key, def) {
return config[key] != null ? config[key] : def
}
}
function _setupProxy(hostname, config, mainconfig, isHTTPS) {
var no_proxy var no_proxy
var proxy_key = isHTTPS ? 'https_proxy' : 'http_proxy' var proxy_key = isHTTPS ? 'https_proxy' : 'http_proxy'
@ -100,6 +60,52 @@ function _setupProxy(hostname, config, mainconfig, isHTTPS) {
} }
} }
//
// Implements Storage interface
// (same for storage.js, local-storage.js, up-storage.js)
//
class Storage {
/**
*
* @param {*} config
* @param {*} mainconfig
*/
constructor(config, mainconfig) {
this.config = config
this.failed_requests = 0
this.userAgent = mainconfig.user_agent
this.ca = config.ca
this.logger = Logger.logger.child({sub: 'out'})
this.server_id = mainconfig.server_id
this.url = URL.parse(this.config.url)
_setupProxy.call(this, this.url.hostname, config, mainconfig, this.url.protocol === 'https:')
this.config.url = this.config.url.replace(/\/$/, '')
if (Number(this.config.timeout) >= 1000) {
this.logger.warn([ 'Too big timeout value: ' + this.config.timeout,
'We changed time format to nginx-like one',
'(see http://wiki.nginx.org/ConfigNotation)',
'so please update your config accordingly' ].join('\n'))
}
// a bunch of different configurable timers
this.maxage = parse_interval(config_get('maxage' , '2m' ))
this.timeout = parse_interval(config_get('timeout' , '30s'))
this.max_fails = Number(config_get('max_fails' , 2 ))
this.fail_timeout = parse_interval(config_get('fail_timeout', '5m' ))
return this
// just a helper (`config[key] || default` doesn't work because of zeroes)
function config_get(key, def) {
return config[key] != null ? config[key] : def
}
}
}
Storage.prototype.request = function(options, cb) { Storage.prototype.request = function(options, cb) {
if (!this.status_check()) { if (!this.status_check()) {
var req = new Stream.Readable() var req = new Stream.Readable()
@ -396,3 +402,6 @@ Storage.prototype._add_proxy_headers = function(req, headers) {
headers['Via'] += '1.1 ' + this.server_id + ' (Verdaccio)' headers['Via'] += '1.1 ' + this.server_id + ' (Verdaccio)'
} }
module.exports = Storage;