diff --git a/lib/local-storage.js b/lib/local-storage.js index 80660d5d2..56618cfb9 100644 --- a/lib/local-storage.js +++ b/lib/local-storage.js @@ -10,7 +10,9 @@ const Error = require('http-errors'); const Path = require('path'); const Stream = require('readable-stream'); const URL = require('url'); + const fs_storage = require('./local-fs'); +const LocalData = require('./local-data'); const Logger = require('./logger'); const Search = require('./search'); const MyStreams = require('./streams'); @@ -39,26 +41,18 @@ class Storage { /** * Constructor * @param {Object} config config list of properties - * @param {LocalData} localList reference */ - constructor(config, localList) { + constructor(config) { this.config = config; - this.localList = localList; + // local data handler is linked with the configuration handler + this.localList = new LocalData(Path.join(Path.resolve(Path.dirname(config.self_path || ''), config.storage), + // FUTURE: the database might be parameterizable from config.yaml + '.sinopia-db.json' + ) + ); this.logger = Logger.logger.child({sub: 'fs'}); } - /** - * Handle internal error - * @param {*} err - * @param {*} file - * @param {*} message - * @return {Object} Error instance - */ - _internal_error(err, file, message) { - this.logger.error( {err: err, file: file}, - message + ' @{file}: @{!err.message}' ); - return Error[500](); - } /** * Add a package. @@ -140,34 +134,6 @@ class Storage { this.localList.remove(name); } - /** - * Retrieve either a previous created local package or a boilerplate. - * @param {*} name - * @param {*} callback - * @return {Function} - */ - _read_create_package(name, callback) { - const storage = this.storage(name); - if (!storage) { - let data = get_boilerplate(name); - this._normalize_package(data); - return callback(null, data); - } - storage.read_json(info_file, (err, data) => { - // TODO: race condition - if (err) { - if (err.code === 'ENOENT') { - // if package doesn't exist, we create it here - data = get_boilerplate(name); - } else { - return callback(this._internal_error(err, info_file, 'error reading')); - } - } - this._normalize_package(data); - callback(null, data); - }); - } - /** * Synchronize remote package info with the local one * @param {*} name @@ -570,62 +536,6 @@ class Storage { }); } - /** - * Walks through each package and calls `on_package` on them. - * @param {*} on_package - * @param {*} on_end - */ - _each_package(on_package, on_end) { - let storages = {}; - - storages[this.config.storage] = true; - if (this.config.packages) { - Object.keys(this.packages || {}).map( (pkg) => { - if (this.config.packages[pkg].storage) { - storages[this.config.packages[pkg].storage] = true; - } - }); - } - const base = Path.dirname(this.config.self_path); - - async.eachSeries(Object.keys(storages), function(storage, cb) { - fs.readdir(Path.resolve(base, storage), function(err, files) { - if (err) { - return cb(err); - } - - async.eachSeries(files, function(file, cb) { - if (file.match(/^@/)) { - // scoped - fs.readdir(Path.resolve(base, storage, file), function(err, files) { - if (err) { - return cb(err); - } - - async.eachSeries(files, function(file2, cb) { - if (Utils.validate_name(file2)) { - on_package({ - name: `${file}/${file2}`, - path: Path.resolve(base, storage, file, file2), - }, cb); - } else { - cb(); - } - }, cb); - }); - } else if (Utils.validate_name(file)) { - on_package({ - name: file, - path: Path.resolve(base, storage, file), - }, cb); - } else { - cb(); - } - }, cb); - }); - }, on_end); - } - /** * This function allows to update the package thread-safely Algorithm: @@ -687,11 +597,11 @@ class Storage { /** * Search a local package. - * @param {*} startkey + * @param {*} startKey * @param {*} options * @return {Function} */ - search(startkey, options) { + search(startKey, options) { const stream = new Stream.PassThrough({objectMode: true}); this._each_package((item, cb) => { @@ -700,7 +610,7 @@ class Storage { return cb(err); } - if (stats.mtime > startkey) { + if (stats.mtime > startKey) { this.get_package(item.name, options, function(err, data) { if (err) { return cb(err); @@ -714,8 +624,8 @@ class Storage { 'name': data.versions[latest].name, 'description': data.versions[latest].description, 'dist-tags': {latest: latest}, - 'maintainers': data.versions[latest].maintainers || - [data.versions[latest].author].filter(Boolean), + 'maintainers': data.versions[latest].maintainers + || [data.versions[latest].author].filter(Boolean), 'author': data.versions[latest].author, 'repository': data.versions[latest].repository, 'readmeFilename': data.versions[latest].readmeFilename || '', @@ -744,6 +654,85 @@ class Storage { return stream; } + /** + * Retrieve a wrapper that provide access to the package location. + * @param {*} pkg package name. + * @return {Object} + */ + storage(pkg) { + let path = this.config.get_package_spec(pkg).storage; + if (path == null) { + path = this.config.storage; + } + if (path == null || path === false) { + this.logger.debug( {name: pkg} + , 'this package has no storage defined: @{name}' ); + return null; + } + return new PathWrapper( + Path.join( + Path.resolve(Path.dirname(this.config.self_path || ''), path), + pkg + ) + ); + } + + /** + * Walks through each package and calls `on_package` on them. + * @param {*} on_package + * @param {*} on_end + */ + _each_package(on_package, on_end) { + let storages = {}; + + storages[this.config.storage] = true; + if (this.config.packages) { + Object.keys(this.packages || {}).map( (pkg) => { + if (this.config.packages[pkg].storage) { + storages[this.config.packages[pkg].storage] = true; + } + }); + } + const base = Path.dirname(this.config.self_path); + + async.eachSeries(Object.keys(storages), function(storage, cb) { + fs.readdir(Path.resolve(base, storage), function(err, files) { + if (err) { + return cb(err); + } + + async.eachSeries(files, function(file, cb) { + if (file.match(/^@/)) { + // scoped + fs.readdir(Path.resolve(base, storage, file), function(err, files) { + if (err) { + return cb(err); + } + + async.eachSeries(files, function(file2, cb) { + if (Utils.validate_name(file2)) { + on_package({ + name: `${file}/${file2}`, + path: Path.resolve(base, storage, file, file2), + }, cb); + } else { + cb(); + } + }, cb); + }); + } else if (Utils.validate_name(file)) { + on_package({ + name: file, + path: Path.resolve(base, storage, file), + }, cb); + } else { + cb(); + } + }, cb); + }); + }, on_end); + } + /** * Normalise package properties, tags, revision id. * @param {Object} pkg package reference. @@ -761,6 +750,47 @@ class Storage { Utils.normalize_dist_tags(pkg); } + /** + * Retrieve either a previous created local package or a boilerplate. + * @param {*} name + * @param {*} callback + * @return {Function} + */ + _read_create_package(name, callback) { + const storage = this.storage(name); + if (!storage) { + let data = get_boilerplate(name); + this._normalize_package(data); + return callback(null, data); + } + storage.read_json(info_file, (err, data) => { + // TODO: race condition + if (err) { + if (err.code === 'ENOENT') { + // if package doesn't exist, we create it here + data = get_boilerplate(name); + } else { + return callback(this._internal_error(err, info_file, 'error reading')); + } + } + this._normalize_package(data); + callback(null, data); + }); + } + + /** + * Handle internal error + * @param {*} err + * @param {*} file + * @param {*} message + * @return {Object} Error instance + */ + _internal_error(err, file, message) { + this.logger.error( {err: err, file: file}, + message + ' @{file}: @{!err.message}' ); + return Error[500](); + } + /** * Update the revision (_rev) string for a package. * @param {*} name @@ -782,29 +812,6 @@ class Storage { } storage.write_json(info_file, json, callback); } - - /** - * Retrieve a wrapper that provide access to the package location. - * @param {*} pkg package name. - * @return {Object} - */ - storage(pkg) { - let path = this.config.get_package_spec(pkg).storage; - if (path == null) { - path = this.config.storage; - } - if (path == null || path === false) { - this.logger.debug( {name: pkg} - , 'this package has no storage defined: @{name}' ); - return null; - } - return new PathWrapper( - Path.join( - Path.resolve(Path.dirname(this.config.self_path || ''), path), - pkg - ) - ); - } } const PathWrapper = (function() { diff --git a/lib/search.js b/lib/search.js index 6fc442b43..8247064e6 100644 --- a/lib/search.js +++ b/lib/search.js @@ -30,7 +30,7 @@ class Search { */ query(q) { return q === '*' - ? this.storage.localList.get().map( function( pkg ) { + ? this.storage.localStorage.localList.get().map( function( pkg ) { return {ref: pkg, score: 1}; }) : this.index.search(q); } diff --git a/lib/storage.js b/lib/storage.js index 7f636d764..e4e296fdd 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -2,13 +2,11 @@ const assert = require('assert'); const async = require('async'); -const path = require('path'); const Error = require('http-errors'); const semver = require('semver'); const Crypto = require('crypto'); const Stream = require('stream'); -const Local = require('./local-storage'); -const LocalData = require('./local-data'); +const LocalStorage = require('./local-storage'); const Logger = require('./logger'); const MyStreams = require('./streams'); const Proxy = require('./up-storage'); @@ -36,24 +34,18 @@ class Storage { this.uplinks[p].upname = p; } } - // local data handler is linked with the configuration handler - this.localList = new LocalData(path.join(path.resolve(path.dirname(config.self_path || ''), config.storage), - // FUTURE: the database might be parameterizable from config.yaml - '.sinopia-db.json' - ) - ); + this.localStorage = new LocalStorage(config); // it generates a secret key // FUTURE: this might be an external secret key, perhaps whitin config file? if (!config.secret) { - config.secret = this.localList.data.secret; + config.secret = this.localStorage.localList.data.secret; if (!config.secret) { config.secret = Crypto.pseudoRandomBytes(32).toString('hex'); - this.localList.data.secret = config.secret; - this.localList.sync(); + this.localStorage.localList.data.secret = config.secret; + this.localStorage.localList.sync(); } } // an instance for local storage - this.local = new Local(config, this.localList); this.logger = Logger.logger.child(); } @@ -92,7 +84,7 @@ class Storage { * @param {*} cb the callback method */ function check_package_local(cb) { - self.local.get_package(name, {}, function(err, results) { + self.localStorage.get_package(name, {}, function(err, results) { if (err && err.status !== 404) { return cb(err); } @@ -136,7 +128,7 @@ class Storage { * @param {*} cb callback method */ function publish_package(cb) { - self.local.add_package(name, metadata, callback); + self.localStorage.add_package(name, metadata, callback); } } @@ -150,7 +142,7 @@ class Storage { * @param {*} callback */ add_version(name, version, metadata, tag, callback) { - this.local.add_version(name, version, metadata, tag, callback); + this.localStorage.add_version(name, version, metadata, tag, callback); } /** @@ -161,7 +153,7 @@ class Storage { * @param {*} callback */ merge_tags(name, tag_hash, callback) { - this.local.merge_tags(name, tag_hash, callback); + this.localStorage.merge_tags(name, tag_hash, callback); } /** @@ -172,7 +164,7 @@ class Storage { * @param {*} callback */ replace_tags(name, tag_hash, callback) { - this.local.replace_tags(name, tag_hash, callback); + this.localStorage.replace_tags(name, tag_hash, callback); } /** @@ -185,7 +177,7 @@ class Storage { * @param {*} callback */ change_package(name, metadata, revision, callback) { - this.local.change_package(name, metadata, revision, callback); + this.localStorage.change_package(name, metadata, revision, callback); } /** @@ -196,7 +188,7 @@ class Storage { * @param {*} callback */ remove_package(name, callback) { - this.local.remove_package(name, callback); + this.localStorage.remove_package(name, callback); } /** @@ -211,7 +203,7 @@ class Storage { * @param {*} callback */ remove_tarball(name, filename, revision, callback) { - this.local.remove_tarball(name, filename, revision, callback); + this.localStorage.remove_tarball(name, filename, revision, callback); } /** @@ -223,7 +215,7 @@ class Storage { * @return {Stream} */ add_tarball(name, filename) { - return this.local.add_tarball(name, filename); + return this.localStorage.add_tarball(name, filename); } /** @@ -246,7 +238,7 @@ class Storage { // information about it, so fetching package info is unnecessary // trying local first - let rstream = self.local.get_tarball(name, filename); + let rstream = self.localStorage.get_tarball(name, filename); let is_open = false; rstream.on('error', function(err) { if (is_open || err.status !== 404) { @@ -257,7 +249,7 @@ class Storage { let err404 = err; rstream.abort(); rstream = null; // gc - self.local.get_package(name, function(err, info) { + self.localStorage.get_package(name, function(err, info) { if (!err && info._distfiles && info._distfiles[filename] != null) { // information about this file exists locally serve_file(info._distfiles[filename]); @@ -304,7 +296,7 @@ class Storage { } let savestream = null; if (uplink.config.cache) { - savestream = self.local.add_tarball(name, filename); + savestream = self.localStorage.add_tarball(name, filename); } let on_open = function() { // prevent it from being called twice @@ -370,7 +362,7 @@ class Storage { callback = options, options = {}; } - this.local.get_package(name, options, (err, data) => { + this.localStorage.get_package(name, options, (err, data) => { if (err && (!err.status || err.status >= 500)) { // report internal errors right away return callback(err); @@ -437,7 +429,7 @@ class Storage { // executed after all series function() { // attach a local search results - let lstream = self.local.search(startkey, options); + let lstream = self.localStorage.search(startkey, options); stream.abort = function() { lstream.abort(); }; @@ -452,23 +444,22 @@ class Storage { } /** - * + * Retrieve only private local packages * @param {*} callback */ get_local(callback) { let self = this; - let locals = this.localList.get(); + let locals = this.localStorage.localList.get(); let packages = []; const getPackage = function(i) { - self.local.get_package(locals[i], function(err, info) { + self.localStorage.get_package(locals[i], function(err, info) { if (!err) { let latest = info['dist-tags'].latest; if (latest && info.versions[latest]) { packages.push(info.versions[latest]); } else { - self.logger.warn( {package: locals[i]} - , 'package @{package} does not have a "latest" tag?' ); + self.logger.warn( {package: locals[i]}, 'package @{package} does not have a "latest" tag?' ); } } @@ -557,7 +548,6 @@ class Storage { if (Object.prototype.hasOwnProperty.call(up_res.versions, i)) { // this won't be serialized to json, // kinda like an ES6 Symbol - // FIXME: perhaps Symbol('_verdaccio_uplink') here? Object.defineProperty(up_res.versions[i], '_verdaccio_uplink', { value: up.upname, enumerable: false, @@ -591,7 +581,7 @@ class Storage { , uplink_errors ); } - self.local.update_versions(name, pkginfo, function(err, pkginfo) { + self.localStorage.update_versions(name, pkginfo, function(err, pkginfo) { if (err) return callback(err); return callback(null, pkginfo, uplink_errors); }); @@ -604,6 +594,7 @@ class Storage { * @param {*} local * @param {*} up * @param {*} config + * @static */ static _merge_versions(local, up, config) { // copy new versions to a cache