diff --git a/lib/index-api.js b/lib/index-api.js index f53d152e1..ee706e88d 100644 --- a/lib/index-api.js +++ b/lib/index-api.js @@ -94,13 +94,38 @@ module.exports = function(config, auth, storage) { }) // searching packages - app.get('/-/all/:anything?', function(req, res, next) { + app.get('/-/all(\/since)?', function(req, res, next) { var received_end = false var response_finished = false var processing_pkgs = 0 + let firstPackage = true res.status(200) - res.write('{"_updated":' + Date.now()); + + /* + * Offical NPM registry (registry.npmjs.org) no longer return whole database, + * They only return packages matched with keyword in `referer: search pkg-name`, + * And NPM client will request server in every search. + * + * The magic number 99999 was sent by NPM registry. Modify it may caused strange + * behaviour in the future. + * + * BTW: NPM will not return result if user-agent does not contain string 'npm', + * See: method 'request' in up-storage.js + * + * If there is no cache in local, NPM will request /-/all, then get response with + * _updated: 99999, after this, NPM will request /-/all/since with another magic + * number 434592768000 + * + * Data structure also different, whel request /-/all, response is an object, but + * when request /-/all/since, response is an array + */ + const respShouldBeArray = req.path.endsWith('/since') + if (respShouldBeArray) { + res.write('[') + } else { + res.write('{"_updated":' + 99999); + } var stream = storage.search(req.query.startkey || 0, { req: req }) @@ -121,7 +146,14 @@ module.exports = function(config, auth, storage) { } if (allowed) { - res.write(',\n' + JSON.stringify(pkg.name) + ':' + JSON.stringify(pkg)) + if (respShouldBeArray) { + res.write(`${firstPackage ? '' : ','}${JSON.stringify(pkg)}\n`) + if (firstPackage) { + firstPackage = false + } + } else { + res.write(',\n' + JSON.stringify(pkg.name) + ':' + JSON.stringify(pkg)) + } } check_finish() @@ -143,7 +175,11 @@ module.exports = function(config, auth, storage) { if (response_finished) return response_finished = true - res.end('}\n') + if (respShouldBeArray) { + res.end(']\n') + } else { + res.end('}\n') + } } }) diff --git a/lib/up-storage.js b/lib/up-storage.js index 8f7606d4d..5792d396d 100644 --- a/lib/up-storage.js +++ b/lib/up-storage.js @@ -113,7 +113,8 @@ Storage.prototype.request = function(options, cb) { var headers = options.headers || {} headers['Accept'] = headers['Accept'] || 'application/json' headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip' - headers['User-Agent'] = headers['User-Agent'] || this.userAgent + // registry.npmjs.org will only return search result if user-agent include string 'npm' + headers['User-Agent'] = headers['User-Agent'] || `npm (${this.userAgent})` this._add_proxy_headers(options.req, headers) // add/override headers specified in the config @@ -337,6 +338,9 @@ Storage.prototype.search = function(startkey, options) { var req = self.request({ uri: options.req.url, req: options.req, + headers: { + referer: options.req.headers.referer + } }) req.on('response', function (res) { @@ -349,7 +353,6 @@ Storage.prototype.search = function(startkey, options) { stream.emit('data', pkg) } }) - res.on('end', function () { stream.emit('end') })