Merge pull request #166 from Meeeeow/fix_upstream_search

fix upstream search

lib/index-api.js

@@ -94,13 +94,40 @@ module.exports = function(config, auth, storage) {
   })
 
   // searching packages
-  app.get('/-/all/:anything?', function(req, res, next) {
+  app.get('/-/all(\/since)?', function(req, res, next) {
     var received_end      = false
     var response_finished = false
     var processing_pkgs   = 0
+    var firstPackage = true
 
     res.status(200)
-    res.write('{"_updated":' + Date.now());
+
+    /*
+     * Offical NPM registry (registry.npmjs.org) no longer return whole database,
+     * They only return packages matched with keyword in `referer: search pkg-name`,
+     * And NPM client will request server in every search.
+     *
+     * The magic number 99999 was sent by NPM registry. Modify it may caused strange
+     * behaviour in the future.
+     *
+     * BTW: NPM will not return result if user-agent does not contain string 'npm',
+     * See: method 'request' in up-storage.js
+     *
+     * If there is no cache in local, NPM will request /-/all, then get response with
+     * _updated: 99999, 'Date' in response header was Mon, 10 Oct 1983 00:12:48 GMT,
+     * this will make NPM always query from server
+     *
+     * Data structure also different, whel request /-/all, response is an object, but
+     * when request /-/all/since, response is an array
+    */
+    var respShouldBeArray = req.path.endsWith('/since')
+    res.set('Date', 'Mon, 10 Oct 1983 00:12:48 GMT')
+
+    if (respShouldBeArray) {
+      res.write('[')
+    } else {
+      res.write('{"_updated":' + 99999);
+    }
 
     var stream = storage.search(req.query.startkey || 0, { req: req })
 
@@ -121,8 +148,15 @@ module.exports = function(config, auth, storage) {
         }
 
         if (allowed) {
+          if (respShouldBeArray) {
+            res.write(`${firstPackage ? '' : ','}${JSON.stringify(pkg)}\n`)
+            if (firstPackage) {
+              firstPackage = false
+            }
+          } else {
             res.write(',\n' + JSON.stringify(pkg.name) + ':' + JSON.stringify(pkg))
           }
+        }
 
         check_finish()
       })
@@ -143,8 +177,12 @@ module.exports = function(config, auth, storage) {
       if (response_finished) return
 
       response_finished = true
+      if (respShouldBeArray) {
+        res.end(']\n')
+      } else {
         res.end('}\n')
       }
+    }
   })
 
   // placeholder 'cause npm require to be authenticated to publish

lib/up-storage.js

@@ -113,7 +113,8 @@ Storage.prototype.request = function(options, cb) {
   var headers = options.headers || {}
   headers['Accept']          = headers['Accept']          || 'application/json'
   headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip'
-  headers['User-Agent']      = headers['User-Agent']      || this.userAgent
+  // registry.npmjs.org will only return search result if user-agent include string 'npm'
+  headers['User-Agent']      = headers['User-Agent']      || `npm (${this.userAgent})`
   this._add_proxy_headers(options.req, headers)
 
   // add/override headers specified in the config
@@ -337,6 +338,9 @@ Storage.prototype.search = function(startkey, options) {
   var req = self.request({
     uri: options.req.url,
     req: options.req,
+    headers: {
+      referer: options.req.headers.referer
+    }
   })
 
   req.on('response', function (res) {
@@ -349,7 +353,6 @@ Storage.prototype.search = function(startkey, options) {
         stream.emit('data', pkg)
       }
     })
-
     res.on('end', function () {
       stream.emit('end')
     })