[GH-131] add cache option to uplinks

.gitignore

@@ -5,6 +5,7 @@ verdaccio-*.tgz
 ###
 !bin/verdaccio
 test-storage*
+.verdaccio_test_env
 node_modules
 
 

conf/full.yaml

@@ -59,6 +59,10 @@ uplinks:
     #headers:
     #  authorization: "Basic YourBase64EncodedCredentials=="
 
+    # set this to false to prevent tarballs from this upstream
+    # to be stored in the local storage (defaults to true)
+    #cache: false
+
 packages:
   # uncomment this for packages with "local-" prefix to be available
   # for admin only, it's a recommended way of handling private packages

lib/config.js

@@ -140,6 +140,9 @@ class Config {
 		}
 		// sanity check for uplinks
 		for (let i in self.uplinks) {
+			if (self.uplinks[i].cache == null) {
+				self.uplinks[i].cache = true;
+			}
 			if (Object.prototype.hasOwnProperty.call(self.uplinks, i)) {
 				check_user_or_uplink(i);
 			}

lib/storage.js

@@ -279,10 +279,14 @@ class Storage {
       if (uplink == null) {
         uplink = new Proxy({
           url: file.url,
+          cache: true,
           _autogenerated: true,
         }, self.config);
       }
-      let savestream = self.local.add_tarball(name, filename);
+      let savestream = null;
+      if (uplink.config.cache) {
+        savestream = self.local.add_tarball(name, filename);
+      }
       let on_open = function() {
         // prevent it from being called twice
         on_open = function() {};
@@ -312,6 +316,7 @@ class Storage {
         }
       };
 
+      if (savestream) {
         savestream.on('open', function() {
           on_open();
         });
@@ -325,6 +330,9 @@ class Storage {
           savestream = null;
           on_open();
         });
+      } else {
+        on_open();
+      }
     }
   }
 

test/functional/config-1.yaml

@@ -16,6 +16,8 @@ uplinks:
     timeout: 100ms
   server2:
     url: http://localhost:55552/
+  server3:
+    url: http://localhost:55553/
   baduplink:
     url: http://localhost:55666/
 

test/functional/config-3.yaml

@@ -0,0 +1,36 @@
+storage: ./test-storage3
+
+users:
+  test:
+    password: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+
+web:
+  enable: true
+
+uplinks:
+  server1:
+    url: http://localhost:55551/
+#    cache: true
+  server2:
+    url: http://localhost:55552/
+    cache: false
+
+logs:
+  - {type: stdout, format: pretty, level: trace}
+
+packages:
+  'pkg-gh131':
+    access: $all
+    proxy: server1
+
+  'pkg-gh1312':
+    access: $all
+    proxy: server2
+
+  '*':
+    access: $all
+
+listen: 55553
+
+# expose internal methods
+_debug: true

test/functional/gh131.js

@@ -0,0 +1,90 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+const crypto = require('crypto');
+
+const STORAGE = 'test-storage3';
+const TARBALL = 'blahblah';
+const PKG_GH131 = 'pkg-gh131';
+const PKG_GH1312 = 'pkg-gh1312';
+
+function isCached(pkgname, tarballname) {
+  return fs.existsSync(path.join(__dirname, STORAGE, pkgname, tarballname));
+}
+
+function readfile(x) {
+  return fs.readFileSync(path.join(__dirname, x));
+}
+
+module.exports = function() {
+  const server = process.server;
+  const server2 = process.server2;
+  const server3 = process.server3;
+
+  before(function() {
+    return server.add_package(PKG_GH131);
+  });
+
+  before(function() {
+    return server.put_tarball(PKG_GH131, TARBALL, readfile('fixtures/binary'))
+             .status(201)
+             .body_ok(/.*/);
+  });
+
+  before(function() {
+    const pkg = require('./lib/package')(PKG_GH131);
+    pkg.dist.shasum = crypto.createHash('sha1').update(readfile('fixtures/binary')).digest('hex');
+    return server.put_version(PKG_GH131, '0.0.1', pkg)
+             .status(201)
+             .body_ok(/published/);
+  });
+
+  before(function() {
+    return server3.get_package(PKG_GH131)
+	    .status(200);
+  });
+
+  before(function() {
+    return server3.get_tarball(PKG_GH131, TARBALL)
+            .status(200);
+  });
+
+  it('should be caching packages from uplink server1', function () {
+    assert.equal(isCached(PKG_GH131, TARBALL), true);
+  });
+
+  before(function() {
+    return server2.add_package(PKG_GH1312);
+  });
+
+  before(function() {
+    return server2.put_tarball(PKG_GH1312, TARBALL, readfile('fixtures/binary'))
+             .status(201)
+             .body_ok(/.*/);
+  });
+
+  before(function() {
+    const pkg = require('./lib/package')(PKG_GH1312);
+    pkg.dist.shasum = crypto.createHash('sha1').update(readfile('fixtures/binary')).digest('hex');
+    return server2.put_version(PKG_GH1312, '0.0.1', pkg)
+             .status(201)
+             .body_ok(/published/);
+  });
+
+  before(function() {
+    return server3.get_package(PKG_GH1312)
+	    .status(200);
+  });
+
+  before(function() {
+    return server3.get_tarball(PKG_GH1312, TARBALL)
+            .status(200);
+  });
+
+  it('must not be caching packages from uplink server2', function () {
+    assert.equal(isCached(PKG_GH1312, TARBALL), false);
+  });
+};
+

test/functional/index.js

@@ -8,18 +8,20 @@ const exec = require('child_process').exec;
 describe('Func', function() {
   const server = process.server;
   const server2 = process.server2;
+  const server3 = process.server3;
 
   before(function(done) {
     Promise.all([
       require('./lib/startup').start('./test-storage', './config-1.yaml'),
       require('./lib/startup').start('./test-storage2', './config-2.yaml'),
+      require('./lib/startup').start('./test-storage3', './config-3.yaml'),
     ]).then(() => {
       done();
   });
   });
 
   before(function() {
-    return Promise.all([server, server2].map(function(server) {
+    return Promise.all([server, server2, server3].map(function(server) {
       return server.debug().status(200).then(function(body) {
         server.pid = body.pid;
         return new Promise(function(resolve, reject) {
@@ -34,7 +36,7 @@ describe('Func', function() {
   });
 
   before(function auth() {
-    return Promise.all([server, server2].map(function(server, cb) {
+    return Promise.all([server, server2, server3].map(function(server, cb) {
       return server.auth('test', 'test').status(201).body_ok(/'test'/);
     }));
   });
@@ -58,6 +60,8 @@ describe('Func', function() {
   require('./logout')();
   require('./addtag')();
   require('./plugins')();
+  // requires packages published to server1/server2
+  require('./gh131')();
 
   after(function(done) {
     const check = (server) => {
@@ -78,7 +82,7 @@ describe('Func', function() {
         });
       });
     };
-    Promise.all([check(server), check(server2)]).then(function() {
+    Promise.all([check(server), check(server2), check(server3)]).then(function() {
       done();
     }, (reason) => {
       assert.equal(reason, null);

test/functional/lib/startup.js

@@ -8,6 +8,7 @@ const Server = require('./server');
 const forks = process.forks = [];
 process.server = Server('http://localhost:55551/');
 process.server2 = Server('http://localhost:55552/');
+process.server3 = Server('http://localhost:55553/');
 process.express = express();
 process.express.listen(55550);
 
@@ -39,6 +40,13 @@ module.exports.start = function(dir, conf) {
 };
 
 process.on('exit', function() {
-  if (forks[0]) forks[0].kill();
+  if (forks[0]) {
-  if (forks[1]) forks[1].kill();
+    forks[0].kill();
+  }
+  if (forks[1]) {
+    forks[1].kill();
+  }
+  if (forks[2]) {
+    forks[2].kill();
+  }
 });

test/integration/config_nocache.yaml

@@ -0,0 +1,25 @@
+storage: ./.verdaccio_test_env/test-storage
+
+users:
+  test:
+    password: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+    cache: false
+
+logs:
+  - {type: stdout, format: pretty, level: trace}
+
+packages:
+  jju:
+    allow_access: all
+    allow_publish: all
+    proxy_access: npmjs
+
+  '*':
+    allow_access: all
+    allow_publish: all
+
+listen: 55501

test/integration/test.pl

@@ -38,7 +38,7 @@ system('npm install jju') and quit('fail');
 (`node -e 'console.log(require("jju").parse("{qwerty:123}").qwerty+456)'` =~ /579/) or quit('fail');
 
 system('npm publish ../verdaccio-test-1.2.3.tgz') and quit('fail');
-system('npm tag verdaccio-test@1.2.3 meow') and quit('fail');
+system('npm dist-tag add verdaccio-test@1.2.3 meow') and quit('fail');
 system('npm install verdaccio-test@meow') and quit('fail');
 
 (`node -e 'require("verdaccio-test")'` =~ /w==w/) or quit('fail');

test/integration/test_nocache.pl

@@ -0,0 +1,44 @@
+#!/usr/bin/perl
+
+# note to readers: in perl it's useful, in javascript it isn't
+use strict;
+
+# setting up working environment && chdir there
+use Cwd 'abs_path';
+use File::Basename;
+$ENV{HOME} = dirname(abs_path( __FILE__ )) . '/.verdaccio_test_env';
+system('rm -rf .verdaccio_test_env ; mkdir .verdaccio_test_env') and quit('fail');
+chdir $ENV{HOME};
+
+use Data::Dumper;
+my $pid;
+
+sub quit {
+	print $_[0]."\n";
+	exec("kill $pid ; exit 1");
+}
+
+# run verdaccio in a child process
+if (($pid = fork()) == 0) {
+	exec "../../../bin/verdaccio ../config_nocache.yaml";
+	die "exec failed";
+}
+
+system('mkdir node_modules') and quit('fail');
+system('npm set verdaccio_test_config 12345') and quit('fail');
+
+if (`cat .npmrc` !~ /verdaccio_test_config/) {
+	quit "npm is using wrong config";
+}
+
+system('npm set registry http://localhost:55501') and quit('fail');
+system(q{/bin/echo -e 'test\ntest\ns@s.s\n' | npm adduser}) and quit('fail');
+
+system('npm install jju') and quit('fail');
+system('test ! -f ./test-storage/jju/jju-*.tgz') and quit('fail');
+
+
+quit("
+==================================================================
+All tests seem to be executed successfully, nothing is broken yet.
+==================================================================");

test/unit/config.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const assert = require('assert');
+const config_hash = require('./partials/config');
+const Config = require('../../lib/config');
+
+
+describe('Config', function() {
+  before(function() {
+    this.config = new Config(config_hash);
+  });
+
+  it('npmjs uplink should have a default cache option that is true', function() {
+    assert.equal(this.config.uplinks['npmjs'].cache, true);
+  });
+});
+

test/unit/partials/config.js

@@ -1,7 +1,12 @@
 'use strict';
 
-let config = {
+const config = {
   storage: __dirname + '/test-storage',
+  uplinks: {
+    'npmjs': {
+      'url': 'https://registry.npmjs.org/'
+    }
+  },
   packages: {
     '*': {
       allow_access: '$all',