feat: tarball url redirect (#1688)

* feat: tarball url redirect

* fix: handle uplinks

* feat: allow function for config.tarball_url_redirect

* fix:  hasLocalTarball was calling localStream,abort when already aborted

* chore: simplify localStream null check in hasLocalTarball

As requested in PR feedback.

* chore: fix sonarcloud code smell on test

the variable `credentials` was already declared before the tarball url tests.

* fix: move tarball_url_redirect to experiments

Co-authored-by: Gord Lea <johlea@cisco.com>
Co-authored-by: Gord Lea <jgordonlea@gmail.com>

conf/default.yaml

@@ -83,6 +83,13 @@ logs: { type: stdout, format: pretty, level: http }
 #  search: false
 #  # disable writing body size to logs, read more on ticket 1912
 #  bytesin_off: false
+#  # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string
+#  tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}'
+#  # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file
+#  tarball_url_redirect(packageName, filename) {
+#    const signedUrl = // generate a signed url
+#    return signedUrl;
+#  }
 
 # This affect the web and api (not developed yet)
 #i18n:

conf/docker.yaml

@@ -86,6 +86,13 @@ logs: { type: stdout, format: pretty, level: http }
 #  token: false
 #  # support for the new v1 search endpoint, functional by incomplete read more on ticket 1732
 #  search: false
+#  # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string
+#  tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}'
+#  # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file
+#  tarball_url_redirect(packageName, filename) {
+#    const signedUrl = // generate a signed url
+#    return signedUrl;
+#  }
 
 # This affect the web and api (not developed yet)
 #i18n:

src/api/endpoint/api/package.ts

@@ -33,6 +33,30 @@ const downloadStream = (
   stream.pipe(res);
 };
 
+const redirectOrDownloadStream = (
+  packageName: string,
+  filename: string,
+  storage: any,
+  req: $RequestExtend,
+  res: $ResponseExtend,
+  config: Config
+): void => {
+  const tarballUrlRedirect = _.get(config, 'experiments.tarball_url_redirect');
+  storage.hasLocalTarball(packageName, filename).then(hasLocalTarball => {
+    if (hasLocalTarball) {
+      const context = { packageName, filename };
+      const tarballUrl = typeof tarballUrlRedirect === 'function'
+        ? tarballUrlRedirect(context)
+        : _.template(tarballUrlRedirect)(context);
+      res.redirect(tarballUrl);
+    } else {
+      downloadStream(packageName, filename, storage, req, res)
+    }
+  }).catch(err => {
+    res.locals.report_error(err);
+  });
+}
+
 export default function (
   route: Router,
   auth: IAuth,
@@ -87,8 +111,11 @@ export default function (
     can('access'),
     function (req: $RequestExtend, res: $ResponseExtend): void {
       const { scopedPackage, filename } = req.params;
-
+      if (_.get(config, 'experiments.tarball_url_redirect') === undefined) {
         downloadStream(scopedPackage, filename, storage, req, res);
+      } else {
+        redirectOrDownloadStream(scopedPackage, filename, storage, req, res, config);
+      }
     }
   );
 
@@ -96,7 +123,11 @@ export default function (
     '/:package/-/:filename',
     can('access'),
     function (req: $RequestExtend, res: $ResponseExtend): void {
+      if (_.get(config, 'experiments.tarball_url_redirect') === undefined) {
         downloadStream(req.params.package, req.params.filename, storage, req, res);
+      } else {
+        redirectOrDownloadStream(req.params.package, req.params.filename, storage, req, res, config);
+      }
     }
   );
 }

src/lib/constants.ts

@@ -76,6 +76,7 @@ export const HTTP_STATUS = {
   OK: 200,
   CREATED: 201,
   MULTIPLE_CHOICES: 300,
+  REDIRECT: 302,
   NOT_MODIFIED: 304,
   BAD_REQUEST: 400,
   UNAUTHORIZED: 401,

src/lib/storage.ts

@@ -176,6 +176,34 @@ class Storage implements IStorageHandler {
     return this.localStorage.addTarball(name, filename);
   }
 
+  public hasLocalTarball(name: string, filename: string): Promise<boolean> {
+    const self = this;
+    return new Promise<boolean>((resolve, reject): void => {
+      let localStream: any = self.localStorage.getTarball(name, filename);
+      let isOpen = false;
+      localStream.on(
+        'error',
+        (err): any => {
+          if (isOpen || err.status !== HTTP_STATUS.NOT_FOUND) {
+            reject(err);
+          }
+          // local reported 404 or request was aborted already
+          if (localStream) {
+              localStream.abort();
+              localStream = null;
+          }
+          resolve(false);
+        }
+      );
+      localStream.on('open', function(): void {
+        isOpen = true;
+        localStream.abort();
+        localStream = null;
+        resolve(true);
+      });
+    });
+  }
+
   /**
    Get a tarball from a storage for {name} package
    Function is synchronous and returns a ReadableStream

test/unit/modules/api/api.spec.ts

@@ -978,6 +978,124 @@ describe('endpoint unit test', () => {
       });
     });
 
+    describe('should test tarball url redirect', () => {
+      const pkgName = 'testTarballPackage';
+      const scopedPkgName = '@tarball_tester/testTarballPackage';
+      const tarballUrlRedirectCredentials = { name: 'tarball_tester', password: 'secretPass' };
+      const store = path.join(__dirname, '../../partials/store/test-storage-api-spec');
+      const mockServerPort = 55549;
+      const baseTestConfig = configDefault({
+        auth: {
+          htpasswd: {
+            file: './test-storage-api-spec/.htpasswd'
+          }
+        },
+        filters: {
+          '../../modules/api/partials/plugin/filter': {
+            pkg: 'npm_test',
+            version: '2.0.0'
+          }
+        },
+        storage: store,
+        self_path: store,
+        uplinks: {
+          npmjs: {
+            url: `http://${DOMAIN_SERVERS}:${mockServerPort}`
+          }
+        },
+        logs: [
+          { type: 'stdout', format: 'pretty', level: 'warn' }
+        ],
+      }, 'api.spec.yaml');
+      let token;
+      beforeAll(async (done) => {
+        token = await getNewToken(request(app), tarballUrlRedirectCredentials);
+        await putPackage(request(app), `/${pkgName}`, generatePackageMetadata(pkgName), token);
+        await putPackage(request(app), `/${scopedPkgName}`, generatePackageMetadata(scopedPkgName), token);
+        done();
+      });
+
+      describe('for a string value of tarball_url_redirect', () => {
+        let app2;
+        beforeAll(async (done) => {
+          app2 = await endPointAPI({
+            ...baseTestConfig,
+            experiments: {
+              tarball_url_redirect: 'https://myapp.sfo1.mycdn.com/verdaccio/${packageName}/${filename}'
+            }
+          });
+          done();
+        });
+
+        test('should redirect for package tarball', (done) => {
+          request(app2)
+            .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+            .expect(HTTP_STATUS.REDIRECT)
+            .end(function (err, res) {
+              if (err) {
+                return done(err);
+              }
+              expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz');
+              done();
+            });
+        });
+
+        test('should redirect for scoped package tarball', (done) => {
+          request(app2)
+            .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+            .expect(HTTP_STATUS.REDIRECT)
+            .end(function (err, res) {
+              if (err) {
+                return done(err);
+              }
+              expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz');
+              done();
+            });
+        });
+      });
+
+      describe('for a function value of tarball_url_redirect', () => {
+        let app2;
+        beforeAll(async (done) => {
+          app2 = await endPointAPI({
+            ...baseTestConfig,
+            experiments: {
+              tarball_url_redirect(context) {
+                return `https://myapp.sfo1.mycdn.com/verdaccio/${context.packageName}/${context.filename}`
+              }
+            }
+          });
+          done();
+        });
+
+        test('should redirect for package tarball', (done) => {
+          request(app2)
+            .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+            .expect(HTTP_STATUS.REDIRECT)
+            .end(function (err, res) {
+              if (err) {
+                return done(err);
+              }
+              expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz');
+              done();
+            });
+        });
+
+        test('should redirect for scoped package tarball', (done) => {
+          request(app2)
+            .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+            .expect(HTTP_STATUS.REDIRECT)
+            .end(function (err, res) {
+              if (err) {
+                return done(err);
+              }
+              expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz');
+              done();
+            });
+        });
+      });
+    });
+
     describe('should test (un)deprecate api', () => {
       const pkgName = '@scope/deprecate';
       const credentials = { name: 'jota_deprecate', password: 'secretPass' };