feat(experiment): accept async tarball_url_redirect function (#3914)

* chore: await tarball_url_redirect function

* test: unit test async tarball_url_redirect

* chore: prettier format

* chore: refactor redirectOrDownloadStream to better error handling

* refactor tests

* format

* format

---------

Co-authored-by: Juan Picado <juanpicado19@gmail.com>

src/api/endpoint/api/package.ts

@@ -34,32 +34,39 @@ const downloadStream = (
   stream.pipe(res);
 };
 
-const redirectOrDownloadStream = (
+const redirectOrDownloadStream = async (
   packageName: string,
   filename: string,
   storage: any,
   req: $RequestExtend,
   res: $ResponseExtend,
   config: Config
-): void => {
+): Promise<void> => {
-  const tarballUrlRedirect = _.get(config, 'experiments.tarball_url_redirect');
+  try {
-  storage
+    const hasLocalTarball = await storage.hasLocalTarball(packageName, filename);
-    .hasLocalTarball(packageName, filename)
+
-    .then((hasLocalTarball) => {
+    if (!hasLocalTarball) {
-      if (hasLocalTarball) {
+      return downloadStream(packageName, filename, storage, req, res);
-        const context = { packageName, filename };
+    }
-        const tarballUrl =
+
-          typeof tarballUrlRedirect === 'function'
+    const tarballUrlRedirect = _.get(config, 'experiments.tarball_url_redirect');
-            ? tarballUrlRedirect(context)
+    const context = { packageName, filename };
-            : _.template(tarballUrlRedirect)(context);
+    let tarballUrl;
-        res.redirect(tarballUrl);
+
+    if (typeof tarballUrlRedirect === 'function') {
+      if (tarballUrlRedirect.constructor.name === 'AsyncFunction') {
+        tarballUrl = await tarballUrlRedirect(context);
       } else {
-        downloadStream(packageName, filename, storage, req, res);
+        tarballUrl = tarballUrlRedirect(context);
       }
-    })
+    } else {
-    .catch((err) => {
+      tarballUrl = _.template(tarballUrlRedirect)(context);
-      res.locals.report_error(err);
+    }
-    });
+
+    res.redirect(tarballUrl);
+  } catch (err) {
+    res.locals.report_error(err);
+  }
 };
 
 export default function (route: Router, auth: Auth, storage: Storage, config: Config): void {

test/helpers/initializeServer.ts

@@ -5,6 +5,7 @@ import path from 'path';
 
 import { errorUtils } from '@verdaccio/core';
 import { final } from '@verdaccio/middleware';
+import { ConfigYaml } from '@verdaccio/types';
 import { generateRandomHexString } from '@verdaccio/utils';
 
 import { errorReportingMiddleware, handleError } from '../../src/api/middleware';
@@ -14,12 +15,12 @@ import Config from '../../src/lib/config';
 const debug = buildDebug('verdaccio:tools:helpers:server');
 
 export async function initializeServer(
-  configName,
+  configuration: ConfigYaml,
   routesMiddleware: any[] = [],
   Storage
 ): Promise<Application> {
   const app = express();
-  const config = new Config(configName);
+  const config = new Config(configuration);
   config.storage = path.join(os.tmpdir(), '/storage', generateRandomHexString());
   // httpass would get path.basename() for configPath thus we need to create a dummy folder
   // to avoid conflics

test/unit/modules/api/_helper.ts

@@ -5,7 +5,7 @@ import supertest from 'supertest';
 
 import { parseConfigFile } from '@verdaccio/config';
 import { HEADERS, HEADER_TYPE, HTTP_STATUS, TOKEN_BEARER } from '@verdaccio/core';
-import { GenericBody, PackageUsers } from '@verdaccio/types';
+import { ConfigYaml, GenericBody, PackageUsers } from '@verdaccio/types';
 import { buildToken, generateRandomHexString } from '@verdaccio/utils';
 
 import apiMiddleware from '../../../../src/api/endpoint';
@@ -24,11 +24,15 @@ export const getConf = (conf) => {
   return config;
 };
 
-export async function initializeServer(configName): Promise<Application> {
+export async function initializeServer(configName: string): Promise<Application> {
   const config = getConf(configName);
   return initializeServerHelper(config, [apiMiddleware], Storage);
 }
 
+export async function initializeServerWithConfig(config: ConfigYaml): Promise<Application> {
+  return initializeServerHelper(config, [apiMiddleware], Storage);
+}
+
 export function createUser(app, name: string, password: string): supertest.Test {
   return supertest(app)
     .put(`/-/user/org.couchdb.user:${name}`)

test/unit/modules/api/config/experiments.yaml

@@ -0,0 +1,25 @@
+storage: ./storage_experiments
+
+auth:
+  htpasswd:
+    file: ./htpasswd-package
+
+web:
+  enable: true
+  title: verdaccio
+
+publish:
+  allow_offline: false
+
+uplinks:
+
+log: { type: stdout, format: pretty, level: trace }
+
+packages:
+  '@*/*':
+    access: $anonymous
+    publish: $anonymous
+  '**':
+    access: $anonymous
+    publish: $anonymous
+_debug: true

test/unit/modules/api/experiments.spec.ts

@@ -0,0 +1,150 @@
+import supertest from 'supertest';
+
+import { DIST_TAGS, HEADERS, HEADER_TYPE, HTTP_STATUS } from '@verdaccio/core';
+
+import { getConf, initializeServer, initializeServerWithConfig, publishVersion } from './_helper';
+
+describe('experiments', () => {
+  describe('for a function value of tarball_url_redirect', () => {
+    let app;
+    beforeEach(async () => {
+      const baseTestConfig = getConf('experiments.yaml');
+      app = await initializeServerWithConfig({
+        ...baseTestConfig,
+        experiments: {
+          // @ts-ignore
+          tarball_url_redirect(context) {
+            return `https://myapp.sfo1.mycdn.com/verdaccio/${context.packageName}/${context.filename}`;
+          },
+        },
+      });
+      await publishVersion(app, '@tarball_tester/testTarballPackage', '1.0.0');
+      await publishVersion(app, 'testTarballPackage', '1.0.0');
+    });
+
+    test('should redirect for package tarball as function', (done) => {
+      supertest(app)
+        .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+
+    test('should redirect for scoped package tarball', (done) => {
+      supertest(app)
+        .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+  });
+  describe('for a async function value of tarball_url_redirect', () => {
+    let app;
+    beforeEach(async () => {
+      const baseTestConfig = getConf('experiments.yaml');
+      app = await initializeServerWithConfig({
+        ...baseTestConfig,
+        experiments: {
+          // @ts-ignore
+          async tarball_url_redirect(context) {
+            await new Promise((resolve) => {
+              setTimeout(resolve, 1000);
+            });
+            return `https://myapp.sfo1.mycdn.com/verdaccio/${context.packageName}/${context.filename}`;
+          },
+        },
+      });
+      await publishVersion(app, '@tarball_tester/testTarballPackage', '1.0.0');
+      await publishVersion(app, 'testTarballPackage', '1.0.0');
+    });
+
+    test('should redirect for package tarball', (done) => {
+      supertest(app)
+        .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+
+    test('should redirect for scoped package tarball', (done) => {
+      supertest(app)
+        .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+  });
+  describe('for a string value of tarball_url_redirect', () => {
+    let app;
+    beforeEach(async () => {
+      const baseTestConfig = getConf('experiments.yaml');
+      app = await initializeServerWithConfig({
+        ...baseTestConfig,
+        experiments: {
+          // @ts-ignore
+          tarball_url_redirect: 'https://myapp.sfo1.mycdn.com/verdaccio/${packageName}/${filename}',
+        },
+      });
+      await publishVersion(app, '@tarball_tester/testTarballPackage', '1.0.0');
+      await publishVersion(app, 'testTarballPackage', '1.0.0');
+    });
+
+    test('should redirect for scoped package tarball', (done) => {
+      supertest(app)
+        .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+
+    test('should redirect for package tarball', (done) => {
+      supertest(app)
+        .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
+        .expect(302)
+        .end(function (err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.headers.location).toEqual(
+            'https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'
+          );
+          done();
+        });
+    });
+  });
+});

test/unit/modules/api/legacy/api.__c.ts

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import nock from 'nock';
 import path from 'path';
-import rimraf from 'rimraf';
+import { rimrafSync } from 'rimraf';
 import { Readable } from 'stream';
 import request from 'supertest';
 
@@ -63,44 +63,43 @@ describe('endpoint unit test', () => {
   const mockServerPort = 55549;
   let mockRegistry;
 
-  beforeAll(function (done) {
+  beforeAll(async function () {
     const store = path.join(__dirname, '../../partials/store/test-storage-api-spec');
-    rimraf(store, async () => {
+    rimrafSync(store);
-      const configForTest = configDefault(
-        {
-          auth: {
-            htpasswd: {
-              file: './test-storage-api-spec/.htpasswd',
-            },
-          },
-          filters: {
-            '../../modules/api/partials/plugin/filter': {
-              pkg: 'npm_test',
-              version: '2.0.0',
-            },
-          },
-          storage: store,
-          self_path: store,
-          uplinks: {
-            npmjs: {
-              url: `http://${DOMAIN_SERVERS}:${mockServerPort}`,
-            },
-            socketTimeout: {
-              url: `http://some.registry.timeout.com`,
-              max_fails: 2,
-              timeout: '1s',
-              fail_timeout: '1s',
-            },
-          },
-          log: { type: 'stdout', format: 'pretty', level: 'warn' },
-        },
-        'api.spec.yaml'
-      );
 
-      app = await endPointAPI(configForTest);
+    const configForTest = configDefault(
-      mockRegistry = await mockServer(mockServerPort).init();
+      {
-      done();
+        auth: {
-    });
+          htpasswd: {
+            file: './test-storage-api-spec/.htpasswd',
+          },
+        },
+        filters: {
+          '../../modules/api/partials/plugin/filter': {
+            pkg: 'npm_test',
+            version: '2.0.0',
+          },
+        },
+        storage: store,
+        self_path: store,
+        uplinks: {
+          npmjs: {
+            url: `http://${DOMAIN_SERVERS}:${mockServerPort}`,
+          },
+          socketTimeout: {
+            url: `http://some.registry.timeout.com`,
+            max_fails: 2,
+            timeout: '1s',
+            fail_timeout: '1s',
+          },
+        },
+        log: { type: 'stdout', format: 'pretty', level: 'warn' },
+      },
+      'api.spec.yaml'
+    );
+
+    app = await endPointAPI(configForTest);
+    mockRegistry = await mockServer(mockServerPort).init();
   });
 
   afterAll(function (done) {
@@ -1016,136 +1015,6 @@ describe('endpoint unit test', () => {
       });
     });
 
-    describe('should test tarball url redirect', () => {
-      const pkgName = 'testTarballPackage';
-      const scopedPkgName = '@tarball_tester/testTarballPackage';
-      const tarballUrlRedirectCredentials = { name: 'tarball_tester', password: 'secretPass' };
-      const store = path.join(__dirname, '../../partials/store/test-storage-api-spec');
-      const mockServerPort = 55549;
-      const baseTestConfig = configDefault(
-        {
-          auth: {
-            htpasswd: {
-              file: './test-storage-api-spec/.htpasswd',
-            },
-          },
-          filters: {
-            '../../modules/api/partials/plugin/filter': {
-              pkg: 'npm_test',
-              version: '2.0.0',
-            },
-          },
-          storage: store,
-          self_path: store,
-          uplinks: {
-            npmjs: {
-              url: `http://${DOMAIN_SERVERS}:${mockServerPort}`,
-            },
-          },
-          log: { type: 'stdout', format: 'pretty', level: 'warn' },
-        },
-        'api.spec.yaml'
-      );
-      let token;
-      beforeAll(async () => {
-        token = await getNewToken(request(app), tarballUrlRedirectCredentials);
-        await putPackage(request(app), `/${pkgName}`, generatePackageMetadata(pkgName), token);
-        await putPackage(
-          request(app),
-          `/${scopedPkgName}`,
-          generatePackageMetadata(scopedPkgName),
-          token
-        );
-      });
-
-      describe('for a string value of tarball_url_redirect', () => {
-        let app2;
-        beforeAll(async () => {
-          app2 = await endPointAPI({
-            ...baseTestConfig,
-            experiments: {
-              tarball_url_redirect:
-                'https://myapp.sfo1.mycdn.com/verdaccio/${packageName}/${filename}',
-            },
-          });
-        });
-
-        test('should redirect for package tarball', (done) => {
-          request(app2)
-            .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
-            .expect(HTTP_STATUS.REDIRECT)
-            .end(function (err, res) {
-              if (err) {
-                return done(err);
-              }
-              expect(res.headers.location).toEqual(
-                'https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'
-              );
-              done();
-            });
-        });
-
-        test('should redirect for scoped package tarball', (done) => {
-          request(app2)
-            .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
-            .expect(HTTP_STATUS.REDIRECT)
-            .end(function (err, res) {
-              if (err) {
-                return done(err);
-              }
-              expect(res.headers.location).toEqual(
-                'https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'
-              );
-              done();
-            });
-        });
-      });
-
-      describe('for a function value of tarball_url_redirect', () => {
-        let app2;
-        beforeAll(async () => {
-          app2 = await endPointAPI({
-            ...baseTestConfig,
-            experiments: {
-              tarball_url_redirect(context) {
-                return `https://myapp.sfo1.mycdn.com/verdaccio/${context.packageName}/${context.filename}`;
-              },
-            },
-          });
-        });
-
-        test('should redirect for package tarball', (done) => {
-          request(app2)
-            .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
-            .expect(HTTP_STATUS.REDIRECT)
-            .end(function (err, res) {
-              if (err) {
-                return done(err);
-              }
-              expect(res.headers.location).toEqual(
-                'https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'
-              );
-              done();
-            });
-        });
-
-        test('should redirect for scoped package tarball', (done) => {
-          request(app2)
-            .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz')
-            .expect(HTTP_STATUS.REDIRECT)
-            .end(function (err, res) {
-              if (err) {
-                return done(err);
-              }
-              expect(res.headers.location).toEqual(
-                'https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'
-              );
-              done();
-            });
-        });
-      });
-    });
-
     describe('should test (un)deprecate api', () => {
       const pkgName = '@scope/deprecate';
       const credentials = { name: 'jota_deprecate', password: 'secretPass' };