From 78d04cf9f8b3071fe0f33b387cb08e01bfcf2d87 Mon Sep 17 00:00:00 2001 From: Favo Yang Date: Tue, 4 May 2021 02:53:00 +0800 Subject: [PATCH] feat: tarball url redirect (#1688) * feat: tarball url redirect * fix: handle uplinks * feat: allow function for config.tarball_url_redirect * fix: hasLocalTarball was calling localStream,abort when already aborted * chore: simplify localStream null check in hasLocalTarball As requested in PR feedback. * chore: fix sonarcloud code smell on test the variable `credentials` was already declared before the tarball url tests. * fix: move tarball_url_redirect to experiments Co-authored-by: Gord Lea Co-authored-by: Gord Lea --- conf/default.yaml | 7 ++ conf/docker.yaml | 7 ++ src/api/endpoint/api/package.ts | 37 +++++++++- src/lib/constants.ts | 1 + src/lib/storage.ts | 28 +++++++ test/unit/modules/api/api.spec.ts | 118 ++++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 3 deletions(-) diff --git a/conf/default.yaml b/conf/default.yaml index cff6ce644..480b7627e 100644 --- a/conf/default.yaml +++ b/conf/default.yaml @@ -83,6 +83,13 @@ logs: { type: stdout, format: pretty, level: http } # search: false # # disable writing body size to logs, read more on ticket 1912 # bytesin_off: false +# # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string +# tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}' +# # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file +# tarball_url_redirect(packageName, filename) { +# const signedUrl = // generate a signed url +# return signedUrl; +# } # This affect the web and api (not developed yet) #i18n: diff --git a/conf/docker.yaml b/conf/docker.yaml index 0e58041d6..3836e054e 100644 --- a/conf/docker.yaml +++ b/conf/docker.yaml @@ -86,6 +86,13 @@ logs: { type: stdout, format: pretty, level: http } # token: false # # support for the new v1 search endpoint, functional by incomplete read more on ticket 1732 # search: false +# # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string +# tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}' +# # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file +# tarball_url_redirect(packageName, filename) { +# const signedUrl = // generate a signed url +# return signedUrl; +# } # This affect the web and api (not developed yet) #i18n: diff --git a/src/api/endpoint/api/package.ts b/src/api/endpoint/api/package.ts index 894aad5d5..8107c2976 100644 --- a/src/api/endpoint/api/package.ts +++ b/src/api/endpoint/api/package.ts @@ -33,6 +33,30 @@ const downloadStream = ( stream.pipe(res); }; +const redirectOrDownloadStream = ( + packageName: string, + filename: string, + storage: any, + req: $RequestExtend, + res: $ResponseExtend, + config: Config +): void => { + const tarballUrlRedirect = _.get(config, 'experiments.tarball_url_redirect'); + storage.hasLocalTarball(packageName, filename).then(hasLocalTarball => { + if (hasLocalTarball) { + const context = { packageName, filename }; + const tarballUrl = typeof tarballUrlRedirect === 'function' + ? tarballUrlRedirect(context) + : _.template(tarballUrlRedirect)(context); + res.redirect(tarballUrl); + } else { + downloadStream(packageName, filename, storage, req, res) + } + }).catch(err => { + res.locals.report_error(err); + }); +} + export default function ( route: Router, auth: IAuth, @@ -87,8 +111,11 @@ export default function ( can('access'), function (req: $RequestExtend, res: $ResponseExtend): void { const { scopedPackage, filename } = req.params; - - downloadStream(scopedPackage, filename, storage, req, res); + if (_.get(config, 'experiments.tarball_url_redirect') === undefined) { + downloadStream(scopedPackage, filename, storage, req, res); + } else { + redirectOrDownloadStream(scopedPackage, filename, storage, req, res, config); + } } ); @@ -96,7 +123,11 @@ export default function ( '/:package/-/:filename', can('access'), function (req: $RequestExtend, res: $ResponseExtend): void { - downloadStream(req.params.package, req.params.filename, storage, req, res); + if (_.get(config, 'experiments.tarball_url_redirect') === undefined) { + downloadStream(req.params.package, req.params.filename, storage, req, res); + } else { + redirectOrDownloadStream(req.params.package, req.params.filename, storage, req, res, config); + } } ); } diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 96495fd19..47db0605b 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -76,6 +76,7 @@ export const HTTP_STATUS = { OK: 200, CREATED: 201, MULTIPLE_CHOICES: 300, + REDIRECT: 302, NOT_MODIFIED: 304, BAD_REQUEST: 400, UNAUTHORIZED: 401, diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 72d2953c0..92864a6f3 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -176,6 +176,34 @@ class Storage implements IStorageHandler { return this.localStorage.addTarball(name, filename); } + public hasLocalTarball(name: string, filename: string): Promise { + const self = this; + return new Promise((resolve, reject): void => { + let localStream: any = self.localStorage.getTarball(name, filename); + let isOpen = false; + localStream.on( + 'error', + (err): any => { + if (isOpen || err.status !== HTTP_STATUS.NOT_FOUND) { + reject(err); + } + // local reported 404 or request was aborted already + if (localStream) { + localStream.abort(); + localStream = null; + } + resolve(false); + } + ); + localStream.on('open', function(): void { + isOpen = true; + localStream.abort(); + localStream = null; + resolve(true); + }); + }); + } + /** Get a tarball from a storage for {name} package Function is synchronous and returns a ReadableStream diff --git a/test/unit/modules/api/api.spec.ts b/test/unit/modules/api/api.spec.ts index 3d5a21e7d..e76097332 100644 --- a/test/unit/modules/api/api.spec.ts +++ b/test/unit/modules/api/api.spec.ts @@ -978,6 +978,124 @@ describe('endpoint unit test', () => { }); }); + describe('should test tarball url redirect', () => { + const pkgName = 'testTarballPackage'; + const scopedPkgName = '@tarball_tester/testTarballPackage'; + const tarballUrlRedirectCredentials = { name: 'tarball_tester', password: 'secretPass' }; + const store = path.join(__dirname, '../../partials/store/test-storage-api-spec'); + const mockServerPort = 55549; + const baseTestConfig = configDefault({ + auth: { + htpasswd: { + file: './test-storage-api-spec/.htpasswd' + } + }, + filters: { + '../../modules/api/partials/plugin/filter': { + pkg: 'npm_test', + version: '2.0.0' + } + }, + storage: store, + self_path: store, + uplinks: { + npmjs: { + url: `http://${DOMAIN_SERVERS}:${mockServerPort}` + } + }, + logs: [ + { type: 'stdout', format: 'pretty', level: 'warn' } + ], + }, 'api.spec.yaml'); + let token; + beforeAll(async (done) => { + token = await getNewToken(request(app), tarballUrlRedirectCredentials); + await putPackage(request(app), `/${pkgName}`, generatePackageMetadata(pkgName), token); + await putPackage(request(app), `/${scopedPkgName}`, generatePackageMetadata(scopedPkgName), token); + done(); + }); + + describe('for a string value of tarball_url_redirect', () => { + let app2; + beforeAll(async (done) => { + app2 = await endPointAPI({ + ...baseTestConfig, + experiments: { + tarball_url_redirect: 'https://myapp.sfo1.mycdn.com/verdaccio/${packageName}/${filename}' + } + }); + done(); + }); + + test('should redirect for package tarball', (done) => { + request(app2) + .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz') + .expect(HTTP_STATUS.REDIRECT) + .end(function (err, res) { + if (err) { + return done(err); + } + expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'); + done(); + }); + }); + + test('should redirect for scoped package tarball', (done) => { + request(app2) + .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz') + .expect(HTTP_STATUS.REDIRECT) + .end(function (err, res) { + if (err) { + return done(err); + } + expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'); + done(); + }); + }); + }); + + describe('for a function value of tarball_url_redirect', () => { + let app2; + beforeAll(async (done) => { + app2 = await endPointAPI({ + ...baseTestConfig, + experiments: { + tarball_url_redirect(context) { + return `https://myapp.sfo1.mycdn.com/verdaccio/${context.packageName}/${context.filename}` + } + } + }); + done(); + }); + + test('should redirect for package tarball', (done) => { + request(app2) + .get('/testTarballPackage/-/testTarballPackage-1.0.0.tgz') + .expect(HTTP_STATUS.REDIRECT) + .end(function (err, res) { + if (err) { + return done(err); + } + expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/testTarballPackage/testTarballPackage-1.0.0.tgz'); + done(); + }); + }); + + test('should redirect for scoped package tarball', (done) => { + request(app2) + .get('/@tarball_tester/testTarballPackage/-/testTarballPackage-1.0.0.tgz') + .expect(HTTP_STATUS.REDIRECT) + .end(function (err, res) { + if (err) { + return done(err); + } + expect(res.headers.location).toEqual('https://myapp.sfo1.mycdn.com/verdaccio/@tarball_tester/testTarballPackage/testTarballPackage-1.0.0.tgz'); + done(); + }); + }); + }); + }); + describe('should test (un)deprecate api', () => { const pkgName = '@scope/deprecate'; const credentials = { name: 'jota_deprecate', password: 'secretPass' };