diff --git a/.changeset/gold-files-speak.md b/.changeset/gold-files-speak.md new file mode 100644 index 000000000..01dfa97c4 --- /dev/null +++ b/.changeset/gold-files-speak.md @@ -0,0 +1,9 @@ +--- +'@verdaccio/config': patch +'@verdaccio/search': patch +'@verdaccio/proxy': patch +'@verdaccio/store': patch +'@verdaccio/utils': patch +--- + +fix: uplink processing order diff --git a/packages/config/src/uplinks.ts b/packages/config/src/uplinks.ts index 467d9bf2c..15c328ad9 100644 --- a/packages/config/src/uplinks.ts +++ b/packages/config/src/uplinks.ts @@ -48,9 +48,13 @@ export function sanityCheckUplinksProps(configUpLinks: UpLinksConfList): UpLinks return uplinks; } -export function hasProxyTo(pkg: string, upLink: string, packages: PackageList): boolean { +export function getProxiesForPackage(pkg: string, packages: PackageList): string[] { const matchedPkg = getMatchedPackagesSpec(pkg, packages); - const proxyList = typeof matchedPkg !== 'undefined' ? matchedPkg.proxy : []; + return matchedPkg?.proxy || []; +} + +export function hasProxyTo(pkg: string, upLink: string, packages: PackageList): boolean { + const proxyList = getProxiesForPackage(pkg, packages); if (proxyList) { return proxyList.some((curr) => upLink === curr); } diff --git a/packages/config/test/package-access.spec.ts b/packages/config/test/package-access.spec.ts index 99861e7ba..ec03d4007 100644 --- a/packages/config/test/package-access.spec.ts +++ b/packages/config/test/package-access.spec.ts @@ -80,6 +80,25 @@ describe('Package access utilities', () => { expect(all.publish).toContain('admin'); }); + test('should test multi proxy definition', () => { + const { packages } = parseConfigFile(parseConfigurationFile('pkgs-multi-proxy')); + const access = normalisePackageAccess(packages); + + expect(access).toBeDefined(); + const scoped = access[`${PACKAGE_ACCESS.SCOPE}`]; + const all = access[`${PACKAGE_ACCESS.ALL}`]; + const testPackage = access['test-package']; + + expect(scoped).toBeDefined(); + expect(scoped.proxy).toEqual(['github', 'npmjs']); + + expect(all).toBeDefined(); + expect(all.proxy).toEqual(['npmjs', 'gitlab']); + + expect(testPackage).toBeDefined(); + expect(testPackage.proxy).toEqual(['npmjs', 'gitlab', 'github']); + }); + test( 'should normalize deprecated packages into the new ones ' + '(backward props compatible)', () => { @@ -92,7 +111,7 @@ describe('Package access utilities', () => { expect(react).toBeDefined(); expect(react.access).toBeDefined(); expect(react.access).toEqual([]); - expect(react.publish[0]).toBe('admin'); + expect(react.publish?.[0]).toBe('admin'); expect(react.proxy).toBeDefined(); expect(react.proxy).toEqual([]); expect(react.storage).toBeDefined(); diff --git a/packages/config/test/partials/config/yaml/pkgs-multi-proxy.yaml b/packages/config/test/partials/config/yaml/pkgs-multi-proxy.yaml new file mode 100644 index 000000000..ab565bde7 --- /dev/null +++ b/packages/config/test/partials/config/yaml/pkgs-multi-proxy.yaml @@ -0,0 +1,26 @@ +uplinks: + github: + url: 'https://npm.pkg.github.com/' + auth: + type: Bearer + token: 'xxx123xxx' + gitlab: + url: 'https://gitlab.com/api/v4/projects/1/packages/npm/' + auth: + type: Basic + token: 'xxx456xxx' + npmjs: + url: https://registry.npmjs.org/ + +# Multiple proxies +# https://verdaccio.org/docs/packages/#use-multiple-uplinks +packages: + '@*/*': + access: $all + proxy: github npmjs + 'test-package': + access: $all + proxy: npmjs gitlab github + '**': + access: $all + proxy: npmjs gitlab diff --git a/packages/config/test/uplinks.spec.ts b/packages/config/test/uplinks.spec.ts index be7b05e85..8d9a117d9 100644 --- a/packages/config/test/uplinks.spec.ts +++ b/packages/config/test/uplinks.spec.ts @@ -1,7 +1,12 @@ import { describe, expect, test } from 'vitest'; import { normalisePackageAccess, parseConfigFile } from '../src'; -import { hasProxyTo, sanityCheckUplinksProps, uplinkSanityCheck } from '../src/uplinks'; +import { + getProxiesForPackage, + hasProxyTo, + sanityCheckUplinksProps, + uplinkSanityCheck, +} from '../src/uplinks'; import { parseConfigurationFile } from './utils'; describe('Uplinks Utilities', () => { @@ -24,7 +29,7 @@ describe('Uplinks Utilities', () => { }); describe('sanityCheckUplinksProps', () => { - test('should fails if url prop is missing', () => { + test('should fail if url prop is missing', () => { const { uplinks } = parseConfigFile(parseConfigurationFile('uplink-wrong')); expect(() => { sanityCheckUplinksProps(uplinks); @@ -37,6 +42,31 @@ describe('Uplinks Utilities', () => { }); }); + describe('multiple uplinks with auth', () => { + test('should fail if url or auth are missing', () => { + const { uplinks } = parseConfigFile(parseConfigurationFile('pkgs-multi-proxy')); + expect(sanityCheckUplinksProps(uplinks)).toEqual(uplinks); + expect(Object.keys(uplinks)).toHaveLength(3); + // No trailing slash in urls + expect(uplinks.github.url).toEqual('https://npm.pkg.github.com'); + expect(uplinks.github?.auth?.type).toEqual('Bearer'); + expect(uplinks.github?.auth?.token).toEqual('xxx123xxx'); + expect(uplinks.gitlab.url).toEqual('https://gitlab.com/api/v4/projects/1/packages/npm'); + expect(uplinks.gitlab?.auth?.type).toEqual('Basic'); + expect(uplinks.gitlab?.auth?.token).toEqual('xxx456xxx'); + expect(uplinks.npmjs.url).toEqual('https://registry.npmjs.org'); + }); + + test('check proxy list for package access', () => { + const packages = normalisePackageAccess( + parseConfigFile(parseConfigurationFile('pkgs-multi-proxy')).packages + ); + expect(getProxiesForPackage('@scope/test', packages)).toEqual(['github', 'npmjs']); + expect(getProxiesForPackage('test', packages)).toEqual(['npmjs', 'gitlab']); + expect(getProxiesForPackage('test-package', packages)).toEqual(['npmjs', 'gitlab', 'github']); + }); + }); + describe('hasProxyTo', () => { test('should test basic config', () => { const packages = normalisePackageAccess( diff --git a/packages/proxy/src/proxy.ts b/packages/proxy/src/proxy.ts index ec7712ec6..779868f30 100644 --- a/packages/proxy/src/proxy.ts +++ b/packages/proxy/src/proxy.ts @@ -60,6 +60,7 @@ export type ProxySearchParams = { retry?: Partial; }; export interface IProxy { + uplinkName: string; config: UpLinkConfLocal; failed_requests: number; userAgent: string; @@ -71,7 +72,6 @@ export interface IProxy { timeout: Delays; max_fails: number; fail_timeout: number; - upname: string; search(options: ProxySearchParams): Promise; getRemoteMetadata( name: string, @@ -97,6 +97,7 @@ export interface ISyncUplinksOptions extends Options { * (same for storage.js, local-storage.js, up-storage.js) */ class ProxyStorage implements IProxy { + public uplinkName: string; public config: UpLinkConfLocal; public failed_requests: number; public userAgent: string; @@ -109,9 +110,6 @@ class ProxyStorage implements IProxy { public max_fails: number; public fail_timeout: number; public agent_options: AgentOptionsConf; - // FIXME: upname is assigned to each instance - // @ts-ignore - public upname: string; public proxy: string | undefined; private agent: Agents; // @ts-ignore @@ -119,7 +117,14 @@ class ProxyStorage implements IProxy { public strict_ssl: boolean; private retry: Partial; - public constructor(config: UpLinkConfLocal, mainConfig: Config, logger: Logger, agent?: Agents) { + public constructor( + uplinkName: string, + config: UpLinkConfLocal, + mainConfig: Config, + logger: Logger, + agent?: Agents + ) { + this.uplinkName = uplinkName; this.config = config; this.failed_requests = 0; this.userAgent = mainConfig.user_agent ?? 'hidden'; @@ -205,17 +210,21 @@ class ProxyStorage implements IProxy { let token: any; const tokenConf: any = auth; if (_.isNil(tokenConf.token) === false && _.isString(tokenConf.token)) { + debug('use token from config'); token = tokenConf.token; } else if (_.isNil(tokenConf.token_env) === false) { if (typeof tokenConf.token_env === 'string') { + debug('use token from env %o', tokenConf.token_env); token = process.env[tokenConf.token_env]; } else if (typeof tokenConf.token_env === 'boolean' && tokenConf.token_env) { + debug('use token from env NPM_TOKEN'); token = process.env.NPM_TOKEN; } else { this.logger.error(constants.ERROR_CODE.token_required); this._throwErrorAuth(constants.ERROR_CODE.token_required); } } else { + debug('use token from env NPM_TOKEN'); token = process.env.NPM_TOKEN; } @@ -225,6 +234,7 @@ class ProxyStorage implements IProxy { // define type Auth allow basic and bearer const type = tokenConf.type || TOKEN_BASIC; + debug('token type %o', type); this._setHeaderAuthorization(headers, type, token); return headers; @@ -254,7 +264,6 @@ class ProxyStorage implements IProxy { this._throwErrorAuth(`Auth type '${_type}' not allowed`); } - type = _.upperFirst(type); headers[HEADERS.AUTHORIZATION] = buildToken(type, token); } @@ -276,20 +285,7 @@ class ProxyStorage implements IProxy { * @param {Object} headers * @private - * @deprecated use applyUplinkHeaders */ - private _overrideWithUpLinkConfLocaligHeaders(headers: Headers): any { - if (!this.config.headers) { - return headers; - } - - // add/override headers specified in the config - /* eslint guard-for-in: 0 */ - for (const key in this.config.headers) { - headers[key] = this.config.headers[key]; - } - } - private applyUplinkHeaders(headers: gotHeaders): gotHeaders { if (!this.config.headers) { return headers; @@ -504,7 +500,10 @@ class ProxyStorage implements IProxy { try { // Incoming URL is relative ie /-/v1/search... const uri = new URL(url, this.url).href; - this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}'); + this.logger.http( + { uri, uplink: this.uplinkName }, + 'search request to uplink @{uplink} - @{uri}' + ); debug('searching on %o', uri); const response = got(uri, { signal: abort ? abort.signal : {}, @@ -527,7 +526,7 @@ class ProxyStorage implements IProxy { throw errorUtils.getInternalError(`bad status code ${err.response.statusCode} from uplink`); } this.logger.error( - { errorMessage: err?.message, name: this.upname }, + { errorMessage: err?.message, name: this.uplinkName }, 'proxy uplink @{name} search error: @{errorMessage}' ); throw err; diff --git a/packages/proxy/src/uplink-util.ts b/packages/proxy/src/uplink-util.ts index 0f8afac64..ad97bfc0f 100644 --- a/packages/proxy/src/uplink-util.ts +++ b/packages/proxy/src/uplink-util.ts @@ -12,15 +12,10 @@ export interface ProxyInstanceList { export function setupUpLinks(config: Config, logger: Logger): ProxyInstanceList { const uplinks: ProxyInstanceList = {}; - for (const uplinkName in config.uplinks) { - if (Object.prototype.hasOwnProperty.call(config.uplinks, uplinkName)) { - // instance for each up-link definition - const proxy: IProxy = new ProxyStorage(config.uplinks[uplinkName], config, logger); - // TODO: review this can be inside ProxyStorage - proxy.upname = uplinkName; - - uplinks[uplinkName] = proxy; - } + for (const uplinkName of Object.keys(config.uplinks)) { + // instance for each up-link definition + const proxy: IProxy = new ProxyStorage(uplinkName, config.uplinks[uplinkName], config, logger); + uplinks[uplinkName] = proxy; } return uplinks; @@ -35,7 +30,7 @@ export function updateVersionsHiddenUpLinkNext(manifest: Manifest, upLink: IProx for (const version of versionsList) { // holds a "hidden" value to be used by the package storage. - versions[version][Symbol.for('__verdaccio_uplink')] = upLink.upname; + versions[version][Symbol.for('__verdaccio_uplink')] = upLink.uplinkName; } return { ...manifest, versions }; diff --git a/packages/proxy/test/conf/multi-proxy.yaml b/packages/proxy/test/conf/multi-proxy.yaml new file mode 100644 index 000000000..b3fe52e69 --- /dev/null +++ b/packages/proxy/test/conf/multi-proxy.yaml @@ -0,0 +1,23 @@ +uplinks: + github: + url: 'https://npm.pkg.github.com/' + auth: + type: Bearer + token: 'xxx123xxx' + gitlab: + url: 'https://gitlab.com/api/v4/projects/1/packages/npm/' + auth: + type: Basic + token: 'xxx456xxx' + npmjs: + url: https://registry.npmjs.org/ + +# Multiple proxies +# https://verdaccio.org/docs/packages/#use-multiple-uplinks +packages: + '@*/*': + access: $all + proxy: github npmjs + '**': + access: $all + proxy: npmjs gitlab diff --git a/packages/proxy/test/headers.auth.spec.ts b/packages/proxy/test/headers.auth.spec.ts index 3187a5e8e..17e14f86b 100644 --- a/packages/proxy/test/headers.auth.spec.ts +++ b/packages/proxy/test/headers.auth.spec.ts @@ -27,7 +27,7 @@ function createUplink(config) { }; const mergeConfig = Object.assign({}, defaultConfig, config); // @ts-ignore - return new ProxyStorage(mergeConfig, {}, logger); + return new ProxyStorage('npmjs', mergeConfig, {}, logger); } function setHeadersNext(config: unknown = {}, headers: any = {}) { @@ -97,6 +97,18 @@ describe('setHeadersNext', () => { expect(headers[HEADERS.AUTHORIZATION]).toEqual(buildToken(TOKEN_BASIC, 'Zm9vX2Jhcg==')); }); + test('set type lower case', () => { + const headers = setHeadersNext({ + auth: { + type: 'basic', // lower case type + token: 'test', + }, + }); + + expect(Object.keys(headers)).toHaveLength(4); + expect(headers[HEADERS.AUTHORIZATION]).toEqual(buildToken(TOKEN_BASIC, 'test')); // capital case type + }); + test('set type auth bearer', () => { const headers = setHeadersNext({ auth: { diff --git a/packages/proxy/test/noProxy.spec.ts b/packages/proxy/test/noProxy.spec.ts index 98c19afaf..c1f192bd9 100644 --- a/packages/proxy/test/noProxy.spec.ts +++ b/packages/proxy/test/noProxy.spec.ts @@ -8,8 +8,8 @@ setup({}); function getProxyInstance(host, uplinkConf, appConfig) { uplinkConf.url = host; - - return new ProxyStorage(uplinkConf, appConfig, logger); + const uplinkName = host.replace(/^https?:\/\//, ''); + return new ProxyStorage(uplinkName, uplinkConf, appConfig, logger); } describe('Use proxy', () => { diff --git a/packages/proxy/test/proxy.error.___.ts b/packages/proxy/test/proxy.error.___.ts index bd95f1fef..4d8d8e4a2 100644 --- a/packages/proxy/test/proxy.error.___.ts +++ b/packages/proxy/test/proxy.error.___.ts @@ -45,7 +45,7 @@ // test('should be offline uplink', (done) => { // const tarball = 'https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'; // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').times(100).replyWithError('some error'); -// const proxy = new ProxyStorage(defaultRequestOptions, conf); +// const proxy = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = proxy.fetchTarball(tarball); // // to test a uplink is offline we have to be try 3 times // // the default failed request are set to 2 @@ -79,7 +79,7 @@ // test('not found tarball', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(404); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getNotFound(API_ERROR.NOT_FILE_UPLINK)); @@ -89,7 +89,7 @@ // test('fail tarball request', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').replyWithError('boom file'); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(Error('boom file')); @@ -99,7 +99,7 @@ // test('bad uplink request', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(409); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getInternalError(`bad uplink status code: 409`)); @@ -115,7 +115,7 @@ // .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), { // [HEADER_TYPE.CONTENT_LENGTH]: 0, // }); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getInternalError(API_ERROR.CONTENT_MISMATCH)); diff --git a/packages/proxy/test/proxy.metadata.spec.ts b/packages/proxy/test/proxy.metadata.spec.ts index 18139b4ce..637b537f9 100644 --- a/packages/proxy/test/proxy.metadata.spec.ts +++ b/packages/proxy/test/proxy.metadata.spec.ts @@ -57,7 +57,7 @@ describe('proxy', () => { }) .get('/jquery') .reply(200, { body: 'test' }); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const [manifest] = await prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', }); @@ -83,7 +83,7 @@ describe('proxy', () => { etag: () => `_ref_4444`, } ); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const [manifest, etag] = await prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', }); @@ -110,7 +110,7 @@ describe('proxy', () => { etag: () => `_ref_4444`, } ); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const [manifest, etag] = await prox1.getRemoteMetadata('jquery', { etag: 'foo', remoteAddress: '127.0.0.1', @@ -125,7 +125,7 @@ describe('proxy', () => { nock(domain) .get('/jquery') .reply(200, { body: { name: 'foo', version: '1.0.0' } }, {}); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', }); @@ -154,7 +154,7 @@ describe('proxy', () => { describe('error handling', () => { test('proxy call with 304', async () => { nock(domain).get('/jquery').reply(304); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect(prox1.getRemoteMetadata('jquery', { etag: 'rev_3333' })).rejects.toThrow( 'no data' ); @@ -162,7 +162,7 @@ describe('proxy', () => { test('reply with error', async () => { nock(domain).get('/jquery').replyWithError('something awful happened'); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', @@ -172,7 +172,7 @@ describe('proxy', () => { test('reply with 409 error', async () => { nock(domain).get('/jquery').reply(409); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect(prox1.getRemoteMetadata('jquery', { retry: { limit: 0 } })).rejects.toThrow( /bad status code: 409/ ); @@ -180,7 +180,7 @@ describe('proxy', () => { test('reply with bad body json format', async () => { nock(domain).get('/jquery').reply(200, 'some-text'); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', @@ -190,7 +190,7 @@ describe('proxy', () => { test('400 error proxy call', async () => { nock(domain).get('/jquery').reply(409); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', @@ -200,7 +200,7 @@ describe('proxy', () => { test('proxy not found', async () => { nock(domain).get('/jquery').reply(404); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', @@ -227,7 +227,7 @@ describe('proxy', () => { .once() .reply(200, { body: { name: 'foo', version: '1.0.0' } }); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const [manifest] = await prox1.getRemoteMetadata('jquery', { retry: { limit: 2 }, }); @@ -246,7 +246,7 @@ describe('proxy', () => { test('retry count is exceded and uplink goes offline with logging activity', async () => { nock(domain).get('/jquery').times(10).reply(500); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { remoteAddress: '127.0.0.1', @@ -280,6 +280,7 @@ describe('proxy', () => { .reply(200, { body: { name: 'foo', version: '1.0.0' } }); const prox1 = new ProxyStorage( + 'uplink', { ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 }, conf, logger @@ -337,7 +338,7 @@ describe('proxy', () => { const confTimeout = { ...defaultRequestOptions }; // @ts-expect-error confTimeout.timeout = '2s'; - const prox1 = new ProxyStorage(confTimeout, conf, logger); + const prox1 = new ProxyStorage('uplink', confTimeout, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { retry: { limit: 0 }, @@ -357,7 +358,7 @@ describe('proxy', () => { const confTimeout = { ...defaultRequestOptions }; // @ts-expect-error confTimeout.timeout = '2s'; - const prox1 = new ProxyStorage(confTimeout, conf, logger); + const prox1 = new ProxyStorage('uplink', confTimeout, conf, logger); await expect( prox1.getRemoteMetadata('jquery', { retry: { limit: 1 }, @@ -368,7 +369,7 @@ describe('proxy', () => { // test('retry count is exceded and uplink goes offline with logging activity', async () => { // nock(domain).get('/jquery').times(10).reply(500); - // const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + // const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf, logger); // await expect( // prox1.getRemoteMetadata('jquery', { // remoteAddress: '127.0.0.1', diff --git a/packages/proxy/test/proxy.protocol.spec.ts b/packages/proxy/test/proxy.protocol.spec.ts index 5c054f7c6..863f17ea1 100644 --- a/packages/proxy/test/proxy.protocol.spec.ts +++ b/packages/proxy/test/proxy.protocol.spec.ts @@ -21,7 +21,7 @@ const logger = { function getProxyInstance(host, uplinkConf, appConfig) { uplinkConf.url = host; - return new ProxyStorage(uplinkConf, appConfig, logger); + return new ProxyStorage('uplink', uplinkConf, appConfig, logger); } describe('Check protocol of proxy', () => { diff --git a/packages/proxy/test/proxy.search.spec.ts b/packages/proxy/test/proxy.search.spec.ts index e361f6e82..a1f8ef0eb 100644 --- a/packages/proxy/test/proxy.search.spec.ts +++ b/packages/proxy/test/proxy.search.spec.ts @@ -44,7 +44,7 @@ describe('proxy', () => { nock(domain) .get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio') .reply(200, response); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const abort = new AbortController(); const stream = await prox1.search({ abort, @@ -60,7 +60,7 @@ describe('proxy', () => { nock(domain + '/') .get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio') .reply(200, response); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const abort = new AbortController(); const stream = await prox1.search({ abort, @@ -76,7 +76,7 @@ describe('proxy', () => { .get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio') .reply(409); const abort = new AbortController(); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); await expect( prox1.search({ abort, @@ -98,7 +98,7 @@ describe('proxy', () => { // const mockClient = mockAgent.get(domain); // mockClient.intercept(options).reply(500, {}); // const abort = new AbortController(); - // const prox1 = new ProxyStorage(defaultRequestOptions, conf); + // const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // await expect( // prox1.search({ // abort, diff --git a/packages/proxy/test/proxy.tarball.spec.ts b/packages/proxy/test/proxy.tarball.spec.ts index 3276df281..3843b0c2a 100644 --- a/packages/proxy/test/proxy.tarball.spec.ts +++ b/packages/proxy/test/proxy.tarball.spec.ts @@ -46,7 +46,7 @@ describe('tarball proxy', () => { nock('https://registry.verdaccio.org') .get('/jquery/-/jquery-0.0.1.tgz') .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz')); - const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const stream = prox1.fetchTarball( 'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz', // @ts-expect-error @@ -69,7 +69,7 @@ describe('tarball proxy', () => { .get('/jquery/-/jquery-0.0.1.tgz') .once() .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz')); - const prox1 = new ProxyStorage(defaultRequestOptions, conf); + const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger); const stream = prox1.fetchTarball( 'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz', { retry: { limit: 2 } } @@ -89,7 +89,7 @@ describe('tarball proxy', () => { // .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), { // [HEADER_TYPE.CONTENT_LENGTH]: 277, // }); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on(HEADER_TYPE.CONTENT_LENGTH, (data) => { // expect(data).toEqual('277'); @@ -101,7 +101,7 @@ describe('tarball proxy', () => { // test('should be offline uplink', (done) => { // const tarball = 'https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'; // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').times(100).replyWithError('some error'); -// const proxy = new ProxyStorage(defaultRequestOptions, conf); +// const proxy = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = proxy.fetchTarball(tarball); // // to test a uplink is offline we have to be try 3 times // // the default failed request are set to 2 @@ -135,7 +135,7 @@ describe('tarball proxy', () => { // test('not found tarball', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(404); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getNotFound(API_ERROR.NOT_FILE_UPLINK)); @@ -145,7 +145,7 @@ describe('tarball proxy', () => { // test('fail tarball request', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').replyWithError('boom file'); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(Error('boom file')); @@ -155,7 +155,7 @@ describe('tarball proxy', () => { // test('bad uplink request', (done) => { // nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(409); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getInternalError(`bad uplink status code: 409`)); @@ -171,7 +171,7 @@ describe('tarball proxy', () => { // .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), { // [HEADER_TYPE.CONTENT_LENGTH]: 0, // }); -// const prox1 = new ProxyStorage(defaultRequestOptions, conf); +// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf); // const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz'); // stream.on('error', (response) => { // expect(response).toEqual(errorUtils.getInternalError(API_ERROR.CONTENT_MISMATCH)); diff --git a/packages/proxy/test/uplink-util.spec.ts b/packages/proxy/test/uplink-util.spec.ts new file mode 100644 index 000000000..5af3304dd --- /dev/null +++ b/packages/proxy/test/uplink-util.spec.ts @@ -0,0 +1,57 @@ +import path from 'path'; +import { describe, expect, test, vi } from 'vitest'; + +import { Config, parseConfigFile } from '@verdaccio/config'; +import { TOKEN_BASIC, TOKEN_BEARER } from '@verdaccio/core'; +import { Logger } from '@verdaccio/types'; + +import { IProxy } from '../src/index'; +import { setupUpLinks } from '../src/uplink-util'; + +const getConf = (name) => path.join(__dirname, '/conf', name); + +const mockDebug = vi.fn(); +const mockInfo = vi.fn(); +const mockHttp = vi.fn(); +const mockError = vi.fn(); +const mockWarn = vi.fn(); + +const logger = { + debug: mockDebug, + info: mockInfo, + http: mockHttp, + error: mockError, + warn: mockWarn, +} as unknown as Logger; + +describe('setupUpLinks', () => { + test('should create uplinks for each proxy configuration', () => { + const proxyPath = getConf('multi-proxy.yaml'); + const config = new Config(parseConfigFile(proxyPath)); + + const uplinks = setupUpLinks(config, logger); + + expect(Object.keys(uplinks)).toHaveLength(3); + expect(uplinks).toHaveProperty('github'); + expect(uplinks).toHaveProperty('gitlab'); + expect(uplinks).toHaveProperty('npmjs'); + + const githubProxy = uplinks.github as IProxy; + expect(githubProxy.uplinkName).toBe('github'); + expect(githubProxy.config.auth).toEqual({ + type: TOKEN_BEARER, + token: 'xxx123xxx', + }); + + const gitlabProxy = uplinks.gitlab as IProxy; + expect(gitlabProxy.uplinkName).toBe('gitlab'); + expect(gitlabProxy.config.auth).toEqual({ + type: TOKEN_BASIC, + token: 'xxx456xxx', + }); + + const npmjsProxy = uplinks.npmjs as IProxy; + expect(npmjsProxy.uplinkName).toBe('npmjs'); + expect(npmjsProxy.config.auth).toBeUndefined(); + }); +}); diff --git a/packages/search/src/search.ts b/packages/search/src/search.ts index 97e267270..bf49e052f 100644 --- a/packages/search/src/search.ts +++ b/packages/search/src/search.ts @@ -36,13 +36,13 @@ class Search { // const transformResults = new TransFormResults({ objectMode: true }); const streamPassThrough = new PassThrough({ objectMode: true }); debug('uplinks found %s', upLinkList.length); - const searchUplinksStreams = upLinkList.map((uplinkId: string) => { - const uplink = this.uplinks[uplinkId]; + const searchUplinksStreams = upLinkList.map((uplinkName: string) => { + const uplink = this.uplinks[uplinkName]; if (!uplink) { // this line should never happens - this.logger.error({ uplinkId }, 'uplink @upLinkId not found'); + this.logger.error({ uplinkName }, 'uplink @uplinkName not found'); } - return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough); + return this.consumeSearchStream(uplinkName, uplink, options, streamPassThrough); }); try { @@ -82,7 +82,7 @@ class Search { * Consume the upstream and pipe it to a transformable stream. */ private consumeSearchStream( - uplinkId: string, + uplinkName: string, uplink: IProxy, options: ProxySearchParams, searchPassThrough: PassThrough @@ -91,8 +91,8 @@ class Search { bodyStream.pipe(searchPassThrough, { end: false }); bodyStream.on('error', (err: any): void => { this.logger.error( - { uplinkId, err: err }, - 'search error for uplink @{uplinkId}: @{err?.message}' + { uplinkName, err: err }, + 'search error for uplink @{uplinkName}: @{err?.message}' ); searchPassThrough.end(); }); diff --git a/packages/store/package.json b/packages/store/package.json index 1836fe2f0..7fcc12a56 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -34,7 +34,7 @@ }, "scripts": { "clean": "rimraf ./build", - "test": "vitest run", + "test": "vitest run --testTimeout 20000", "type-check": "tsc --noEmit -p tsconfig.build.json", "build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json", "build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps", diff --git a/packages/store/src/lib/storage-utils.ts b/packages/store/src/lib/storage-utils.ts index 584d396dc..6a2566eec 100644 --- a/packages/store/src/lib/storage-utils.ts +++ b/packages/store/src/lib/storage-utils.ts @@ -213,10 +213,10 @@ export function mergeUplinkTimeIntoLocalNext( return cacheManifest; } -export function updateUpLinkMetadata(uplinkId, manifest: Manifest, etag: string) { +export function updateUpLinkMetadata(uplinkName: string, manifest: Manifest, etag: string) { const _uplinks = { ...manifest._uplinks, - [uplinkId]: { + [uplinkName]: { etag, fetched: Date.now(), }, diff --git a/packages/store/src/storage.ts b/packages/store/src/storage.ts index 1d4683413..31a7e0e32 100644 --- a/packages/store/src/storage.ts +++ b/packages/store/src/storage.ts @@ -6,7 +6,7 @@ import { PassThrough, Readable, Transform } from 'stream'; import { pipeline } from 'stream/promises'; import { default as URL } from 'url'; -import { hasProxyTo } from '@verdaccio/config'; +import { getProxiesForPackage, hasProxyTo } from '@verdaccio/config'; import { API_ERROR, API_MESSAGE, @@ -316,10 +316,19 @@ class Storage { return; } + if (res.statusCode === HTTP_STATUS.UNAUTHORIZED) { + debug('remote stream response 401'); + passThroughRemoteStream.emit( + 'error', + errorUtils.getUnauthorized(errorUtils.API_ERROR.UNAUTHORIZED_ACCESS) + ); + return; + } + if ( !(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES) ) { - debug('remote stream response ok'); + debug('remote stream response %o', res.statusCode); passThroughRemoteStream.emit( 'error', errorUtils.getInternalError(`bad uplink status code: ${res.statusCode}`) @@ -898,16 +907,17 @@ class Storage { private getUpLinkForDistFile(pkgName: string, distFile: DistFile): IProxy { let uplink: IProxy | null = null; - for (const uplinkId in this.uplinks) { + for (const uplinkName in this.uplinks) { // refer to https://github.com/verdaccio/verdaccio/issues/1642 - if (hasProxyTo(pkgName, uplinkId, this.config.packages)) { - uplink = this.uplinks[uplinkId]; + if (hasProxyTo(pkgName, uplinkName, this.config.packages)) { + uplink = this.uplinks[uplinkName]; } } if (uplink == null) { - debug('upstream not found creating one for %o', pkgName); + debug('upstream not found, creating one for %o', pkgName); uplink = new ProxyStorage( + `verdaccio-${pkgName}`, { url: distFile.url, cache: true, @@ -1700,13 +1710,13 @@ class Storage { # one uplink setup proxy: npmjs - A package requires uplinks syncronization if enables the proxy section, uplinks - can be more than one, the more are the most slow request will take, the request - are made in serial and if 1st call fails, the second will be triggered, otherwise + A package requires uplinks syncronization if the proxy section is defined. There can be + more than one uplink. The more uplinks are defined, the longer the request will take. + The requests are made in serial and if 1st call fails, the second will be triggered, otherwise the 1st will reply and others will be discarded. The order is important. - Errors on upkinks are considered are, time outs, connection fails and http status 304, - in that case the request returns empty body and we want ask next on the list if has fresh + Errors on uplinks that are considered are time outs, connection fails, and http status 304. + In these cases the request returns empty body and we want ask next on the list if has fresh updates. */ public async syncUplinksMetadata( @@ -1716,20 +1726,18 @@ class Storage { ): Promise<[Manifest | null, any]> { let found = localManifest !== null; let syncManifest: Manifest | null = null; - const upLinks: string[] = []; + let upLinks: string[] = []; const hasToLookIntoUplinks = _.isNil(options.uplinksLook) || options.uplinksLook; debug('is sync uplink enabled %o', hasToLookIntoUplinks); - for (const uplink in this.uplinks) { - if (hasProxyTo(name, uplink, this.config.packages) && hasToLookIntoUplinks) { - debug('sync uplink %o', uplink); - upLinks.push(uplink); - } + if (hasToLookIntoUplinks) { + upLinks = getProxiesForPackage(name, this.config.packages); + debug('uplinks found for %o: %o', name, upLinks); } - // if none uplink match we return the local manifest + // if no uplinks match we return the local manifest if (upLinks.length === 0) { - debug('no uplinks found for %o upstream update aborted', name); + debug('no uplinks found for %o, upstream update aborted', name); return [localManifest, []]; } @@ -1808,7 +1816,7 @@ class Storage { options: Partial ): Promise { // we store which uplink is updating the manifest - const upLinkMeta = cachedManifest._uplinks[uplink.upname]; + const upLinkMeta = cachedManifest._uplinks[uplink.uplinkName]; let _cacheManifest = { ...cachedManifest }; if (validatioUtils.isObject(upLinkMeta)) { @@ -1816,7 +1824,7 @@ class Storage { // we check the uplink cache is fresh if (fetched && Date.now() - fetched < uplink.maxage) { - debug('returning cached manifest for %o', uplink.upname); + debug('returning cached manifest for %o', uplink.uplinkName); return cachedManifest; } } @@ -1838,7 +1846,7 @@ class Storage { throw err; } // updates the _uplink metadata fields, cache, etc - _cacheManifest = updateUpLinkMetadata(uplink.upname, _cacheManifest, etag); + _cacheManifest = updateUpLinkMetadata(uplink.uplinkName, _cacheManifest, etag); // merge time field cache and remote _cacheManifest = mergeUplinkTimeIntoLocalNext(_cacheManifest, remoteManifest); // update the _uplinks field in the cache diff --git a/packages/store/test/storage.spec.ts b/packages/store/test/storage.spec.ts index 972ef4c72..4de45854d 100644 --- a/packages/store/test/storage.spec.ts +++ b/packages/store/test/storage.spec.ts @@ -1039,7 +1039,7 @@ describe('storage', () => { .then((stream) => { stream.on('error', (err) => { expect(err).toEqual(errorUtils.getNotFound(API_ERROR.NO_PACKAGE)); - done(); + done(true); }); }); }); diff --git a/packages/utils/test/matcher.spec.ts b/packages/utils/test/matcher.spec.ts index ad6b5ed9d..a6bb6cf4c 100644 --- a/packages/utils/test/matcher.spec.ts +++ b/packages/utils/test/matcher.spec.ts @@ -60,7 +60,35 @@ describe('getMatchedPackagesSpec', () => { expect(getMatchedPackagesSpec('angular', packages).proxy).toMatch('google'); // @ts-expect-error expect(getMatchedPackagesSpec('@fake/angular', packages).proxy).toMatch('npmjs'); + // @ts-expect-error expect(getMatchedPackagesSpec('vue', packages)).toBeUndefined(); + // @ts-expect-error expect(getMatchedPackagesSpec('@scope/vue', packages)).toBeUndefined(); }); + + test('should return multiple uplinks in given order', () => { + const packages = { + react: { + access: 'admin', + publish: 'admin', + proxy: 'github npmjs', + }, + angular: { + access: 'admin', + publish: 'admin', + proxy: 'npmjs gitlab', + }, + '@fake/*': { + access: '$all', + publish: '$authenticated', + proxy: 'npmjs gitlab github', + }, + }; + // @ts-expect-error + expect(getMatchedPackagesSpec('react', packages).proxy).toMatch('github npmjs'); + // @ts-expect-error + expect(getMatchedPackagesSpec('angular', packages).proxy).toMatch('npmjs gitlab'); + // @ts-expect-error + expect(getMatchedPackagesSpec('@fake/angular', packages).proxy).toMatch('npmjs gitlab github'); + }); });