Infrastructure/verdaccio
0
Fork 0
mirror of https://github.com/verdaccio/verdaccio.git synced 2025-04-01 02:42:23 -05:00
Code Issues Activity

fix(proxy): uplink processing order (#5131)

Browse source 
* fix: uplink processing order

* Replace upname with uplinkName

* fix proxy uplinkname
This commit is contained in:
Marc Bernard 2025-03-23 12:36:27 +01:00 • committed by GitHub
parent d633685d9e
commit b3fa5df7bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 316 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.changeset/gold-files-speak.md Normal file
View file

@ -0,0 +1,9 @@
---
'@verdaccio/config': patch
'@verdaccio/search': patch
'@verdaccio/proxy': patch
'@verdaccio/store': patch
'@verdaccio/utils': patch
---
fix: uplink processing order

8
packages/config/src/uplinks.ts
View file

@ -48,9 +48,13 @@ export function sanityCheckUplinksProps(configUpLinks: UpLinksConfList): UpLinks
return uplinks;
}
export function hasProxyTo(pkg: string, upLink: string, packages: PackageList): boolean {
export function getProxiesForPackage(pkg: string, packages: PackageList): string[] {
const matchedPkg = getMatchedPackagesSpec(pkg, packages);
const proxyList = typeof matchedPkg !== 'undefined' ? matchedPkg.proxy : [];
return matchedPkg?.proxy || [];
}
export function hasProxyTo(pkg: string, upLink: string, packages: PackageList): boolean {
const proxyList = getProxiesForPackage(pkg, packages);
if (proxyList) {
return proxyList.some((curr) => upLink === curr);
}

21
packages/config/test/package-access.spec.ts
View file

@ -80,6 +80,25 @@ describe('Package access utilities', () => {
expect(all.publish).toContain('admin');
});
test('should test multi proxy definition', () => {
const { packages } = parseConfigFile(parseConfigurationFile('pkgs-multi-proxy'));
const access = normalisePackageAccess(packages);
expect(access).toBeDefined();
const scoped = access[`${PACKAGE_ACCESS.SCOPE}`];
const all = access[`${PACKAGE_ACCESS.ALL}`];
const testPackage = access['test-package'];
expect(scoped).toBeDefined();
expect(scoped.proxy).toEqual(['github', 'npmjs']);
expect(all).toBeDefined();
expect(all.proxy).toEqual(['npmjs', 'gitlab']);
expect(testPackage).toBeDefined();
expect(testPackage.proxy).toEqual(['npmjs', 'gitlab', 'github']);
});
test(
'should normalize deprecated packages into the new ones ' + '(backward props compatible)',
() => {
@ -92,7 +111,7 @@ describe('Package access utilities', () => {
expect(react).toBeDefined();
expect(react.access).toBeDefined();
expect(react.access).toEqual([]);
expect(react.publish[0]).toBe('admin');
expect(react.publish?.[0]).toBe('admin');
expect(react.proxy).toBeDefined();
expect(react.proxy).toEqual([]);
expect(react.storage).toBeDefined();

26
packages/config/test/partials/config/yaml/pkgs-multi-proxy.yaml Normal file
View file

@ -0,0 +1,26 @@
uplinks:
github:
url: 'https://npm.pkg.github.com/'
auth:
type: Bearer
token: 'xxx123xxx'
gitlab:
url: 'https://gitlab.com/api/v4/projects/1/packages/npm/'
auth:
type: Basic
token: 'xxx456xxx'
npmjs:
url: https://registry.npmjs.org/
# Multiple proxies
# https://verdaccio.org/docs/packages/#use-multiple-uplinks
packages:
'@*/*':
access: $all
proxy: github npmjs
'test-package':
access: $all
proxy: npmjs gitlab github
'**':
access: $all
proxy: npmjs gitlab

34
packages/config/test/uplinks.spec.ts
View file

@ -1,7 +1,12 @@
import { describe, expect, test } from 'vitest';
import { normalisePackageAccess, parseConfigFile } from '../src';
import { hasProxyTo, sanityCheckUplinksProps, uplinkSanityCheck } from '../src/uplinks';
import {
getProxiesForPackage,
hasProxyTo,
sanityCheckUplinksProps,
uplinkSanityCheck,
} from '../src/uplinks';
import { parseConfigurationFile } from './utils';
describe('Uplinks Utilities', () => {
@ -24,7 +29,7 @@ describe('Uplinks Utilities', () => {
});
describe('sanityCheckUplinksProps', () => {
test('should fails if url prop is missing', () => {
test('should fail if url prop is missing', () => {
const { uplinks } = parseConfigFile(parseConfigurationFile('uplink-wrong'));
expect(() => {
sanityCheckUplinksProps(uplinks);
@ -37,6 +42,31 @@ describe('Uplinks Utilities', () => {
});
});
describe('multiple uplinks with auth', () => {
test('should fail if url or auth are missing', () => {
const { uplinks } = parseConfigFile(parseConfigurationFile('pkgs-multi-proxy'));
expect(sanityCheckUplinksProps(uplinks)).toEqual(uplinks);
expect(Object.keys(uplinks)).toHaveLength(3);
// No trailing slash in urls
expect(uplinks.github.url).toEqual('https://npm.pkg.github.com');
expect(uplinks.github?.auth?.type).toEqual('Bearer');
expect(uplinks.github?.auth?.token).toEqual('xxx123xxx');
expect(uplinks.gitlab.url).toEqual('https://gitlab.com/api/v4/projects/1/packages/npm');
expect(uplinks.gitlab?.auth?.type).toEqual('Basic');
expect(uplinks.gitlab?.auth?.token).toEqual('xxx456xxx');
expect(uplinks.npmjs.url).toEqual('https://registry.npmjs.org');
});
test('check proxy list for package access', () => {
const packages = normalisePackageAccess(
parseConfigFile(parseConfigurationFile('pkgs-multi-proxy')).packages
);
expect(getProxiesForPackage('@scope/test', packages)).toEqual(['github', 'npmjs']);
expect(getProxiesForPackage('test', packages)).toEqual(['npmjs', 'gitlab']);
expect(getProxiesForPackage('test-package', packages)).toEqual(['npmjs', 'gitlab', 'github']);
});
});
describe('hasProxyTo', () => {
test('should test basic config', () => {
const packages = normalisePackageAccess(

41
packages/proxy/src/proxy.ts
View file

@ -60,6 +60,7 @@ export type ProxySearchParams = {
retry?: Partial<RetryOptions>;
};
export interface IProxy {
uplinkName: string;
config: UpLinkConfLocal;
failed_requests: number;
userAgent: string;
@ -71,7 +72,6 @@ export interface IProxy {
timeout: Delays;
max_fails: number;
fail_timeout: number;
upname: string;
search(options: ProxySearchParams): Promise<Stream.Readable>;
getRemoteMetadata(
name: string,
@ -97,6 +97,7 @@ export interface ISyncUplinksOptions extends Options {
* (same for storage.js, local-storage.js, up-storage.js)
*/
class ProxyStorage implements IProxy {
public uplinkName: string;
public config: UpLinkConfLocal;
public failed_requests: number;
public userAgent: string;
@ -109,9 +110,6 @@ class ProxyStorage implements IProxy {
public max_fails: number;
public fail_timeout: number;
public agent_options: AgentOptionsConf;
// FIXME: upname is assigned to each instance
// @ts-ignore
public upname: string;
public proxy: string | undefined;
private agent: Agents;
// @ts-ignore
@ -119,7 +117,14 @@ class ProxyStorage implements IProxy {
public strict_ssl: boolean;
private retry: Partial<RetryOptions>;
public constructor(config: UpLinkConfLocal, mainConfig: Config, logger: Logger, agent?: Agents) {
public constructor(
uplinkName: string,
config: UpLinkConfLocal,
mainConfig: Config,
logger: Logger,
agent?: Agents
) {
this.uplinkName = uplinkName;
this.config = config;
this.failed_requests = 0;
this.userAgent = mainConfig.user_agent ?? 'hidden';
@ -205,17 +210,21 @@ class ProxyStorage implements IProxy {
let token: any;
const tokenConf: any = auth;
if (_.isNil(tokenConf.token) === false && _.isString(tokenConf.token)) {
debug('use token from config');
token = tokenConf.token;
} else if (_.isNil(tokenConf.token_env) === false) {
if (typeof tokenConf.token_env === 'string') {
debug('use token from env %o', tokenConf.token_env);
token = process.env[tokenConf.token_env];
} else if (typeof tokenConf.token_env === 'boolean' && tokenConf.token_env) {
debug('use token from env NPM_TOKEN');
token = process.env.NPM_TOKEN;
} else {
this.logger.error(constants.ERROR_CODE.token_required);
this._throwErrorAuth(constants.ERROR_CODE.token_required);
}
} else {
debug('use token from env NPM_TOKEN');
token = process.env.NPM_TOKEN;
}
@ -225,6 +234,7 @@ class ProxyStorage implements IProxy {
// define type Auth allow basic and bearer
const type = tokenConf.type || TOKEN_BASIC;
debug('token type %o', type);
this._setHeaderAuthorization(headers, type, token);
return headers;
@ -254,7 +264,6 @@ class ProxyStorage implements IProxy {
this._throwErrorAuth(`Auth type '${_type}' not allowed`);
}
type = _.upperFirst(type);
headers[HEADERS.AUTHORIZATION] = buildToken(type, token);
}
@ -276,20 +285,7 @@ class ProxyStorage implements IProxy {
* @param {Object} headers
* @private
* @deprecated use applyUplinkHeaders
*/
private _overrideWithUpLinkConfLocaligHeaders(headers: Headers): any {
if (!this.config.headers) {
return headers;
}
// add/override headers specified in the config
/* eslint guard-for-in: 0 */
for (const key in this.config.headers) {
headers[key] = this.config.headers[key];
}
}
private applyUplinkHeaders(headers: gotHeaders): gotHeaders {
if (!this.config.headers) {
return headers;
@ -504,7 +500,10 @@ class ProxyStorage implements IProxy {
try {
// Incoming URL is relative ie /-/v1/search...
const uri = new URL(url, this.url).href;
this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}');
this.logger.http(
{ uri, uplink: this.uplinkName },
'search request to uplink @{uplink} - @{uri}'
);
debug('searching on %o', uri);
const response = got(uri, {
signal: abort ? abort.signal : {},
@ -527,7 +526,7 @@ class ProxyStorage implements IProxy {
throw errorUtils.getInternalError(`bad status code ${err.response.statusCode} from uplink`);
}
this.logger.error(
{ errorMessage: err?.message, name: this.upname },
{ errorMessage: err?.message, name: this.uplinkName },
'proxy uplink @{name} search error: @{errorMessage}'
);
throw err;

15
packages/proxy/src/uplink-util.ts
View file

@ -12,15 +12,10 @@ export interface ProxyInstanceList {
export function setupUpLinks(config: Config, logger: Logger): ProxyInstanceList {
const uplinks: ProxyInstanceList = {};
for (const uplinkName in config.uplinks) {
if (Object.prototype.hasOwnProperty.call(config.uplinks, uplinkName)) {
// instance for each up-link definition
const proxy: IProxy = new ProxyStorage(config.uplinks[uplinkName], config, logger);
// TODO: review this can be inside ProxyStorage
proxy.upname = uplinkName;
uplinks[uplinkName] = proxy;
}
for (const uplinkName of Object.keys(config.uplinks)) {
// instance for each up-link definition
const proxy: IProxy = new ProxyStorage(uplinkName, config.uplinks[uplinkName], config, logger);
uplinks[uplinkName] = proxy;
}
return uplinks;
@ -35,7 +30,7 @@ export function updateVersionsHiddenUpLinkNext(manifest: Manifest, upLink: IProx
for (const version of versionsList) {
// holds a "hidden" value to be used by the package storage.
versions[version][Symbol.for('__verdaccio_uplink')] = upLink.upname;
versions[version][Symbol.for('__verdaccio_uplink')] = upLink.uplinkName;
}
return { ...manifest, versions };

23
packages/proxy/test/conf/multi-proxy.yaml Normal file
View file

@ -0,0 +1,23 @@
uplinks:
github:
url: 'https://npm.pkg.github.com/'
auth:
type: Bearer
token: 'xxx123xxx'
gitlab:
url: 'https://gitlab.com/api/v4/projects/1/packages/npm/'
auth:
type: Basic
token: 'xxx456xxx'
npmjs:
url: https://registry.npmjs.org/
# Multiple proxies
# https://verdaccio.org/docs/packages/#use-multiple-uplinks
packages:
'@*/*':
access: $all
proxy: github npmjs
'**':
access: $all
proxy: npmjs gitlab

14
packages/proxy/test/headers.auth.spec.ts
View file

@ -27,7 +27,7 @@ function createUplink(config) {
};
const mergeConfig = Object.assign({}, defaultConfig, config);
// @ts-ignore
return new ProxyStorage(mergeConfig, {}, logger);
return new ProxyStorage('npmjs', mergeConfig, {}, logger);
}
function setHeadersNext(config: unknown = {}, headers: any = {}) {
@ -97,6 +97,18 @@ describe('setHeadersNext', () => {
expect(headers[HEADERS.AUTHORIZATION]).toEqual(buildToken(TOKEN_BASIC, 'Zm9vX2Jhcg=='));
});
test('set type lower case', () => {
const headers = setHeadersNext({
auth: {
type: 'basic', // lower case type
token: 'test',
},
});
expect(Object.keys(headers)).toHaveLength(4);
expect(headers[HEADERS.AUTHORIZATION]).toEqual(buildToken(TOKEN_BASIC, 'test')); // capital case type
});
test('set type auth bearer', () => {
const headers = setHeadersNext({
auth: {

4
packages/proxy/test/noProxy.spec.ts
View file

@ -8,8 +8,8 @@ setup({});
function getProxyInstance(host, uplinkConf, appConfig) {
uplinkConf.url = host;
return new ProxyStorage(uplinkConf, appConfig, logger);
const uplinkName = host.replace(/^https?:\/\//, '');
return new ProxyStorage(uplinkName, uplinkConf, appConfig, logger);
}
describe('Use proxy', () => {

10
packages/proxy/test/proxy.error.___.ts
View file

@ -45,7 +45,7 @@
// test('should be offline uplink', (done) => {
// const tarball = 'https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz';
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').times(100).replyWithError('some error');
// const proxy = new ProxyStorage(defaultRequestOptions, conf);
// const proxy = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = proxy.fetchTarball(tarball);
// // to test a uplink is offline we have to be try 3 times
// // the default failed request are set to 2
@ -79,7 +79,7 @@
// test('not found tarball', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(404);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getNotFound(API_ERROR.NOT_FILE_UPLINK));
@ -89,7 +89,7 @@
// test('fail tarball request', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').replyWithError('boom file');
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(Error('boom file'));
@ -99,7 +99,7 @@
// test('bad uplink request', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(409);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getInternalError(`bad uplink status code: 409`));
@ -115,7 +115,7 @@
// .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), {
// [HEADER_TYPE.CONTENT_LENGTH]: 0,
// });
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getInternalError(API_ERROR.CONTENT_MISMATCH));

31
packages/proxy/test/proxy.metadata.spec.ts
View file

@ -57,7 +57,7 @@ describe('proxy', () => {
})
.get('/jquery')
.reply(200, { body: 'test' });
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const [manifest] = await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
});
@ -83,7 +83,7 @@ describe('proxy', () => {
etag: () => `_ref_4444`,
}
);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
});
@ -110,7 +110,7 @@ describe('proxy', () => {
etag: () => `_ref_4444`,
}
);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
etag: 'foo',
remoteAddress: '127.0.0.1',
@ -125,7 +125,7 @@ describe('proxy', () => {
nock(domain)
.get('/jquery')
.reply(200, { body: { name: 'foo', version: '1.0.0' } }, {});
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
});
@ -154,7 +154,7 @@ describe('proxy', () => {
describe('error handling', () => {
test('proxy call with 304', async () => {
nock(domain).get('/jquery').reply(304);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(prox1.getRemoteMetadata('jquery', { etag: 'rev_3333' })).rejects.toThrow(
'no data'
);
@ -162,7 +162,7 @@ describe('proxy', () => {
test('reply with error', async () => {
nock(domain).get('/jquery').replyWithError('something awful happened');
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
@ -172,7 +172,7 @@ describe('proxy', () => {
test('reply with 409 error', async () => {
nock(domain).get('/jquery').reply(409);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(prox1.getRemoteMetadata('jquery', { retry: { limit: 0 } })).rejects.toThrow(
/bad status code: 409/
);
@ -180,7 +180,7 @@ describe('proxy', () => {
test('reply with bad body json format', async () => {
nock(domain).get('/jquery').reply(200, 'some-text');
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
@ -190,7 +190,7 @@ describe('proxy', () => {
test('400 error proxy call', async () => {
nock(domain).get('/jquery').reply(409);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
@ -200,7 +200,7 @@ describe('proxy', () => {
test('proxy not found', async () => {
nock(domain).get('/jquery').reply(404);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
@ -227,7 +227,7 @@ describe('proxy', () => {
.once()
.reply(200, { body: { name: 'foo', version: '1.0.0' } });
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const [manifest] = await prox1.getRemoteMetadata('jquery', {
retry: { limit: 2 },
});
@ -246,7 +246,7 @@ describe('proxy', () => {
test('retry count is exceded and uplink goes offline with logging activity', async () => {
nock(domain).get('/jquery').times(10).reply(500);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1',
@ -280,6 +280,7 @@ describe('proxy', () => {
.reply(200, { body: { name: 'foo', version: '1.0.0' } });
const prox1 = new ProxyStorage(
'uplink',
{ ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 },
conf,
logger
@ -337,7 +338,7 @@ describe('proxy', () => {
const confTimeout = { ...defaultRequestOptions };
// @ts-expect-error
confTimeout.timeout = '2s';
const prox1 = new ProxyStorage(confTimeout, conf, logger);
const prox1 = new ProxyStorage('uplink', confTimeout, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
retry: { limit: 0 },
@ -357,7 +358,7 @@ describe('proxy', () => {
const confTimeout = { ...defaultRequestOptions };
// @ts-expect-error
confTimeout.timeout = '2s';
const prox1 = new ProxyStorage(confTimeout, conf, logger);
const prox1 = new ProxyStorage('uplink', confTimeout, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
retry: { limit: 1 },
@ -368,7 +369,7 @@ describe('proxy', () => {
// test('retry count is exceded and uplink goes offline with logging activity', async () => {
// nock(domain).get('/jquery').times(10).reply(500);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf, logger);
// await expect(
// prox1.getRemoteMetadata('jquery', {
// remoteAddress: '127.0.0.1',

2
packages/proxy/test/proxy.protocol.spec.ts
View file

@ -21,7 +21,7 @@ const logger = {
function getProxyInstance(host, uplinkConf, appConfig) {
uplinkConf.url = host;
return new ProxyStorage(uplinkConf, appConfig, logger);
return new ProxyStorage('uplink', uplinkConf, appConfig, logger);
}
describe('Check protocol of proxy', () => {

8
packages/proxy/test/proxy.search.spec.ts
View file

@ -44,7 +44,7 @@ describe('proxy', () => {
nock(domain)
.get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio')
.reply(200, response);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const abort = new AbortController();
const stream = await prox1.search({
abort,
@ -60,7 +60,7 @@ describe('proxy', () => {
nock(domain + '/')
.get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio')
.reply(200, response);
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const abort = new AbortController();
const stream = await prox1.search({
abort,
@ -76,7 +76,7 @@ describe('proxy', () => {
.get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio')
.reply(409);
const abort = new AbortController();
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
await expect(
prox1.search({
abort,
@ -98,7 +98,7 @@ describe('proxy', () => {
// const mockClient = mockAgent.get(domain);
// mockClient.intercept(options).reply(500, {});
// const abort = new AbortController();
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// await expect(
// prox1.search({
// abort,

16
packages/proxy/test/proxy.tarball.spec.ts
View file

@ -46,7 +46,7 @@ describe('tarball proxy', () => {
nock('https://registry.verdaccio.org')
.get('/jquery/-/jquery-0.0.1.tgz')
.replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'));
const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const stream = prox1.fetchTarball(
'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz',
// @ts-expect-error
@ -69,7 +69,7 @@ describe('tarball proxy', () => {
.get('/jquery/-/jquery-0.0.1.tgz')
.once()
.replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'));
const prox1 = new ProxyStorage(defaultRequestOptions, conf);
const prox1 = new ProxyStorage('uplink', defaultRequestOptions, conf, logger);
const stream = prox1.fetchTarball(
'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz',
{ retry: { limit: 2 } }
@ -89,7 +89,7 @@ describe('tarball proxy', () => {
// .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), {
// [HEADER_TYPE.CONTENT_LENGTH]: 277,
// });
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on(HEADER_TYPE.CONTENT_LENGTH, (data) => {
// expect(data).toEqual('277');
@ -101,7 +101,7 @@ describe('tarball proxy', () => {
// test('should be offline uplink', (done) => {
// const tarball = 'https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz';
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').times(100).replyWithError('some error');
// const proxy = new ProxyStorage(defaultRequestOptions, conf);
// const proxy = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = proxy.fetchTarball(tarball);
// // to test a uplink is offline we have to be try 3 times
// // the default failed request are set to 2
@ -135,7 +135,7 @@ describe('tarball proxy', () => {
// test('not found tarball', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(404);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getNotFound(API_ERROR.NOT_FILE_UPLINK));
@ -145,7 +145,7 @@ describe('tarball proxy', () => {
// test('fail tarball request', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').replyWithError('boom file');
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(Error('boom file'));
@ -155,7 +155,7 @@ describe('tarball proxy', () => {
// test('bad uplink request', (done) => {
// nock(domain).get('/jquery/-/jquery-0.0.1.tgz').reply(409);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getInternalError(`bad uplink status code: 409`));
@ -171,7 +171,7 @@ describe('tarball proxy', () => {
// .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'), {
// [HEADER_TYPE.CONTENT_LENGTH]: 0,
// });
// const prox1 = new ProxyStorage(defaultRequestOptions, conf);
// const prox1 = new ProxyStorage('uplink',defaultRequestOptions, conf);
// const stream = prox1.fetchTarball('https://registry.npmjs.org/jquery/-/jquery-0.0.1.tgz');
// stream.on('error', (response) => {
// expect(response).toEqual(errorUtils.getInternalError(API_ERROR.CONTENT_MISMATCH));

57
packages/proxy/test/uplink-util.spec.ts Normal file
View file

@ -0,0 +1,57 @@
import path from 'path';
import { describe, expect, test, vi } from 'vitest';
import { Config, parseConfigFile } from '@verdaccio/config';
import { TOKEN_BASIC, TOKEN_BEARER } from '@verdaccio/core';
import { Logger } from '@verdaccio/types';
import { IProxy } from '../src/index';
import { setupUpLinks } from '../src/uplink-util';
const getConf = (name) => path.join(__dirname, '/conf', name);
const mockDebug = vi.fn();
const mockInfo = vi.fn();
const mockHttp = vi.fn();
const mockError = vi.fn();
const mockWarn = vi.fn();
const logger = {
debug: mockDebug,
info: mockInfo,
http: mockHttp,
error: mockError,
warn: mockWarn,
} as unknown as Logger;
describe('setupUpLinks', () => {
test('should create uplinks for each proxy configuration', () => {
const proxyPath = getConf('multi-proxy.yaml');
const config = new Config(parseConfigFile(proxyPath));
const uplinks = setupUpLinks(config, logger);
expect(Object.keys(uplinks)).toHaveLength(3);
expect(uplinks).toHaveProperty('github');
expect(uplinks).toHaveProperty('gitlab');
expect(uplinks).toHaveProperty('npmjs');
const githubProxy = uplinks.github as IProxy;
expect(githubProxy.uplinkName).toBe('github');
expect(githubProxy.config.auth).toEqual({
type: TOKEN_BEARER,
token: 'xxx123xxx',
});
const gitlabProxy = uplinks.gitlab as IProxy;
expect(gitlabProxy.uplinkName).toBe('gitlab');
expect(gitlabProxy.config.auth).toEqual({
type: TOKEN_BASIC,
token: 'xxx456xxx',
});
const npmjsProxy = uplinks.npmjs as IProxy;
expect(npmjsProxy.uplinkName).toBe('npmjs');
expect(npmjsProxy.config.auth).toBeUndefined();
});
});

14
packages/search/src/search.ts
View file

@ -36,13 +36,13 @@ class Search {
// const transformResults = new TransFormResults({ objectMode: true });
const streamPassThrough = new PassThrough({ objectMode: true });
debug('uplinks found %s', upLinkList.length);
const searchUplinksStreams = upLinkList.map((uplinkId: string) => {
const uplink = this.uplinks[uplinkId];
const searchUplinksStreams = upLinkList.map((uplinkName: string) => {
const uplink = this.uplinks[uplinkName];
if (!uplink) {
// this line should never happens
this.logger.error({ uplinkId }, 'uplink @upLinkId not found');
this.logger.error({ uplinkName }, 'uplink @uplinkName not found');
}
return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
return this.consumeSearchStream(uplinkName, uplink, options, streamPassThrough);
});
try {
@ -82,7 +82,7 @@ class Search {
* Consume the upstream and pipe it to a transformable stream.
*/
private consumeSearchStream(
uplinkId: string,
uplinkName: string,
uplink: IProxy,
options: ProxySearchParams,
searchPassThrough: PassThrough
@ -91,8 +91,8 @@ class Search {
bodyStream.pipe(searchPassThrough, { end: false });
bodyStream.on('error', (err: any): void => {
this.logger.error(
{ uplinkId, err: err },
'search error for uplink @{uplinkId}: @{err?.message}'
{ uplinkName, err: err },
'search error for uplink @{uplinkName}: @{err?.message}'
);
searchPassThrough.end();
});

2
packages/store/package.json
View file

@ -34,7 +34,7 @@
},
"scripts": {
"clean": "rimraf ./build",
"test": "vitest run",
"test": "vitest run --testTimeout 20000",
"type-check": "tsc --noEmit -p tsconfig.build.json",
"build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json",
"build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps",

4
packages/store/src/lib/storage-utils.ts
View file

@ -213,10 +213,10 @@ export function mergeUplinkTimeIntoLocalNext(
return cacheManifest;
}
export function updateUpLinkMetadata(uplinkId, manifest: Manifest, etag: string) {
export function updateUpLinkMetadata(uplinkName: string, manifest: Manifest, etag: string) {
const _uplinks = {
...manifest._uplinks,
[uplinkId]: {
[uplinkName]: {
etag,
fetched: Date.now(),
},

52
packages/store/src/storage.ts
View file

@ -6,7 +6,7 @@ import { PassThrough, Readable, Transform } from 'stream';
import { pipeline } from 'stream/promises';
import { default as URL } from 'url';
import { hasProxyTo } from '@verdaccio/config';
import { getProxiesForPackage, hasProxyTo } from '@verdaccio/config';
import {
API_ERROR,
API_MESSAGE,
@ -316,10 +316,19 @@ class Storage {
return;
}
if (res.statusCode === HTTP_STATUS.UNAUTHORIZED) {
debug('remote stream response 401');
passThroughRemoteStream.emit(
'error',
errorUtils.getUnauthorized(errorUtils.API_ERROR.UNAUTHORIZED_ACCESS)
);
return;
}
if (
!(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES)
) {
debug('remote stream response ok');
debug('remote stream response %o', res.statusCode);
passThroughRemoteStream.emit(
'error',
errorUtils.getInternalError(`bad uplink status code: ${res.statusCode}`)
@ -898,16 +907,17 @@ class Storage {
private getUpLinkForDistFile(pkgName: string, distFile: DistFile): IProxy {
let uplink: IProxy | null = null;
for (const uplinkId in this.uplinks) {
for (const uplinkName in this.uplinks) {
// refer to https://github.com/verdaccio/verdaccio/issues/1642
if (hasProxyTo(pkgName, uplinkId, this.config.packages)) {
uplink = this.uplinks[uplinkId];
if (hasProxyTo(pkgName, uplinkName, this.config.packages)) {
uplink = this.uplinks[uplinkName];
}
}
if (uplink == null) {
debug('upstream not found creating one for %o', pkgName);
debug('upstream not found, creating one for %o', pkgName);
uplink = new ProxyStorage(
`verdaccio-${pkgName}`,
{
url: distFile.url,
cache: true,
@ -1700,13 +1710,13 @@ class Storage {
# one uplink setup
proxy: npmjs
A package requires uplinks syncronization if enables the proxy section, uplinks
can be more than one, the more are the most slow request will take, the request
are made in serial and if 1st call fails, the second will be triggered, otherwise
A package requires uplinks syncronization if the proxy section is defined. There can be
more than one uplink. The more uplinks are defined, the longer the request will take.
The requests are made in serial and if 1st call fails, the second will be triggered, otherwise
the 1st will reply and others will be discarded. The order is important.
Errors on upkinks are considered are, time outs, connection fails and http status 304,
in that case the request returns empty body and we want ask next on the list if has fresh
Errors on uplinks that are considered are time outs, connection fails, and http status 304.
In these cases the request returns empty body and we want ask next on the list if has fresh
updates.
*/
public async syncUplinksMetadata(
@ -1716,20 +1726,18 @@ class Storage {
): Promise<[Manifest | null, any]> {
let found = localManifest !== null;
let syncManifest: Manifest | null = null;
const upLinks: string[] = [];
let upLinks: string[] = [];
const hasToLookIntoUplinks = _.isNil(options.uplinksLook) || options.uplinksLook;
debug('is sync uplink enabled %o', hasToLookIntoUplinks);
for (const uplink in this.uplinks) {
if (hasProxyTo(name, uplink, this.config.packages) && hasToLookIntoUplinks) {
debug('sync uplink %o', uplink);
upLinks.push(uplink);
}
if (hasToLookIntoUplinks) {
upLinks = getProxiesForPackage(name, this.config.packages);
debug('uplinks found for %o: %o', name, upLinks);
}
// if none uplink match we return the local manifest
// if no uplinks match we return the local manifest
if (upLinks.length === 0) {
debug('no uplinks found for %o upstream update aborted', name);
debug('no uplinks found for %o, upstream update aborted', name);
return [localManifest, []];
}
@ -1808,7 +1816,7 @@ class Storage {
options: Partial<ISyncUplinksOptions>
): Promise<Manifest> {
// we store which uplink is updating the manifest
const upLinkMeta = cachedManifest._uplinks[uplink.upname];
const upLinkMeta = cachedManifest._uplinks[uplink.uplinkName];
let _cacheManifest = { ...cachedManifest };
if (validatioUtils.isObject(upLinkMeta)) {
@ -1816,7 +1824,7 @@ class Storage {
// we check the uplink cache is fresh
if (fetched && Date.now() - fetched < uplink.maxage) {
debug('returning cached manifest for %o', uplink.upname);
debug('returning cached manifest for %o', uplink.uplinkName);
return cachedManifest;
}
}
@ -1838,7 +1846,7 @@ class Storage {
throw err;
}
// updates the _uplink metadata fields, cache, etc
_cacheManifest = updateUpLinkMetadata(uplink.upname, _cacheManifest, etag);
_cacheManifest = updateUpLinkMetadata(uplink.uplinkName, _cacheManifest, etag);
// merge time field cache and remote
_cacheManifest = mergeUplinkTimeIntoLocalNext(_cacheManifest, remoteManifest);
// update the _uplinks field in the cache

2
packages/store/test/storage.spec.ts
View file

@ -1039,7 +1039,7 @@ describe('storage', () => {
.then((stream) => {
stream.on('error', (err) => {
expect(err).toEqual(errorUtils.getNotFound(API_ERROR.NO_PACKAGE));
done();
done(true);
});
});
});

28
packages/utils/test/matcher.spec.ts
View file

@ -60,7 +60,35 @@ describe('getMatchedPackagesSpec', () => {
expect(getMatchedPackagesSpec('angular', packages).proxy).toMatch('google');
// @ts-expect-error
expect(getMatchedPackagesSpec('@fake/angular', packages).proxy).toMatch('npmjs');
// @ts-expect-error
expect(getMatchedPackagesSpec('vue', packages)).toBeUndefined();
// @ts-expect-error
expect(getMatchedPackagesSpec('@scope/vue', packages)).toBeUndefined();
});
test('should return multiple uplinks in given order', () => {
const packages = {
react: {
access: 'admin',
publish: 'admin',
proxy: 'github npmjs',
},
angular: {
access: 'admin',
publish: 'admin',
proxy: 'npmjs gitlab',
},
'@fake/*': {
access: '$all',
publish: '$authenticated',
proxy: 'npmjs gitlab github',
},
};
// @ts-expect-error
expect(getMatchedPackagesSpec('react', packages).proxy).toMatch('github npmjs');
// @ts-expect-error
expect(getMatchedPackagesSpec('angular', packages).proxy).toMatch('npmjs gitlab');
// @ts-expect-error
expect(getMatchedPackagesSpec('@fake/angular', packages).proxy).toMatch('npmjs gitlab github');
});
});