0
Fork 0
mirror of https://github.com/verdaccio/verdaccio.git synced 2024-12-30 22:34:10 -05:00

feat: refactor upstream proxy and hooks with got v12 (#3946)

* feat: proxy with got v12

* fix tests

* refactor hooks module
This commit is contained in:
Juan Picado 2023-08-06 17:42:20 +02:00 committed by GitHub
parent ab09f03b63
commit 0a6412ca97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 391 additions and 238 deletions

View file

@ -0,0 +1,9 @@
---
'@verdaccio/core': patch
'@verdaccio/hooks': patch
'@verdaccio/proxy': patch
'@verdaccio/store': patch
'@verdaccio/web': patch
---
refactor: got instead undici

View file

@ -0,0 +1,6 @@
---
'@verdaccio/proxy': minor
'@verdaccio/store': minor
---
feat: refactor proxy with got v12

View file

@ -29,6 +29,7 @@ export const API_ERROR = {
REGISTRATION_DISABLED: 'user registration disabled', REGISTRATION_DISABLED: 'user registration disabled',
UNAUTHORIZED_ACCESS: 'unauthorized access', UNAUTHORIZED_ACCESS: 'unauthorized access',
BAD_STATUS_CODE: 'bad status code', BAD_STATUS_CODE: 'bad status code',
SERVER_TIME_OUT: 'looks like the server is taking to long to respond',
PACKAGE_EXIST: 'this package is already present', PACKAGE_EXIST: 'this package is already present',
BAD_AUTH_HEADER: 'bad authorization header', BAD_AUTH_HEADER: 'bad authorization header',
WEB_DISABLED: 'Web interface is disabled in the config file', WEB_DISABLED: 'Web interface is disabled in the config file',

View file

@ -26,7 +26,7 @@
"verdaccio" "verdaccio"
], ],
"engines": { "engines": {
"node": ">=16" "node": ">=12"
}, },
"dependencies": { "dependencies": {
"@verdaccio/core": "workspace:6.0.0-6-next.74", "@verdaccio/core": "workspace:6.0.0-6-next.74",
@ -34,12 +34,13 @@
"core-js": "3.30.2", "core-js": "3.30.2",
"debug": "4.3.4", "debug": "4.3.4",
"handlebars": "4.7.7", "handlebars": "4.7.7",
"undici": "4.16.0" "got-cjs": "12.5.4"
}, },
"devDependencies": { "devDependencies": {
"@verdaccio/auth": "workspace:6.0.0-6-next.53", "@verdaccio/auth": "workspace:6.0.0-6-next.53",
"@verdaccio/config": "workspace:6.0.0-6-next.74", "@verdaccio/config": "workspace:6.0.0-6-next.74",
"@verdaccio/types": "workspace:11.0.0-6-next.25" "@verdaccio/types": "workspace:11.0.0-6-next.25",
"nock": "13.2.9"
}, },
"scripts": { "scripts": {
"clean": "rimraf ./build", "clean": "rimraf ./build",

View file

@ -1,5 +1,5 @@
import buildDebug from 'debug'; import buildDebug from 'debug';
import { fetch } from 'undici'; import got from 'got-cjs';
import { HTTP_STATUS } from '@verdaccio/core'; import { HTTP_STATUS } from '@verdaccio/core';
import { logger } from '@verdaccio/logger'; import { logger } from '@verdaccio/logger';
@ -16,7 +16,7 @@ export async function notifyRequest(url: string, options: FetchOptions): Promise
let response; let response;
try { try {
debug('uri %o', url); debug('uri %o', url);
response = await fetch(url, { response = got.post(url, {
body: JSON.stringify(options.body), body: JSON.stringify(options.body),
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },

View file

@ -1,4 +1,4 @@
import { MockAgent, setGlobalDispatcher } from 'undici'; import nock from 'nock';
import { createRemoteUser, parseConfigFile } from '@verdaccio/config'; import { createRemoteUser, parseConfigFile } from '@verdaccio/config';
import { setup } from '@verdaccio/logger'; import { setup } from '@verdaccio/logger';
@ -21,26 +21,21 @@ const domain = 'http://slack-service';
const options = { const options = {
path: '/foo?auth_token=mySecretToken', path: '/foo?auth_token=mySecretToken',
method: 'POST',
}; };
describe('Notifications:: notifyRequest', () => { describe('Notifications:: notifyRequest', () => {
beforeEach(() => {
nock.cleanAll();
});
test('when sending a empty notification', async () => { test('when sending a empty notification', async () => {
const mockAgent = new MockAgent({ connections: 1 }); nock(domain).post(options.path).reply(200, { body: 'test' });
setGlobalDispatcher(mockAgent);
const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(200, { body: 'test' });
const notificationResponse = await notify({}, {}, createRemoteUser('foo', []), 'bar'); const notificationResponse = await notify({}, {}, createRemoteUser('foo', []), 'bar');
expect(notificationResponse).toEqual([false]); expect(notificationResponse).toEqual([false]);
}); });
test('when sending a single notification', async () => { test('when sending a single notification', async () => {
const mockAgent = new MockAgent({ connections: 1 }); nock(domain).post(options.path).reply(200, { body: 'test' });
setGlobalDispatcher(mockAgent);
const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(200, { body: 'test' });
const notificationResponse = await notify( const notificationResponse = await notify(
{}, {},
singleHeaderNotificationConfig, singleHeaderNotificationConfig,
@ -48,14 +43,10 @@ describe('Notifications:: notifyRequest', () => {
'bar' 'bar'
); );
expect(notificationResponse).toEqual([true]); expect(notificationResponse).toEqual([true]);
await mockClient.close();
}); });
test('when notification endpoint is missing', async () => { test('when notification endpoint is missing', async () => {
const mockAgent = new MockAgent({ connections: 1 }); nock(domain).post(options.path).reply(200, { body: 'test' });
setGlobalDispatcher(mockAgent);
const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(200, { body: 'test' });
const name = 'package'; const name = 'package';
const config: Partial<Config> = { const config: Partial<Config> = {
// @ts-ignore // @ts-ignore
@ -70,16 +61,22 @@ describe('Notifications:: notifyRequest', () => {
}); });
test('when multiple notifications', async () => { test('when multiple notifications', async () => {
const mockAgent = new MockAgent({ connections: 1 }); nock(domain)
setGlobalDispatcher(mockAgent); .post(options.path)
const mockClient = mockAgent.get(domain); .once()
mockClient.intercept(options).reply(200, { body: 'test' }); .reply(200, { body: 'test' })
mockClient.intercept(options).reply(400, {}); .post(options.path)
mockClient.intercept(options).reply(500, { message: 'Something bad happened' }); .once()
.reply(400, {})
.post(options.path)
.once()
.reply(500, { message: 'Something bad happened' });
// mockClient.intercept(options).reply(200, { body: 'test' });
// mockClient.intercept(options).reply(400, {});
// mockClient.intercept(options).reply(500, { message: 'Something bad happened' });
const name = 'package'; const name = 'package';
const responses = await notify({ name }, multiNotificationConfig, { name: 'foo' }, 'bar'); const responses = await notify({ name }, multiNotificationConfig, { name: 'foo' }, 'bar');
expect(responses).toEqual([true, false, false]); expect(responses).toEqual([true, false, false]);
await mockClient.close();
}); });
}); });

View file

@ -26,8 +26,7 @@
"verdaccio" "verdaccio"
], ],
"engines": { "engines": {
"node": ">=16", "node": ">=12"
"npm": ">=6"
}, },
"scripts": { "scripts": {
"clean": "rimraf ./build", "clean": "rimraf ./build",
@ -41,22 +40,20 @@
"dependencies": { "dependencies": {
"@verdaccio/config": "workspace:6.0.0-6-next.74", "@verdaccio/config": "workspace:6.0.0-6-next.74",
"@verdaccio/core": "workspace:6.0.0-6-next.74", "@verdaccio/core": "workspace:6.0.0-6-next.74",
"@verdaccio/local-storage": "workspace:11.0.0-6-next.44",
"@verdaccio/logger": "workspace:6.0.0-6-next.42",
"@verdaccio/utils": "workspace:6.0.0-6-next.42", "@verdaccio/utils": "workspace:6.0.0-6-next.42",
"JSONStream": "1.3.5", "JSONStream": "1.3.5",
"debug": "4.3.4", "debug": "4.3.4",
"lodash": "4.17.21", "got-cjs": "12.5.4",
"got": "11.8.6",
"hpagent": "1.2.0", "hpagent": "1.2.0",
"undici": "4.16.0" "lodash": "4.17.21"
}, },
"devDependencies": { "devDependencies": {
"p-cancelable": "2.1.1",
"@verdaccio/types": "workspace:11.0.0-6-next.25", "@verdaccio/types": "workspace:11.0.0-6-next.25",
"@verdaccio/logger": "workspace:6.0.0-6-next.42",
"get-stream": "^6.0.1", "get-stream": "^6.0.1",
"nock": "13.2.9", "nock": "13.2.9",
"node-mocks-http": "1.12.1", "node-mocks-http": "1.12.1",
"p-cancelable": "2.1.1",
"semver": "7.5.4" "semver": "7.5.4"
}, },
"funding": { "funding": {

View file

@ -1,4 +1,4 @@
import { Agents } from 'got'; import { Agents } from 'got-cjs';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Agent as HttpAgent, AgentOptions as HttpAgentOptions } from 'http'; import { Agent as HttpAgent, AgentOptions as HttpAgentOptions } from 'http';
import { Agent as HttpsAgent, AgentOptions as HttpsAgentOptions } from 'https'; import { Agent as HttpsAgent, AgentOptions as HttpsAgentOptions } from 'https';

View file

@ -1,10 +1,15 @@
import JSONStream from 'JSONStream'; import JSONStream from 'JSONStream';
import buildDebug from 'debug'; import buildDebug from 'debug';
import got, { RequiredRetryOptions, Headers as gotHeaders } from 'got'; import got, {
import type { Agents, Options } from 'got'; Agents,
Delays,
Options,
RequestError,
RetryOptions,
Headers as gotHeaders,
} from 'got-cjs';
import _ from 'lodash'; import _ from 'lodash';
import Stream, { PassThrough, Readable } from 'stream'; import Stream, { PassThrough, Readable } from 'stream';
import { Headers, fetch as undiciFetch } from 'undici';
import { URL } from 'url'; import { URL } from 'url';
import { import {
@ -24,8 +29,6 @@ import { buildToken } from '@verdaccio/utils';
import CustomAgents, { AgentOptionsConf } from './agent'; import CustomAgents, { AgentOptionsConf } from './agent';
import { parseInterval } from './proxy-utils'; import { parseInterval } from './proxy-utils';
const LoggerApi = require('@verdaccio/logger');
const debug = buildDebug('verdaccio:proxy'); const debug = buildDebug('verdaccio:proxy');
const encode = function (thing): string { const encode = function (thing): string {
@ -51,10 +54,11 @@ export interface ProxyList {
} }
export type ProxySearchParams = { export type ProxySearchParams = {
headers?: Headers;
url: string; url: string;
query?: searchUtils.SearchQuery;
abort: AbortController; abort: AbortController;
query?: searchUtils.SearchQuery;
headers?: Headers;
retry?: Partial<RetryOptions>;
}; };
export interface IProxy { export interface IProxy {
config: UpLinkConfLocal; config: UpLinkConfLocal;
@ -65,15 +69,18 @@ export interface IProxy {
server_id: string; server_id: string;
url: URL; url: URL;
maxage: number; maxage: number;
timeout: number; timeout: Delays;
max_fails: number; max_fails: number;
fail_timeout: number; fail_timeout: number;
upname: string; upname: string;
search(options: ProxySearchParams): Promise<Stream.Readable>; search(options: ProxySearchParams): Promise<Stream.Readable>;
getRemoteMetadata(name: string, options: ISyncUplinksOptions): Promise<[Manifest, string]>; getRemoteMetadata(
name: string,
options: Partial<ISyncUplinksOptions>
): Promise<[Manifest, string]>;
fetchTarball( fetchTarball(
url: string, url: string,
options: Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'> options: Partial<Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'>>
): PassThrough; ): PassThrough;
} }
@ -99,7 +106,7 @@ class ProxyStorage implements IProxy {
public server_id: string; public server_id: string;
public url: URL; public url: URL;
public maxage: number; public maxage: number;
public timeout: number; public timeout: Delays;
public max_fails: number; public max_fails: number;
public fail_timeout: number; public fail_timeout: number;
public agent_options: AgentOptionsConf; public agent_options: AgentOptionsConf;
@ -111,14 +118,14 @@ class ProxyStorage implements IProxy {
// @ts-ignore // @ts-ignore
public last_request_time: number | null; public last_request_time: number | null;
public strict_ssl: boolean; public strict_ssl: boolean;
private retry: Partial<RequiredRetryOptions> | number; private retry: Partial<RetryOptions>;
public constructor(config: UpLinkConfLocal, mainConfig: Config, agent?: Agents) { public constructor(config: UpLinkConfLocal, mainConfig: Config, logger: Logger, agent?: Agents) {
this.config = config; this.config = config;
this.failed_requests = 0; this.failed_requests = 0;
this.userAgent = mainConfig.user_agent ?? 'hidden'; this.userAgent = mainConfig.user_agent ?? 'hidden';
this.ca = config.ca; this.ca = config.ca;
this.logger = LoggerApi.logger.child({ sub: 'out' }); this.logger = logger;
this.server_id = mainConfig.server_id; this.server_id = mainConfig.server_id;
this.agent_options = setConfig(this.config, 'agent_options', { this.agent_options = setConfig(this.config, 'agent_options', {
keepAlive: true, keepAlive: true,
@ -145,7 +152,10 @@ class ProxyStorage implements IProxy {
// a bunch of different configurable timers // a bunch of different configurable timers
this.maxage = parseInterval(setConfig(this.config, 'maxage', '2m')); this.maxage = parseInterval(setConfig(this.config, 'maxage', '2m'));
// https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md // https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md
this.timeout = parseInterval(setConfig(this.config, 'timeout', '30s')); this.timeout = {
request: parseInterval(setConfig(this.config, 'timeout', '30s')),
};
debug('set timeout %s', this.timeout);
this.max_fails = Number(setConfig(this.config, 'max_fails', this.config.max_fails ?? 2)); this.max_fails = Number(setConfig(this.config, 'max_fails', this.config.max_fails ?? 2));
this.fail_timeout = parseInterval(setConfig(this.config, 'fail_timeout', '5m')); this.fail_timeout = parseInterval(setConfig(this.config, 'fail_timeout', '5m'));
this.strict_ssl = Boolean(setConfig(this.config, 'strict_ssl', true)); this.strict_ssl = Boolean(setConfig(this.config, 'strict_ssl', true));
@ -162,7 +172,7 @@ class ProxyStorage implements IProxy {
} }
} }
public getHeadersNext(headers = {}): gotHeaders { public getHeaders(headers = {}): gotHeaders {
const accept = HEADERS.ACCEPT; const accept = HEADERS.ACCEPT;
const acceptEncoding = HEADERS.ACCEPT_ENCODING; const acceptEncoding = HEADERS.ACCEPT_ENCODING;
const userAgent = HEADERS.USER_AGENT; const userAgent = HEADERS.USER_AGENT;
@ -296,15 +306,15 @@ class ProxyStorage implements IProxy {
public async getRemoteMetadata( public async getRemoteMetadata(
name: string, name: string,
options: ISyncUplinksOptions options: Partial<ISyncUplinksOptions>
): Promise<[Manifest, string]> { ): Promise<[Manifest, string]> {
if (this._ifRequestFailure()) { if (this._ifRequestFailure()) {
throw errorUtils.getInternalError(API_ERROR.UPLINK_OFFLINE); throw errorUtils.getInternalError(API_ERROR.UPLINK_OFFLINE);
} }
// FUTURE: allow mix headers that comes from the client // FUTURE: allow mix headers that comes from the client
debug('get metadata for %s', name); debug('getting metadata for package %s', name);
let headers = this.getHeadersNext(options?.headers); let headers = this.getHeaders(options?.headers);
headers = this.addProxyHeaders(headers, options.remoteAddress); headers = this.addProxyHeaders(headers, options.remoteAddress);
headers = this.applyUplinkHeaders(headers); headers = this.applyUplinkHeaders(headers);
// the following headers cannot be overwritten // the following headers cannot be overwritten
@ -314,25 +324,24 @@ class ProxyStorage implements IProxy {
} }
const method = options.method || 'GET'; const method = options.method || 'GET';
const uri = this.config.url + `/${encode(name)}`; const uri = this.config.url + `/${encode(name)}`;
debug('request uri for %s retry %s', uri); debug('set retry limit is %s', this.retry.limit);
let response; let response;
let responseLength = 0; let responseLength = 0;
try { try {
const retry = options?.retry ?? this.retry; const retry = options?.retry ?? this.retry;
debug('retry times %s for %s', retry, uri); debug('retry initial count %s', retry);
response = await got(uri, { response = await got(uri, {
headers, headers,
responseType: 'json', responseType: 'json',
method, method,
agent: this.agent, agent: this.agent,
retry, retry,
// @ts-ignore timeout: this.timeout,
timeout: { request: options?.timeout ?? this.timeout },
hooks: { hooks: {
afterResponse: [ afterResponse: [
(afterResponse) => { (afterResponse) => {
const code = afterResponse.statusCode; const code = afterResponse.statusCode;
debug('code response %s', code); debug('after response code is %s', code);
if (code >= HTTP_STATUS.OK && code < HTTP_STATUS.MULTIPLE_CHOICES) { if (code >= HTTP_STATUS.OK && code < HTTP_STATUS.MULTIPLE_CHOICES) {
if (this.failed_requests >= this.max_fails) { if (this.failed_requests >= this.max_fails) {
this.failed_requests = 0; this.failed_requests = 0;
@ -349,8 +358,8 @@ class ProxyStorage implements IProxy {
}, },
], ],
beforeRetry: [ beforeRetry: [
// FUTURE: got 12.0.0, the option arg should be removed (error: RequestError, count: number) => {
(_options, error: any, count) => { debug('retry %s count: %s', uri, count);
this.failed_requests = count ?? 0; this.failed_requests = count ?? 0;
this.logger.info( this.logger.info(
{ {
@ -378,7 +387,7 @@ class ProxyStorage implements IProxy {
.on('request', () => { .on('request', () => {
this.last_request_time = Date.now(); this.last_request_time = Date.now();
}) })
.on('response', (eventResponse) => { .on<any>('response', (eventResponse) => {
const message = "@{!status}, req: '@{request.method} @{request.url}' (streaming)"; const message = "@{!status}, req: '@{request.method} @{request.url}' (streaming)";
this.logger.http( this.logger.http(
{ {
@ -422,9 +431,10 @@ class ProxyStorage implements IProxy {
); );
return [data, etag]; return [data, etag];
} catch (err: any) { } catch (err: any) {
debug('uri %s fail', uri); debug('error %s on uri %s', err.code, uri);
if (err.code === 'ERR_NON_2XX_3XX_RESPONSE') { if (err.code === 'ERR_NON_2XX_3XX_RESPONSE') {
const code = err.response.statusCode; const code = err.response.statusCode;
debug('error code %s', code);
if (code === HTTP_STATUS.NOT_FOUND) { if (code === HTTP_STATUS.NOT_FOUND) {
throw errorUtils.getNotFound(errorUtils.API_ERROR.NOT_PACKAGE_UPLINK); throw errorUtils.getNotFound(errorUtils.API_ERROR.NOT_PACKAGE_UPLINK);
} }
@ -437,6 +447,15 @@ class ProxyStorage implements IProxy {
error.remoteStatus = code; error.remoteStatus = code;
throw error; throw error;
} }
} else if (err.code === 'ETIMEDOUT') {
debug('error code timeout');
const code = err.code;
const error = errorUtils.getInternalError(
`${errorUtils.API_ERROR.SERVER_TIME_OUT}: ${code}`
);
// we need this code to identify outside which status code triggered the error
error.remoteStatus = code;
throw error;
} }
throw err; throw err;
} }
@ -449,7 +468,7 @@ class ProxyStorage implements IProxy {
): any { ): any {
debug('fetching url for %s', url); debug('fetching url for %s', url);
const options = { ...this.config, ...overrideOptions }; const options = { ...this.config, ...overrideOptions };
let headers = this.getHeadersNext(options?.headers); let headers = this.getHeaders(options?.headers);
headers = this.addProxyHeaders(headers, options.remoteAddress); headers = this.addProxyHeaders(headers, options.remoteAddress);
headers = this.applyUplinkHeaders(headers); headers = this.applyUplinkHeaders(headers);
// the following headers cannot be overwritten // the following headers cannot be overwritten
@ -482,37 +501,31 @@ class ProxyStorage implements IProxy {
* @param {*} options request options * @param {*} options request options
* @return {Stream} * @return {Stream}
*/ */
public async search({ url, abort }: ProxySearchParams): Promise<Stream.Readable> { public async search({ url, abort, retry }: ProxySearchParams): Promise<Stream.Readable> {
debug('search url %o', url);
let response;
try { try {
const fullURL = new URL(`${this.url}${url}`); const fullURL = new URL(`${this.url}${url}`);
// FIXME: a better way to remove duplicate slashes? // FIXME: a better way to remove duplicate slashes?
const uri = fullURL.href.replace(/([^:]\/)\/+/g, '$1'); const uri = fullURL.href.replace(/([^:]\/)\/+/g, '$1');
this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}'); this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}');
response = await undiciFetch(uri, { debug('searching on %s', uri);
method: 'GET', const response = got(uri, {
// FUTURE: whitelist domains what we are sending not need it headers, security check signal: abort ? abort.signal : {},
// headers: new Headers({ agent: this.agent,
// ...headers, timeout: this.timeout,
// connection: 'keep-alive', retry: retry ?? this.retry,
// }),
signal: abort?.signal,
}); });
debug('response.status %o', response.status);
if (response.status >= HTTP_STATUS.BAD_REQUEST) {
throw errorUtils.getInternalError(`bad status code ${response.status} from uplink`);
}
const streamSearch = new PassThrough({ objectMode: true });
const res = await response.text(); const res = await response.text();
const streamSearch = new PassThrough({ objectMode: true });
const streamResponse = Readable.from(res); const streamResponse = Readable.from(res);
// objects is one of the properties on the body, it ignores date and total // objects is one of the properties on the body, it ignores date and total
streamResponse.pipe(JSONStream.parse('objects')).pipe(streamSearch, { end: true }); streamResponse.pipe(JSONStream.parse('objects')).pipe(streamSearch, { end: true });
return streamSearch; return streamSearch;
} catch (err: any) { } catch (err: any) {
debug('search error %s', err);
if (err.response.statusCode === 409) {
throw errorUtils.getInternalError(`bad status code ${err.response.statusCode} from uplink`);
}
this.logger.error( this.logger.error(
{ errorMessage: err?.message, name: this.upname }, { errorMessage: err?.message, name: this.upname },
'proxy uplink @{name} search error: @{errorMessage}' 'proxy uplink @{name} search error: @{errorMessage}'

View file

@ -1,11 +1,23 @@
import { DEFAULT_REGISTRY } from '@verdaccio/config'; import { DEFAULT_REGISTRY } from '@verdaccio/config';
import { HEADERS, TOKEN_BASIC, TOKEN_BEARER, constants } from '@verdaccio/core'; import { HEADERS, TOKEN_BASIC, TOKEN_BEARER, constants } from '@verdaccio/core';
import { setup } from '@verdaccio/logger'; import { Logger } from '@verdaccio/types';
import { buildToken } from '@verdaccio/utils'; import { buildToken } from '@verdaccio/utils';
import { ProxyStorage } from '../src'; import { ProxyStorage } from '../src';
setup(); const mockDebug = jest.fn();
const mockInfo = jest.fn();
const mockHttp = jest.fn();
const mockError = jest.fn();
const mockWarn = jest.fn();
const logger = {
debug: mockDebug,
info: mockInfo,
http: mockHttp,
error: mockError,
warn: mockWarn,
} as unknown as Logger;
function createUplink(config) { function createUplink(config) {
const defaultConfig = { const defaultConfig = {
@ -13,12 +25,12 @@ function createUplink(config) {
}; };
const mergeConfig = Object.assign({}, defaultConfig, config); const mergeConfig = Object.assign({}, defaultConfig, config);
// @ts-ignore // @ts-ignore
return new ProxyStorage(mergeConfig, {}); return new ProxyStorage(mergeConfig, {}, logger);
} }
function setHeadersNext(config: unknown = {}, headers: any = {}) { function setHeadersNext(config: unknown = {}, headers: any = {}) {
const uplink = createUplink(config); const uplink = createUplink(config);
return uplink.getHeadersNext({ ...headers }); return uplink.getHeaders({ ...headers });
} }
describe('setHeadersNext', () => { describe('setHeadersNext', () => {

View file

@ -1,11 +1,13 @@
import { logger, setup } from '@verdaccio/logger';
import { ProxyStorage } from '../src'; import { ProxyStorage } from '../src';
require('@verdaccio/logger').setup([]); setup({});
function getProxyInstance(host, uplinkConf, appConfig) { function getProxyInstance(host, uplinkConf, appConfig) {
uplinkConf.url = host; uplinkConf.url = host;
return new ProxyStorage(uplinkConf, appConfig); return new ProxyStorage(uplinkConf, appConfig, logger);
} }
describe('Use proxy', () => { describe('Use proxy', () => {

View file

@ -4,6 +4,7 @@ import { setTimeout } from 'timers/promises';
import { Config, parseConfigFile } from '@verdaccio/config'; import { Config, parseConfigFile } from '@verdaccio/config';
import { API_ERROR, errorUtils } from '@verdaccio/core'; import { API_ERROR, errorUtils } from '@verdaccio/core';
import { Logger } from '@verdaccio/types';
import { ProxyStorage } from '../src'; import { ProxyStorage } from '../src';
@ -15,6 +16,14 @@ const mockHttp = jest.fn();
const mockError = jest.fn(); const mockError = jest.fn();
const mockWarn = jest.fn(); const mockWarn = jest.fn();
const logger = {
debug: mockDebug,
info: mockInfo,
http: mockHttp,
error: mockError,
warn: mockWarn,
} as unknown as Logger;
// mock to get the headers fixed value // mock to get the headers fixed value
jest.mock('crypto', () => { jest.mock('crypto', () => {
return { return {
@ -31,22 +40,6 @@ jest.mock('crypto', () => {
}; };
}); });
jest.mock('@verdaccio/logger', () => {
const originalLogger = jest.requireActual('@verdaccio/logger');
return {
...originalLogger,
logger: {
child: () => ({
debug: (arg, msg) => mockDebug(arg, msg),
info: (arg, msg) => mockInfo(arg, msg),
http: (arg, msg) => mockHttp(arg, msg),
error: (arg, msg) => mockError(arg, msg),
warn: (arg, msg) => mockWarn(arg, msg),
}),
},
};
});
const domain = 'https://registry.npmjs.org'; const domain = 'https://registry.npmjs.org';
describe('proxy', () => { describe('proxy', () => {
@ -77,7 +70,7 @@ describe('proxy', () => {
}) })
.get('/jquery') .get('/jquery')
.reply(200, { body: 'test' }); .reply(200, { body: 'test' });
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const [manifest] = await prox1.getRemoteMetadata('jquery', { const [manifest] = await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
}); });
@ -103,7 +96,7 @@ describe('proxy', () => {
etag: () => `_ref_4444`, etag: () => `_ref_4444`,
} }
); );
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const [manifest, etag] = await prox1.getRemoteMetadata('jquery', { const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
}); });
@ -130,7 +123,7 @@ describe('proxy', () => {
etag: () => `_ref_4444`, etag: () => `_ref_4444`,
} }
); );
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const [manifest, etag] = await prox1.getRemoteMetadata('jquery', { const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
etag: 'foo', etag: 'foo',
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -145,7 +138,7 @@ describe('proxy', () => {
nock(domain) nock(domain)
.get('/jquery') .get('/jquery')
.reply(200, { body: { name: 'foo', version: '1.0.0' } }, {}); .reply(200, { body: { name: 'foo', version: '1.0.0' } }, {});
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await prox1.getRemoteMetadata('jquery', { await prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
}); });
@ -174,7 +167,7 @@ describe('proxy', () => {
describe('error handling', () => { describe('error handling', () => {
test('proxy call with 304', async () => { test('proxy call with 304', async () => {
nock(domain).get('/jquery').reply(304); nock(domain).get('/jquery').reply(304);
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect(prox1.getRemoteMetadata('jquery', { etag: 'rev_3333' })).rejects.toThrow( await expect(prox1.getRemoteMetadata('jquery', { etag: 'rev_3333' })).rejects.toThrow(
'no data' 'no data'
); );
@ -182,7 +175,7 @@ describe('proxy', () => {
test('reply with error', async () => { test('reply with error', async () => {
nock(domain).get('/jquery').replyWithError('something awful happened'); nock(domain).get('/jquery').replyWithError('something awful happened');
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.getRemoteMetadata('jquery', { prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -192,15 +185,15 @@ describe('proxy', () => {
test('reply with 409 error', async () => { test('reply with 409 error', async () => {
nock(domain).get('/jquery').reply(409); nock(domain).get('/jquery').reply(409);
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect(prox1.getRemoteMetadata('jquery', { retry: 0 })).rejects.toThrow( await expect(prox1.getRemoteMetadata('jquery', { retry: { limit: 0 } })).rejects.toThrow(
new Error('bad status code: 409') new Error('bad status code: 409')
); );
}); });
test('reply with bad body json format', async () => { test('reply with bad body json format', async () => {
nock(domain).get('/jquery').reply(200, 'some-text'); nock(domain).get('/jquery').reply(200, 'some-text');
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.getRemoteMetadata('jquery', { prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -214,7 +207,7 @@ describe('proxy', () => {
test('400 error proxy call', async () => { test('400 error proxy call', async () => {
nock(domain).get('/jquery').reply(409); nock(domain).get('/jquery').reply(409);
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.getRemoteMetadata('jquery', { prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -226,7 +219,7 @@ describe('proxy', () => {
test('proxy not found', async () => { test('proxy not found', async () => {
nock(domain).get('/jquery').reply(404); nock(domain).get('/jquery').reply(404);
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.getRemoteMetadata('jquery', { prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -253,7 +246,7 @@ describe('proxy', () => {
.once() .once()
.reply(200, { body: { name: 'foo', version: '1.0.0' } }); .reply(200, { body: { name: 'foo', version: '1.0.0' } });
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const [manifest] = await prox1.getRemoteMetadata('jquery', { const [manifest] = await prox1.getRemoteMetadata('jquery', {
retry: { limit: 2 }, retry: { limit: 2 },
}); });
@ -269,10 +262,10 @@ describe('proxy', () => {
); );
}); });
test('retry is exceded and uplink goes offline with logging activity', async () => { test('retry count is exceded and uplink goes offline with logging activity', async () => {
nock(domain).get('/jquery').times(10).reply(500); nock(domain).get('/jquery').times(10).reply(500);
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.getRemoteMetadata('jquery', { prox1.getRemoteMetadata('jquery', {
remoteAddress: '127.0.0.1', remoteAddress: '127.0.0.1',
@ -307,7 +300,8 @@ describe('proxy', () => {
const prox1 = new ProxyStorage( const prox1 = new ProxyStorage(
{ ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 }, { ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 },
conf conf,
logger
); );
// force retry // force retry
await expect( await expect(
@ -350,5 +344,127 @@ describe('proxy', () => {
); );
}, 10000); }, 10000);
}); });
describe('timeout', () => {
test('fail for timeout (2 seconds)', async () => {
nock(domain)
.get('/jquery')
.times(10)
.delayConnection(6000)
.reply(200, { body: { name: 'foo', version: '1.0.0' } });
const confTimeout = { ...defaultRequestOptions };
// @ts-expect-error
confTimeout.timeout = '2s';
const prox1 = new ProxyStorage(confTimeout, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
retry: { limit: 0 },
})
).rejects.toThrow('ETIMEDOUT');
}, 10000);
test('fail for one failure and timeout (2 seconds)', async () => {
nock(domain)
.get('/jquery')
.times(1)
.reply(500)
.get('/jquery')
.delayConnection(4000)
.reply(200, { body: { name: 'foo', version: '1.0.0' } });
const confTimeout = { ...defaultRequestOptions };
// @ts-expect-error
confTimeout.timeout = '2s';
const prox1 = new ProxyStorage(confTimeout, conf, logger);
await expect(
prox1.getRemoteMetadata('jquery', {
retry: { limit: 1 },
})
).rejects.toThrow('ETIMEDOUT');
}, 10000);
// test('retry count is exceded and uplink goes offline with logging activity', async () => {
// nock(domain).get('/jquery').times(10).reply(500);
// const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
// await expect(
// prox1.getRemoteMetadata('jquery', {
// remoteAddress: '127.0.0.1',
// retry: { limit: 2 },
// })
// ).rejects.toThrow();
// await expect(
// prox1.getRemoteMetadata('jquery', {
// remoteAddress: '127.0.0.1',
// retry: { limit: 2 },
// })
// ).rejects.toThrow(errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
// expect(mockWarn).toHaveBeenCalledTimes(1);
// expect(mockWarn).toHaveBeenLastCalledWith(
// {
// host: 'registry.npmjs.org',
// },
// 'host @{host} is now offline'
// );
// });
// test('fails calls and recover with 200 with log online activity', async () => {
// // This unit test is designed to verify if the uplink goes to offline
// // and recover after the fail_timeout has expired.
// nock(domain)
// .get('/jquery')
// .thrice()
// .reply(500, 'some-text')
// .get('/jquery')
// .once()
// .reply(200, { body: { name: 'foo', version: '1.0.0' } });
// const prox1 = new ProxyStorage(
// { ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 },
// conf,
// logger
// );
// // force retry
// await expect(
// prox1.getRemoteMetadata('jquery', {
// remoteAddress: '127.0.0.1',
// retry: { limit: 2 },
// })
// ).rejects.toThrow();
// // display offline error on exausted retry
// await expect(
// prox1.getRemoteMetadata('jquery', {
// remoteAddress: '127.0.0.1',
// retry: { limit: 2 },
// })
// ).rejects.toThrow(errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
// expect(mockWarn).toHaveBeenCalledTimes(2);
// expect(mockWarn).toHaveBeenLastCalledWith(
// {
// host: 'registry.npmjs.org',
// },
// 'host @{host} is now offline'
// );
// expect(mockWarn).toHaveBeenLastCalledWith(
// {
// host: 'registry.npmjs.org',
// },
// 'host @{host} is now offline'
// );
// // this is based on max_fails, if change that also change here acordingly
// await setTimeout(3000);
// const [manifest] = await prox1.getRemoteMetadata('jquery', {
// retry: { limit: 2 },
// });
// expect(manifest).toEqual({ body: { name: 'foo', version: '1.0.0' } });
// expect(mockWarn).toHaveBeenLastCalledWith(
// {
// host: 'registry.npmjs.org',
// },
// 'host @{host} is now online'
// );
// }, 10000);
});
}); });
}); });

View file

@ -2,37 +2,30 @@
/* global AbortController */ /* global AbortController */
import getStream from 'get-stream'; import getStream from 'get-stream';
import nock from 'nock';
import path from 'path'; import path from 'path';
import { MockAgent, setGlobalDispatcher } from 'undici';
import { Config, parseConfigFile } from '@verdaccio/config'; import { Config, parseConfigFile } from '@verdaccio/config';
import { streamUtils } from '@verdaccio/core'; import { streamUtils } from '@verdaccio/core';
import { Logger } from '@verdaccio/types';
import { ProxyStorage } from '../src'; import { ProxyStorage } from '../src';
const getConf = (name) => path.join(__dirname, '/conf', name); const getConf = (name) => path.join(__dirname, '/conf', name);
// TODO: we can mock this globally maybe
const mockDebug = jest.fn(); const mockDebug = jest.fn();
const mockInfo = jest.fn(); const mockInfo = jest.fn();
const mockHttp = jest.fn(); const mockHttp = jest.fn();
const mockError = jest.fn(); const mockError = jest.fn();
const mockWarn = jest.fn(); const mockWarn = jest.fn();
jest.mock('@verdaccio/logger', () => {
const originalLogger = jest.requireActual('@verdaccio/logger'); const logger = {
return { debug: mockDebug,
...originalLogger, info: mockInfo,
logger: { http: mockHttp,
child: () => ({ error: mockError,
debug: (arg) => mockDebug(arg), warn: mockWarn,
info: (arg) => mockInfo(arg), } as unknown as Logger;
http: (arg) => mockHttp(arg),
error: (arg) => mockError(arg),
warn: (arg) => mockWarn(arg),
}),
},
};
});
const domain = 'https://registry.npmjs.org'; const domain = 'https://registry.npmjs.org';
@ -44,20 +37,13 @@ describe('proxy', () => {
const proxyPath = getConf('proxy1.yaml'); const proxyPath = getConf('proxy1.yaml');
const conf = new Config(parseConfigFile(proxyPath)); const conf = new Config(parseConfigFile(proxyPath));
const options = {
path: '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio',
method: 'GET',
};
describe('search', () => { describe('search', () => {
test('get response from endpoint', async () => { test('get response from endpoint', async () => {
const response = require('./partials/search-v1.json'); const response = require('./partials/search-v1.json');
const mockAgent = new MockAgent({ connections: 1 }); nock(domain)
mockAgent.disableNetConnect(); .get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio')
setGlobalDispatcher(mockAgent); .reply(200, response);
const mockClient = mockAgent.get(domain); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
mockClient.intercept(options).reply(200, JSON.stringify(response));
const prox1 = new ProxyStorage(defaultRequestOptions, conf);
const abort = new AbortController(); const abort = new AbortController();
const stream = await prox1.search({ const stream = await prox1.search({
abort, abort,
@ -69,13 +55,11 @@ describe('proxy', () => {
}); });
test('handle bad response 409', async () => { test('handle bad response 409', async () => {
const mockAgent = new MockAgent({ connections: 1 }); nock(domain)
mockAgent.disableNetConnect(); .get('/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio')
setGlobalDispatcher(mockAgent); .reply(409);
const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(409, {});
const abort = new AbortController(); const abort = new AbortController();
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
await expect( await expect(
prox1.search({ prox1.search({
abort, abort,
@ -84,26 +68,26 @@ describe('proxy', () => {
).rejects.toThrow('bad status code 409 from uplink'); ).rejects.toThrow('bad status code 409 from uplink');
}); });
test.todo('abort search from endpoint'); // test.todo('abort search from endpoint');
// TODO: we should test the gzip deflate here, but is hard to test // // TODO: we should test the gzip deflate here, but is hard to test
// fix me if you can deal with Incorrect Header Check issue // // fix me if you can deal with Incorrect Header Check issue
test.todo('get file from endpoint with gzip headers'); // test.todo('get file from endpoint with gzip headers');
test('search endpoint fails', async () => { // test('search endpoint fails', async () => {
const mockAgent = new MockAgent({ connections: 1 }); // const mockAgent = new MockAgent({ connections: 1 });
mockAgent.disableNetConnect(); // mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent); // setGlobalDispatcher(mockAgent);
const mockClient = mockAgent.get(domain); // const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(500, {}); // mockClient.intercept(options).reply(500, {});
const abort = new AbortController(); // const abort = new AbortController();
const prox1 = new ProxyStorage(defaultRequestOptions, conf); // const prox1 = new ProxyStorage(defaultRequestOptions, conf);
await expect( // await expect(
prox1.search({ // prox1.search({
abort, // abort,
url: queryUrl, // url: queryUrl,
}) // })
).rejects.toThrow('bad status code 500 from uplink'); // ).rejects.toThrow('bad status code 500 from uplink');
}); // });
}); });
}); });

View file

@ -2,7 +2,7 @@ import nock from 'nock';
import path from 'path'; import path from 'path';
import { Config, parseConfigFile } from '@verdaccio/config'; import { Config, parseConfigFile } from '@verdaccio/config';
import { setup } from '@verdaccio/logger'; import { logger, setup } from '@verdaccio/logger';
import { ProxyStorage } from '../src'; import { ProxyStorage } from '../src';
@ -44,7 +44,7 @@ describe('tarball proxy', () => {
nock('https://registry.verdaccio.org') nock('https://registry.verdaccio.org')
.get('/jquery/-/jquery-0.0.1.tgz') .get('/jquery/-/jquery-0.0.1.tgz')
.replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz')); .replyWithFile(201, path.join(__dirname, 'partials/jquery-0.0.1.tgz'));
const prox1 = new ProxyStorage(defaultRequestOptions, conf); const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
const stream = prox1.fetchTarball( const stream = prox1.fetchTarball(
'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz', 'https://registry.verdaccio.org/jquery/-/jquery-0.0.1.tgz',
{} {}

View file

@ -58,7 +58,6 @@
"devDependencies": { "devDependencies": {
"@verdaccio/types": "workspace:11.0.0-6-next.25", "@verdaccio/types": "workspace:11.0.0-6-next.25",
"@verdaccio/test-helper": "workspace:2.0.0-6-next.8", "@verdaccio/test-helper": "workspace:2.0.0-6-next.8",
"undici": "4.16.0",
"nock": "13.2.9", "nock": "13.2.9",
"node-mocks-http": "1.12.1", "node-mocks-http": "1.12.1",
"mockdate": "3.0.5" "mockdate": "3.0.5"

View file

@ -1,3 +1,4 @@
import { logger } from '@verdaccio/logger';
import { IProxy, ProxyStorage } from '@verdaccio/proxy'; import { IProxy, ProxyStorage } from '@verdaccio/proxy';
import { Config, Manifest } from '@verdaccio/types'; import { Config, Manifest } from '@verdaccio/types';
@ -14,7 +15,7 @@ export function setupUpLinks(config: Config): ProxyInstanceList {
for (const uplinkName in config.uplinks) { for (const uplinkName in config.uplinks) {
if (Object.prototype.hasOwnProperty.call(config.uplinks, uplinkName)) { if (Object.prototype.hasOwnProperty.call(config.uplinks, uplinkName)) {
// instance for each up-link definition // instance for each up-link definition
const proxy: IProxy = new ProxyStorage(config.uplinks[uplinkName], config); const proxy: IProxy = new ProxyStorage(config.uplinks[uplinkName], config, logger);
// TODO: review this can be inside ProxyStorage // TODO: review this can be inside ProxyStorage
proxy.upname = uplinkName; proxy.upname = uplinkName;

View file

@ -226,22 +226,22 @@ class Storage {
const transformResults = new TransFormResults({ objectMode: true }); const transformResults = new TransFormResults({ objectMode: true });
const streamPassThrough = new PassThrough({ objectMode: true }); const streamPassThrough = new PassThrough({ objectMode: true });
const upLinkList = this.getProxyList(); const upLinkList = this.getProxyList();
debug('uplinks found %s', upLinkList.length);
const searchUplinksStreams = upLinkList.map((uplinkId: string) => { const searchUplinksStreams = upLinkList.map((uplinkId: string) => {
const uplink = this.uplinks[uplinkId]; const uplink = this.uplinks[uplinkId];
if (!uplink) { if (!uplink) {
// this should never tecnically happens // this line should never happens
this.logger.error({ uplinkId }, 'uplink @upLinkId not found'); this.logger.error({ uplinkId }, 'uplink @upLinkId not found');
} }
return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough); return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
}); });
try { try {
debug('search uplinks'); debug('searching on %s uplinks...', searchUplinksStreams?.length);
// we only process those streams end successfully, if all fails // only process those streams end successfully, if all request fails
// we just include local storage // just include local storage results (if local fails then return 500)
await Promise.allSettled([...searchUplinksStreams]); await Promise.allSettled([...searchUplinksStreams]);
debug('search uplinks done'); debug('searching all uplinks done');
} catch (err: any) { } catch (err: any) {
this.logger.error({ err: err?.message }, ' error on uplinks search @{err}'); this.logger.error({ err: err?.message }, ' error on uplinks search @{err}');
streamPassThrough.emit('error', err); streamPassThrough.emit('error', err);
@ -912,7 +912,8 @@ class Storage {
url: distFile.url, url: distFile.url,
cache: true, cache: true,
}, },
this.config this.config,
logger
); );
} }
return uplink; return uplink;
@ -1619,7 +1620,7 @@ class Storage {
public async syncUplinksMetadataNext( public async syncUplinksMetadataNext(
name: string, name: string,
localManifest: Manifest | null, localManifest: Manifest | null,
options: ISyncUplinksOptions = {} options: Partial<ISyncUplinksOptions> = {}
): Promise<[Manifest | null, any]> { ): Promise<[Manifest | null, any]> {
let found = localManifest !== null; let found = localManifest !== null;
let syncManifest: Manifest | null = null; let syncManifest: Manifest | null = null;
@ -1641,7 +1642,7 @@ class Storage {
} }
const uplinksErrors: any[] = []; const uplinksErrors: any[] = [];
// we resolve uplinks async in serie, first come first serve // we resolve uplinks async in series, first come first serve
for (const uplink of upLinks) { for (const uplink of upLinks) {
try { try {
const tempManifest = _.isNil(localManifest) const tempManifest = _.isNil(localManifest)
@ -1712,7 +1713,7 @@ class Storage {
private async mergeCacheRemoteMetadata( private async mergeCacheRemoteMetadata(
uplink: IProxy, uplink: IProxy,
cachedManifest: Manifest, cachedManifest: Manifest,
options: ISyncUplinksOptions options: Partial<ISyncUplinksOptions>
): Promise<Manifest> { ): Promise<Manifest> {
// we store which uplink is updating the manifest // we store which uplink is updating the manifest
const upLinkMeta = cachedManifest._uplinks[uplink.upname]; const upLinkMeta = cachedManifest._uplinks[uplink.upname];

View file

@ -1,6 +1,7 @@
uplinks: uplinks:
timeout: timeout:
url: https://registry.domain.com/ url: https://registry.domain.com/
timeout: 2s
some: some:
url: https://registry.domain.com/ url: https://registry.domain.com/
ver: ver:

View file

@ -1,4 +1,4 @@
import { setGlobalDispatcher } from 'undici'; import nock from 'nock';
import { Config, getDefaultConfig } from '@verdaccio/config'; import { Config, getDefaultConfig } from '@verdaccio/config';
import { searchUtils } from '@verdaccio/core'; import { searchUtils } from '@verdaccio/core';
@ -6,7 +6,7 @@ import { setup } from '@verdaccio/logger';
import { Storage, removeDuplicates } from '../src'; import { Storage, removeDuplicates } from '../src';
setup([]); setup({});
describe('search', () => { describe('search', () => {
describe('search manager utils', () => { describe('search manager utils', () => {
@ -28,26 +28,17 @@ describe('search', () => {
}); });
test('search items', async () => { test('search items', async () => {
const { MockAgent } = require('undici');
// FIXME: fetch is already part of undici // FIXME: fetch is already part of undici
const domain = 'https://registry.npmjs.org'; const domain = 'https://registry.npmjs.org';
const url = '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio'; const url = '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio';
const response = require('./fixtures/search.json'); const response = require('./fixtures/search.json');
const options = { nock(domain).get(url).reply(200, response);
path: url,
method: 'GET',
};
const mockAgent = new MockAgent({ connections: 1 });
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
const mockClient = mockAgent.get(domain);
mockClient.intercept(options).reply(200, JSON.stringify(response));
const config = new Config(getDefaultConfig()); const config = new Config(getDefaultConfig());
const storage = new Storage(config); const storage = new Storage(config);
await storage.init(config); await storage.init(config);
const abort = new AbortController();
// @ts-expect-error const results = await storage.search({ url, query: { text: 'verdaccio' }, abort });
const results = await storage.search({ url, query: { text: 'foo' } });
expect(results).toHaveLength(4); expect(results).toHaveLength(4);
}); });
}); });

View file

@ -864,15 +864,21 @@ describe('storage', () => {
const fooManifest = generatePackageMetadata('timeout', '8.0.0'); const fooManifest = generatePackageMetadata('timeout', '8.0.0');
nock('https://registry.domain.com') nock('https://registry.domain.com')
.get('/timeout') .get(`/${fooManifest.name}`)
.times(10) .times(10)
.delayConnection(2000) .delayConnection(4000)
.reply(201, manifestFooRemoteNpmjs); .reply(201, manifestFooRemoteNpmjs);
const config = new Config( const config = new Config(
configExample( configExample(
{ {
storage: generateRandomStorage(), storage: generateRandomStorage(),
uplinks: {
npmjs: {
url: 'https://registry.npmjs.org',
timeout: '2s',
},
},
}, },
'./fixtures/config/syncDoubleUplinksMetadata.yaml', './fixtures/config/syncDoubleUplinksMetadata.yaml',
__dirname __dirname
@ -885,15 +891,10 @@ describe('storage', () => {
storage.syncUplinksMetadataNext(fooManifest.name, null, { storage.syncUplinksMetadataNext(fooManifest.name, null, {
retry: { limit: 0 }, retry: { limit: 0 },
timeout: { timeout: {
lookup: 100, request: 1000,
connect: 50,
secureConnect: 50,
socket: 500,
// send: 10000,
response: 1000,
}, },
}) })
).rejects.toThrow('ETIMEDOUT'); ).rejects.toThrow(API_ERROR.NO_PACKAGE);
}, 10000); }, 10000);
test('should handle one proxy fails', async () => { test('should handle one proxy fails', async () => {
@ -1468,7 +1469,7 @@ describe('storage', () => {
host: req.get('host') as string, host: req.get('host') as string,
}, },
}) })
).rejects.toThrow(errorUtils.getServiceUnavailable('ETIMEDOUT')); ).rejects.toThrow(errorUtils.getServiceUnavailable(API_ERROR.NO_PACKAGE));
}); });
test('should fetch abbreviated version of manifest ', async () => { test('should fetch abbreviated version of manifest ', async () => {

View file

@ -47,7 +47,6 @@
"supertest": "6.3.3", "supertest": "6.3.3",
"nock": "13.2.9", "nock": "13.2.9",
"jsdom": "20.0.3", "jsdom": "20.0.3",
"undici": "4.16.0",
"verdaccio-auth-memory": "workspace:11.0.0-6-next.39", "verdaccio-auth-memory": "workspace:11.0.0-6-next.39",
"verdaccio-memory": "workspace:11.0.0-6-next.41" "verdaccio-memory": "workspace:11.0.0-6-next.41"
}, },

View file

@ -773,12 +773,12 @@ importers:
debug: debug:
specifier: 4.3.4 specifier: 4.3.4
version: 4.3.4(supports-color@6.1.0) version: 4.3.4(supports-color@6.1.0)
got-cjs:
specifier: 12.5.4
version: 12.5.4
handlebars: handlebars:
specifier: 4.7.7 specifier: 4.7.7
version: 4.7.7 version: 4.7.7
undici:
specifier: 4.16.0
version: 4.16.0
devDependencies: devDependencies:
'@verdaccio/auth': '@verdaccio/auth':
specifier: workspace:6.0.0-6-next.53 specifier: workspace:6.0.0-6-next.53
@ -789,6 +789,9 @@ importers:
'@verdaccio/types': '@verdaccio/types':
specifier: workspace:11.0.0-6-next.25 specifier: workspace:11.0.0-6-next.25
version: link:../core/types version: link:../core/types
nock:
specifier: 13.2.9
version: 13.2.9
packages/loaders: packages/loaders:
dependencies: dependencies:
@ -1389,12 +1392,6 @@ importers:
'@verdaccio/core': '@verdaccio/core':
specifier: workspace:6.0.0-6-next.74 specifier: workspace:6.0.0-6-next.74
version: link:../core/core version: link:../core/core
'@verdaccio/local-storage':
specifier: workspace:11.0.0-6-next.44
version: link:../plugins/local-storage
'@verdaccio/logger':
specifier: workspace:6.0.0-6-next.42
version: link:../logger/logger
'@verdaccio/utils': '@verdaccio/utils':
specifier: workspace:6.0.0-6-next.42 specifier: workspace:6.0.0-6-next.42
version: link:../utils version: link:../utils
@ -1404,19 +1401,19 @@ importers:
debug: debug:
specifier: 4.3.4 specifier: 4.3.4
version: 4.3.4(supports-color@6.1.0) version: 4.3.4(supports-color@6.1.0)
got: got-cjs:
specifier: 11.8.5 specifier: 12.5.4
version: 11.8.5 version: 12.5.4
hpagent: hpagent:
specifier: 1.2.0 specifier: 1.2.0
version: 1.2.0 version: 1.2.0
lodash: lodash:
specifier: 4.17.21 specifier: 4.17.21
version: 4.17.21 version: 4.17.21
undici:
specifier: 4.16.0
version: 4.16.0
devDependencies: devDependencies:
'@verdaccio/logger':
specifier: workspace:6.0.0-6-next.42
version: link:../logger/logger
'@verdaccio/types': '@verdaccio/types':
specifier: workspace:11.0.0-6-next.25 specifier: workspace:11.0.0-6-next.25
version: link:../core/types version: link:../core/types
@ -1664,9 +1661,6 @@ importers:
node-mocks-http: node-mocks-http:
specifier: 1.12.1 specifier: 1.12.1
version: 1.12.1 version: 1.12.1
undici:
specifier: 4.16.0
version: 4.16.0
packages/tools/docusaurus-plugin-contributors: packages/tools/docusaurus-plugin-contributors:
dependencies: dependencies:
@ -2115,9 +2109,6 @@ importers:
supertest: supertest:
specifier: 6.3.3 specifier: 6.3.3
version: 6.3.3 version: 6.3.3
undici:
specifier: 4.16.0
version: 4.16.0
verdaccio-auth-memory: verdaccio-auth-memory:
specifier: workspace:11.0.0-6-next.39 specifier: workspace:11.0.0-6-next.39
version: link:../plugins/auth-memory version: link:../plugins/auth-memory
@ -13515,6 +13506,11 @@ packages:
resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==} resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==}
engines: {node: '>=10.6.0'} engines: {node: '>=10.6.0'}
/cacheable-lookup@6.1.0:
resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==}
engines: {node: '>=10.6.0'}
dev: false
/cacheable-request@7.0.2: /cacheable-request@7.0.2:
resolution: {integrity: sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==} resolution: {integrity: sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==}
engines: {node: '>=8'} engines: {node: '>=8'}
@ -17551,6 +17547,10 @@ packages:
typescript: 4.9.4 typescript: 4.9.4
webpack: 5.82.1(webpack-cli@4.10.0) webpack: 5.82.1(webpack-cli@4.10.0)
/form-data-encoder@1.7.2:
resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}
dev: false
/form-data@2.3.3: /form-data@2.3.3:
resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==}
engines: {node: '>= 0.12'} engines: {node: '>= 0.12'}
@ -17980,6 +17980,24 @@ packages:
dependencies: dependencies:
get-intrinsic: 1.2.0 get-intrinsic: 1.2.0
/got-cjs@12.5.4:
resolution: {integrity: sha512-Uas6lAsP8bRCt5WXGMhjFf/qEHTrm4v4qxGR02rLG2kdG9qedctvlkdwXVcDJ7Cs84X+r4dPU7vdwGjCaspXug==}
engines: {node: '>=12'}
dependencies:
'@sindresorhus/is': 4.6.0
'@szmarczak/http-timer': 4.0.6
'@types/responselike': 1.0.0
cacheable-lookup: 6.1.0
cacheable-request: 7.0.2
decompress-response: 6.0.0
form-data-encoder: 1.7.2
get-stream: 6.0.1
http2-wrapper: 2.2.0
lowercase-keys: 2.0.0
p-cancelable: 2.1.1
responselike: 2.0.1
dev: false
/got@11.8.5: /got@11.8.5:
resolution: {integrity: sha512-o0Je4NvQObAuZPHLFoRSkdG2lTgtcynqymzg2Vupdx6PorhaT5MCbIyXG6d4D94kk8ZG57QeosgdiqfJWhEhlQ==} resolution: {integrity: sha512-o0Je4NvQObAuZPHLFoRSkdG2lTgtcynqymzg2Vupdx6PorhaT5MCbIyXG6d4D94kk8ZG57QeosgdiqfJWhEhlQ==}
engines: {node: '>=10.19.0'} engines: {node: '>=10.19.0'}
@ -18492,6 +18510,14 @@ packages:
quick-lru: 5.1.1 quick-lru: 5.1.1
resolve-alpn: 1.2.1 resolve-alpn: 1.2.1
/http2-wrapper@2.2.0:
resolution: {integrity: sha512-kZB0wxMo0sh1PehyjJUWRFEd99KC5TLjZ2cULC4f9iqJBAmKQQXEICjxl5iPJRwP40dpeHFqqhm7tYCvODpqpQ==}
engines: {node: '>=10.19.0'}
dependencies:
quick-lru: 5.1.1
resolve-alpn: 1.2.1
dev: false
/https-browserify@1.0.0: /https-browserify@1.0.0:
resolution: {integrity: sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==} resolution: {integrity: sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==}
dev: true dev: true
@ -27245,10 +27271,6 @@ packages:
resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==}
dev: true dev: true
/undici@4.16.0:
resolution: {integrity: sha512-tkZSECUYi+/T1i4u+4+lwZmQgLXd4BLGlrc7KZPcLIW7Jpq99+Xpc30ONv7nS6F5UNOxp/HBZSSL9MafUrvJbw==}
engines: {node: '>=12.18'}
/unfetch@4.2.0: /unfetch@4.2.0:
resolution: {integrity: sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==} resolution: {integrity: sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==}
dev: true dev: true