mirror of
https://github.com/verdaccio/verdaccio.git
synced 2024-12-30 22:34:10 -05:00
refactor: search package (#4489)
* refactor: search package refactor: search package * update deps * refactor * refactor tests * add tests
This commit is contained in:
Juan Picado
GitHub
parent
bed68b2ceb
commit
87c16127b4
44 changed files with 946 additions and 1103 deletions
|
|
@ -39,7 +39,7 @@
|
|||
"verdaccio-memory": "11.0.0",
|
||||
"@verdaccio/ui-theme": "6.0.0",
|
||||
"@verdaccio/proxy": "6.0.0",
|
||||
"@verdaccio/search": "6.0.0",
|
||||
"@verdaccio/search-indexer": "6.0.0",
|
||||
"@verdaccio/server": "6.0.0",
|
||||
"@verdaccio/server-fastify": "6.0.0",
|
||||
"@verdaccio/signature": "6.0.0",
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ export default function (route: Router, auth: Auth, storage: Storage): void {
|
|||
const { package: pkgName, filename } = req.params;
|
||||
const abort = new AbortController();
|
||||
try {
|
||||
const stream = (await storage.getTarballNext(pkgName, filename, {
|
||||
const stream = (await storage.getTarball(pkgName, filename, {
|
||||
signal: abort.signal,
|
||||
// TODO: review why this param
|
||||
// enableRemote: true,
|
||||
|
|
|
|||
|
|
@ -50,12 +50,13 @@ export default function (route, auth: Auth, storage: Storage): void {
|
|||
from = parseInt(from, 10) || 0;
|
||||
|
||||
try {
|
||||
debug('storage search initiated');
|
||||
data = await storage.search({
|
||||
query,
|
||||
url,
|
||||
abort,
|
||||
});
|
||||
debug('stream finish');
|
||||
debug('storage items tota: %o', data.length);
|
||||
const checkAccessPromises: searchUtils.SearchItemPkg[] = await Promise.all(
|
||||
data.map((pkgItem) => {
|
||||
return checkAccess(pkgItem, auth, req.remote_user);
|
||||
|
|
|
|||
|
|
@ -558,9 +558,10 @@ describe('AuthTest', () => {
|
|||
const getServer = async function (auth) {
|
||||
const app = express();
|
||||
app.use(express.json({ strict: false, limit: '10mb' }));
|
||||
|
||||
app.use(auth.apiJWTmiddleware());
|
||||
// @ts-expect-error
|
||||
app.use(errorReportingMiddleware(logger));
|
||||
app.use(auth.apiJWTmiddleware());
|
||||
app.get('/*', (req, res, next) => {
|
||||
if ((req as $RequestExtend).remote_user.error) {
|
||||
next(new Error((req as $RequestExtend).remote_user.error));
|
||||
|
|
@ -575,6 +576,7 @@ describe('AuthTest', () => {
|
|||
app.use(final);
|
||||
return app;
|
||||
};
|
||||
|
||||
describe('legacy signature', () => {
|
||||
describe('error cases', () => {
|
||||
test('should handle invalid auth token', async () => {
|
||||
|
|
|
|||
|
|
@ -64,12 +64,6 @@ export async function searchOnStorage(
|
|||
): Promise<searchUtils.SearchItemPkg[]> {
|
||||
const matchedStorages = Array.from(storages);
|
||||
const storageFolders = Array.from(storages.keys());
|
||||
// const getScopedFolders = async (pkgName) => {
|
||||
// const scopedPackages = await getFolders(join(storagePath, pkgName), '*');
|
||||
// const listScoped = scopedPackages.map((scoped) => ({
|
||||
// name: `${pkgName}/${scoped}`,
|
||||
// }));
|
||||
// };
|
||||
debug('search on %o', storagePath);
|
||||
debug('storage folders %o', matchedStorages.length);
|
||||
let results: searchUtils.SearchItemPkg[] = [];
|
||||
|
|
|
|||
|
|
@ -136,13 +136,17 @@ class LocalDatabase extends pluginUtils.Plugin<{}> implements Storage {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param query
|
||||
* @returns
|
||||
*/
|
||||
public async search(query: searchUtils.SearchQuery): Promise<searchUtils.SearchItem[]> {
|
||||
debug('search query to %o', query.text);
|
||||
const results: searchUtils.SearchItem[] = [];
|
||||
const storagePath = this.getStoragePath();
|
||||
const packagesOnStorage = await this.filterByQuery(
|
||||
await searchOnStorage(storagePath, this.storages),
|
||||
query
|
||||
);
|
||||
const localResults = await searchOnStorage(storagePath, this.storages);
|
||||
const packagesOnStorage = await this.filterByQuery(localResults, query);
|
||||
debug('packages found %o', packagesOnStorage.length);
|
||||
for (let storage of packagesOnStorage) {
|
||||
// check if package is listed on the cache private database
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ describe('searchOnFolders', () => {
|
|||
{
|
||||
name: 'pkg1',
|
||||
},
|
||||
|
||||
{
|
||||
name: 'pkg2',
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ module.exports = Object.assign({}, config, {
|
|||
coverageThreshold: {
|
||||
global: {
|
||||
branches: 79,
|
||||
functions: 94,
|
||||
lines: 87,
|
||||
statements: 87,
|
||||
functions: 90,
|
||||
lines: 86,
|
||||
statements: 86,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
export * from './proxy';
|
||||
export * from './uplink-util';
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
import { logger } from '@verdaccio/logger';
|
||||
import { IProxy, ProxyStorage } from '@verdaccio/proxy';
|
||||
import { Config, Manifest } from '@verdaccio/types';
|
||||
import { Config, Logger, Manifest } from '@verdaccio/types';
|
||||
|
||||
import { IProxy, ProxyStorage } from './index';
|
||||
|
||||
export interface ProxyInstanceList {
|
||||
[key: string]: IProxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the Up Storage for each link.
|
||||
* Set up uplinks for each proxy configuration.
|
||||
*/
|
||||
export function setupUpLinks(config: Config): ProxyInstanceList {
|
||||
export function setupUpLinks(config: Config, logger: Logger): ProxyInstanceList {
|
||||
const uplinks: ProxyInstanceList = {};
|
||||
|
||||
for (const uplinkName in config.uplinks) {
|
||||
|
|
@ -38,5 +38,5 @@ export function updateVersionsHiddenUpLinkNext(manifest: Manifest, upLink: IProx
|
|||
versions[version][Symbol.for('__verdaccio_uplink')] = upLink.upname;
|
||||
}
|
||||
|
||||
return { ...manifest, versions: versions };
|
||||
return { ...manifest, versions };
|
||||
}
|
||||
3
packages/search-indexer/.babelrc
Normal file
3
packages/search-indexer/.babelrc
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"extends": "../../.babelrc"
|
||||
}
|
||||
12
packages/search-indexer/jest.config.js
Normal file
12
packages/search-indexer/jest.config.js
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
const config = require('../../jest/config');
|
||||
|
||||
module.exports = Object.assign({}, config, {
|
||||
coverageThreshold: {
|
||||
global: {
|
||||
branches: 79,
|
||||
functions: 94,
|
||||
lines: 87,
|
||||
statements: 87,
|
||||
},
|
||||
},
|
||||
});
|
||||
49
packages/search-indexer/package.json
Normal file
49
packages/search-indexer/package.json
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{
|
||||
"name": "@verdaccio/search-indexer",
|
||||
"version": "7.0.0-next.0",
|
||||
"description": "verdaccio search indexer",
|
||||
"main": "./build/dist.js",
|
||||
"types": "build/index.d.ts",
|
||||
"author": {
|
||||
"name": "Juan Picado",
|
||||
"email": "juanpicado19@gmail.com"
|
||||
},
|
||||
"repository": {
|
||||
"type": "https",
|
||||
"url": "https://github.com/verdaccio/verdaccio"
|
||||
},
|
||||
"license": "MIT",
|
||||
"homepage": "https://verdaccio.org",
|
||||
"keywords": [
|
||||
"private",
|
||||
"package",
|
||||
"repository",
|
||||
"registry",
|
||||
"enterprise",
|
||||
"modules",
|
||||
"proxy",
|
||||
"server",
|
||||
"verdaccio"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf ./build",
|
||||
"test": "vitest run",
|
||||
"type-check": "tsc --noEmit -p tsconfig.build.json",
|
||||
"build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json",
|
||||
"build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps",
|
||||
"build": "esbuild src/index.ts --bundle --outfile=build/dist.js --platform=node --target=node12 && pnpm run build:types"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@verdaccio/types": "workspace:12.0.0-next.2",
|
||||
"@orama/orama": "1.2.4",
|
||||
"debug": "4.3.4",
|
||||
"esbuild": "0.14.10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/verdaccio"
|
||||
}
|
||||
}
|
||||
1
packages/search-indexer/src/index.ts
Normal file
1
packages/search-indexer/src/index.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { default as SearchMemoryIndexer } from './indexer';
|
||||
9
packages/search-indexer/tsconfig.build.json
Normal file
9
packages/search-indexer/tsconfig.build.json
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"extends": "../../tsconfig.base",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./build"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
23
packages/search-indexer/tsconfig.json
Normal file
23
packages/search-indexer/tsconfig.json
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"extends": "../../tsconfig.reference.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./build",
|
||||
"noImplicitAny": false
|
||||
},
|
||||
"include": ["src/**/*.ts", "types/*.d.ts"],
|
||||
"references": [
|
||||
{
|
||||
"path": "../config"
|
||||
},
|
||||
{
|
||||
"path": "../core/core"
|
||||
},
|
||||
{
|
||||
"path": "../logger/logger"
|
||||
},
|
||||
{
|
||||
"path": "../utils"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,763 +0,0 @@
|
|||
# @verdaccio/proxy
|
||||
|
||||
## 7.0.0-next.2
|
||||
|
||||
### Major Changes
|
||||
|
||||
- e7ebccb61: update major dependencies, remove old nodejs support
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- daceb6d87: restore legacy support
|
||||
|
||||
## 7.0.0-next.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 35cc57b79: fix: keyword undefined errors
|
||||
|
||||
## 7.0.0-next.0
|
||||
|
||||
### Major Changes
|
||||
|
||||
- feat!: bump to v7
|
||||
|
||||
## 6.0.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 15e58d988: feat: add search package utilities
|
||||
|
||||
## 6.0.0-6-next.2
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 15e58d98: feat: add search package utilities
|
||||
|
||||
## 6.0.0-6-next.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.68
|
||||
- @verdaccio/config@6.0.0-6-next.68
|
||||
- @verdaccio/local-storage@11.0.0-6-next.38
|
||||
- @verdaccio/utils@6.0.0-6-next.36
|
||||
- @verdaccio/logger@6.0.0-6-next.36
|
||||
|
||||
## 6.0.0-6-next.45
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [16e38df8]
|
||||
- @verdaccio/config@6.0.0-6-next.67
|
||||
- @verdaccio/core@6.0.0-6-next.67
|
||||
- @verdaccio/local-storage@11.0.0-6-next.37
|
||||
- @verdaccio/utils@6.0.0-6-next.35
|
||||
- @verdaccio/logger@6.0.0-6-next.35
|
||||
|
||||
## 6.0.0-6-next.44
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.66
|
||||
- @verdaccio/logger@6.0.0-6-next.34
|
||||
- @verdaccio/local-storage@11.0.0-6-next.36
|
||||
- @verdaccio/config@6.0.0-6-next.66
|
||||
- @verdaccio/utils@6.0.0-6-next.34
|
||||
|
||||
## 6.0.0-6-next.43
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [a1da1130]
|
||||
- @verdaccio/core@6.0.0-6-next.65
|
||||
- @verdaccio/config@6.0.0-6-next.65
|
||||
- @verdaccio/local-storage@11.0.0-6-next.35
|
||||
- @verdaccio/utils@6.0.0-6-next.33
|
||||
- @verdaccio/logger@6.0.0-6-next.33
|
||||
|
||||
## 6.0.0-6-next.42
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [974cd8c1]
|
||||
- @verdaccio/core@6.0.0-6-next.64
|
||||
- @verdaccio/config@6.0.0-6-next.64
|
||||
- @verdaccio/local-storage@11.0.0-6-next.34
|
||||
- @verdaccio/utils@6.0.0-6-next.32
|
||||
- @verdaccio/logger@6.0.0-6-next.32
|
||||
|
||||
## 6.0.0-6-next.41
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [ddb6a223]
|
||||
- Updated dependencies [dc571aab]
|
||||
- @verdaccio/config@6.0.0-6-next.63
|
||||
- @verdaccio/core@6.0.0-6-next.63
|
||||
- @verdaccio/local-storage@11.0.0-6-next.33
|
||||
- @verdaccio/utils@6.0.0-6-next.31
|
||||
- @verdaccio/logger@6.0.0-6-next.31
|
||||
|
||||
## 6.0.0-6-next.40
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [378e907d]
|
||||
- @verdaccio/core@6.0.0-6-next.62
|
||||
- @verdaccio/logger@6.0.0-6-next.30
|
||||
- @verdaccio/local-storage@11.0.0-6-next.32
|
||||
- @verdaccio/config@6.0.0-6-next.62
|
||||
- @verdaccio/utils@6.0.0-6-next.30
|
||||
|
||||
## 6.0.0-6-next.39
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [d167f92e]
|
||||
- @verdaccio/config@6.0.0-6-next.61
|
||||
- @verdaccio/local-storage@11.0.0-6-next.31
|
||||
- @verdaccio/core@6.0.0-6-next.61
|
||||
- @verdaccio/utils@6.0.0-6-next.29
|
||||
- @verdaccio/logger@6.0.0-6-next.29
|
||||
|
||||
## 6.0.0-6-next.38
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 45c03819: refactor: render html middleware
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [45c03819]
|
||||
- @verdaccio/config@6.0.0-6-next.60
|
||||
- @verdaccio/local-storage@11.0.0-6-next.30
|
||||
- @verdaccio/core@6.0.0-6-next.60
|
||||
- @verdaccio/logger@6.0.0-6-next.28
|
||||
- @verdaccio/utils@6.0.0-6-next.28
|
||||
|
||||
## 6.0.0-6-next.37
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [65f88b82]
|
||||
- @verdaccio/logger@6.0.0-6-next.27
|
||||
- @verdaccio/local-storage@11.0.0-6-next.29
|
||||
- @verdaccio/core@6.0.0-6-next.59
|
||||
- @verdaccio/config@6.0.0-6-next.59
|
||||
- @verdaccio/utils@6.0.0-6-next.27
|
||||
|
||||
## 6.0.0-6-next.36
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.58
|
||||
- @verdaccio/config@6.0.0-6-next.58
|
||||
- @verdaccio/local-storage@11.0.0-6-next.28
|
||||
- @verdaccio/utils@6.0.0-6-next.26
|
||||
- @verdaccio/logger@6.0.0-6-next.26
|
||||
|
||||
## 6.0.0-6-next.35
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/local-storage@11.0.0-6-next.27
|
||||
- @verdaccio/core@6.0.0-6-next.57
|
||||
- @verdaccio/config@6.0.0-6-next.57
|
||||
- @verdaccio/logger@6.0.0-6-next.25
|
||||
- @verdaccio/utils@6.0.0-6-next.25
|
||||
|
||||
## 6.0.0-6-next.34
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [a1986e09]
|
||||
- @verdaccio/utils@6.0.0-6-next.24
|
||||
- @verdaccio/config@6.0.0-6-next.56
|
||||
- @verdaccio/local-storage@11.0.0-6-next.26
|
||||
- @verdaccio/core@6.0.0-6-next.56
|
||||
- @verdaccio/logger@6.0.0-6-next.24
|
||||
|
||||
## 6.0.0-6-next.33
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [9718e033]
|
||||
- @verdaccio/config@6.0.0-6-next.55
|
||||
- @verdaccio/core@6.0.0-6-next.55
|
||||
- @verdaccio/utils@6.0.0-6-next.23
|
||||
- @verdaccio/local-storage@11.0.0-6-next.25
|
||||
- @verdaccio/logger@6.0.0-6-next.23
|
||||
|
||||
## 6.0.0-6-next.32
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [ef88da3b]
|
||||
- @verdaccio/config@6.0.0-6-next.54
|
||||
- @verdaccio/core@6.0.0-6-next.54
|
||||
- @verdaccio/logger@6.0.0-6-next.22
|
||||
- @verdaccio/local-storage@11.0.0-6-next.24
|
||||
- @verdaccio/utils@6.0.0-6-next.22
|
||||
|
||||
## 6.0.0-6-next.31
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.53
|
||||
- @verdaccio/logger@6.0.0-6-next.21
|
||||
- @verdaccio/local-storage@11.0.0-6-next.23
|
||||
- @verdaccio/config@6.0.0-6-next.53
|
||||
- @verdaccio/utils@6.0.0-6-next.21
|
||||
|
||||
## 6.0.0-6-next.30
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.52
|
||||
- @verdaccio/config@6.0.0-6-next.52
|
||||
- @verdaccio/logger@6.0.0-6-next.20
|
||||
- @verdaccio/local-storage@11.0.0-6-next.22
|
||||
- @verdaccio/utils@6.0.0-6-next.20
|
||||
|
||||
## 6.0.0-6-next.29
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [4b29d715]
|
||||
- @verdaccio/config@6.0.0-6-next.51
|
||||
- @verdaccio/core@6.0.0-6-next.51
|
||||
- @verdaccio/local-storage@11.0.0-6-next.21
|
||||
- @verdaccio/logger@6.0.0-6-next.19
|
||||
- @verdaccio/utils@6.0.0-6-next.19
|
||||
|
||||
## 6.0.0-6-next.28
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.50
|
||||
- @verdaccio/config@6.0.0-6-next.50
|
||||
- @verdaccio/logger@6.0.0-6-next.18
|
||||
- @verdaccio/local-storage@11.0.0-6-next.20
|
||||
- @verdaccio/utils@6.0.0-6-next.18
|
||||
|
||||
## 6.0.0-6-next.27
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/local-storage@11.0.0-6-next.19
|
||||
- @verdaccio/core@6.0.0-6-next.49
|
||||
- @verdaccio/config@6.0.0-6-next.49
|
||||
- @verdaccio/logger@6.0.0-6-next.17
|
||||
- @verdaccio/utils@6.0.0-6-next.17
|
||||
|
||||
## 6.0.0-6-next.26
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [43f32687]
|
||||
- Updated dependencies [9fc2e796]
|
||||
- Updated dependencies [62c24b63]
|
||||
- @verdaccio/core@6.0.0-6-next.48
|
||||
- @verdaccio/config@6.0.0-6-next.48
|
||||
- @verdaccio/local-storage@11.0.0-6-next.18
|
||||
- @verdaccio/utils@6.0.0-6-next.16
|
||||
- @verdaccio/logger@6.0.0-6-next.16
|
||||
|
||||
## 6.0.0-6-next.25
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @verdaccio/core@6.0.0-6-next.47
|
||||
- @verdaccio/config@6.0.0-6-next.47
|
||||
- @verdaccio/logger@6.0.0-6-next.15
|
||||
- @verdaccio/local-storage@11.0.0-6-next.17
|
||||
- @verdaccio/utils@6.0.0-6-next.15
|
||||
|
||||
## 6.0.0-6-next.24
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [b849128d]
|
||||
- @verdaccio/core@6.0.0-6-next.8
|
||||
- @verdaccio/config@6.0.0-6-next.17
|
||||
- @verdaccio/logger@6.0.0-6-next.14
|
||||
- @verdaccio/local-storage@11.0.0-6-next.16
|
||||
- @verdaccio/utils@6.0.0-6-next.14
|
||||
|
||||
## 6.0.0-6-next.23
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 351aeeaa: fix(deps): @verdaccio/utils should be a prod dep of local-storage
|
||||
- Updated dependencies [351aeeaa]
|
||||
- @verdaccio/core@6.0.0-6-next.7
|
||||
- @verdaccio/logger@6.0.0-6-next.13
|
||||
- @verdaccio/local-storage@11.0.0-6-next.15
|
||||
- @verdaccio/config@6.0.0-6-next.16
|
||||
- @verdaccio/utils@6.0.0-6-next.13
|
||||
|
||||
## 6.0.0-6-next.22
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [37274e4c]
|
||||
- @verdaccio/local-storage@11.0.0-6-next.14
|
||||
- @verdaccio/core@6.0.0-6-next.6
|
||||
- @verdaccio/logger@6.0.0-6-next.12
|
||||
|
||||
## 6.0.0-6-next.21
|
||||
|
||||
### Major Changes
|
||||
|
||||
- 292c0a37: feat!: replace deprecated request dependency by got
|
||||
|
||||
This is a big refactoring of the core, fetching dependencies, improve code, more tests and better stability. This is essential for the next release, will take some time but would allow modularize more the core.
|
||||
|
||||
## Notes
|
||||
|
||||
- Remove deprecated `request` by other `got`, retry improved, custom Agent ( got does not include it built-in)
|
||||
- Remove `async` dependency from storage (used by core) it was linked with proxy somehow safe to remove now
|
||||
- Refactor with promises instead callback wherever is possible
|
||||
- ~Document the API~
|
||||
- Improve testing, integration tests
|
||||
- Bugfix
|
||||
- Clean up old validations
|
||||
- Improve performance
|
||||
|
||||
## 💥 Breaking changes
|
||||
|
||||
- Plugin API methods were callbacks based are returning promises, this will break current storage plugins, check documentation for upgrade.
|
||||
- Write Tarball, Read Tarball methods parameters change, a new set of options like `AbortController` signals are being provided to the `addAbortSignal` can be internally used with Streams when a request is aborted. eg: `addAbortSignal(signal, fs.createReadStream(pathName));`
|
||||
- `@verdaccio/streams` stream abort support is legacy is being deprecated removed
|
||||
- Remove AWS and Google Cloud packages for future refactoring [#2574](https://github.com/verdaccio/verdaccio/pull/2574).
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [292c0a37]
|
||||
- Updated dependencies [a3a209b5]
|
||||
- Updated dependencies [00d1d2a1]
|
||||
- @verdaccio/config@6.0.0-6-next.15
|
||||
- @verdaccio/core@6.0.0-6-next.6
|
||||
- @verdaccio/logger@6.0.0-6-next.12
|
||||
- @verdaccio/local-storage@11.0.0-6-next.13
|
||||
- @verdaccio/utils@6.0.0-6-next.12
|
||||
|
||||
## 6.0.0-6-next.20
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [d43894e8]
|
||||
- Updated dependencies [d08fe29d]
|
||||
- @verdaccio/config@6.0.0-6-next.14
|
||||
- @verdaccio/local-storage@11.0.0-6-next.12
|
||||
- @verdaccio/core@6.0.0-6-next.5
|
||||
- @verdaccio/streams@11.0.0-6-next.5
|
||||
- @verdaccio/logger@6.0.0-6-next.11
|
||||
|
||||
## 6.0.0-6-next.19
|
||||
|
||||
### Major Changes
|
||||
|
||||
- 82cb0f2b: feat!: config.logs throw an error, logging config not longer accept array or logs property
|
||||
|
||||
### 💥 Breaking change
|
||||
|
||||
This is valid
|
||||
|
||||
```yaml
|
||||
log: { type: stdout, format: pretty, level: http }
|
||||
```
|
||||
|
||||
This is invalid
|
||||
|
||||
```yaml
|
||||
logs: { type: stdout, format: pretty, level: http }
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```yaml
|
||||
logs:
|
||||
- [{ type: stdout, format: pretty, level: http }]
|
||||
```
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 5167bb52: feat: ui search support for remote, local and private packages
|
||||
|
||||
The command `npm search` search globally and return all matches, with this improvement the user interface
|
||||
is powered with the same capabilities.
|
||||
|
||||
The UI also tag where is the origin the package with a tag, also provide the latest version and description of the package.
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [82cb0f2b]
|
||||
- Updated dependencies [5167bb52]
|
||||
- @verdaccio/config@6.0.0-6-next.13
|
||||
- @verdaccio/core@6.0.0-6-next.5
|
||||
- @verdaccio/logger@6.0.0-6-next.11
|
||||
- @verdaccio/local-storage@11.0.0-6-next.12
|
||||
- @verdaccio/utils@6.0.0-6-next.11
|
||||
- @verdaccio/streams@11.0.0-6-next.5
|
||||
|
||||
## 6.0.0-6-next.18
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [b78f3525]
|
||||
- @verdaccio/logger@6.0.0-6-next.10
|
||||
|
||||
## 6.0.0-6-next.17
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [730b5d8c]
|
||||
- @verdaccio/logger@6.0.0-6-next.9
|
||||
|
||||
## 6.0.0-6-next.16
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [a828271d]
|
||||
- Updated dependencies [24b9be02]
|
||||
- Updated dependencies [e75c0a3b]
|
||||
- Updated dependencies [b13a3fef]
|
||||
- @verdaccio/local-storage@11.0.0-6-next.11
|
||||
- @verdaccio/utils@6.0.0-6-next.10
|
||||
- @verdaccio/core@6.0.0-6-next.4
|
||||
- @verdaccio/logger@6.0.0-6-next.8
|
||||
- @verdaccio/config@6.0.0-6-next.12
|
||||
- @verdaccio/streams@11.0.0-6-next.5
|
||||
|
||||
## 6.0.0-6-next.15
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [f86c31ed]
|
||||
- @verdaccio/utils@6.0.0-6-next.9
|
||||
- @verdaccio/config@6.0.0-6-next.11
|
||||
- @verdaccio/local-storage@11.0.0-6-next.10
|
||||
|
||||
## 6.0.0-6-next.14
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6c1eb021]
|
||||
- @verdaccio/core@6.0.0-6-next.3
|
||||
- @verdaccio/logger@6.0.0-6-next.7
|
||||
- @verdaccio/config@6.0.0-6-next.10
|
||||
- @verdaccio/local-storage@11.0.0-6-next.10
|
||||
- @verdaccio/utils@6.0.0-6-next.8
|
||||
|
||||
## 6.0.0-6-next.13
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- b702ea36: abort search request support for proxy
|
||||
- 154b2ecd: refactor: remove @verdaccio/commons-api in favor @verdaccio/core and remove duplications
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [794af76c]
|
||||
- Updated dependencies [154b2ecd]
|
||||
- @verdaccio/config@6.0.0-6-next.9
|
||||
- @verdaccio/core@6.0.0-6-next.2
|
||||
- @verdaccio/streams@11.0.0-6-next.5
|
||||
- @verdaccio/logger@6.0.0-6-next.6
|
||||
- @verdaccio/utils@6.0.0-6-next.7
|
||||
- @verdaccio/local-storage@11.0.0-6-next.9
|
||||
|
||||
## 6.0.0-6-next.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [2c594910]
|
||||
- @verdaccio/logger@6.0.0-6-next.5
|
||||
|
||||
## 6.0.0-6-next.11
|
||||
|
||||
### Major Changes
|
||||
|
||||
- 459b6fa7: refactor: search v1 endpoint and local-database
|
||||
|
||||
- refactor search `api v1` endpoint, improve performance
|
||||
- remove usage of `async` dependency https://github.com/verdaccio/verdaccio/issues/1225
|
||||
- refactor method storage class
|
||||
- create new module `core` to reduce the ammount of modules with utilities
|
||||
- use `undici` instead `node-fetch`
|
||||
- use `fastify` instead `express` for functional test
|
||||
|
||||
### Breaking changes
|
||||
|
||||
- plugin storage API changes
|
||||
- remove old search endpoint (return 404)
|
||||
- filter local private packages at plugin level
|
||||
|
||||
The storage api changes for methods `get`, `add`, `remove` as promise base. The `search` methods also changes and recieves a `query` object that contains all query params from the client.
|
||||
|
||||
```ts
|
||||
export interface IPluginStorage<T> extends IPlugin {
|
||||
add(name: string): Promise<void>;
|
||||
remove(name: string): Promise<void>;
|
||||
get(): Promise<any>;
|
||||
init(): Promise<void>;
|
||||
getSecret(): Promise<string>;
|
||||
setSecret(secret: string): Promise<any>;
|
||||
getPackageStorage(packageInfo: string): IPackageStorage;
|
||||
search(query: searchUtils.SearchQuery): Promise<searchUtils.SearchItem[]>;
|
||||
saveToken(token: Token): Promise<any>;
|
||||
deleteToken(user: string, tokenKey: string): Promise<any>;
|
||||
readTokens(filter: TokenFilter): Promise<Token[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [459b6fa7]
|
||||
- @verdaccio/config@6.0.0-6-next.8
|
||||
- @verdaccio/commons-api@11.0.0-6-next.4
|
||||
- @verdaccio/core@6.0.0-6-next.1
|
||||
- @verdaccio/local-storage@11.0.0-6-next.8
|
||||
- @verdaccio/streams@11.0.0-6-next.4
|
||||
- @verdaccio/utils@6.0.0-6-next.6
|
||||
- @verdaccio/logger@6.0.0-6-next.4
|
||||
|
||||
## 6.0.0-6-next.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [df0da3d6]
|
||||
- @verdaccio/local-storage@11.0.0-6-next.7
|
||||
|
||||
## 6.0.0-6-next.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [d2c65da9]
|
||||
- @verdaccio/utils@6.0.0-6-next.5
|
||||
- @verdaccio/config@6.0.0-6-next.7
|
||||
|
||||
## 6.0.0-6-next.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1b217fd3]
|
||||
- @verdaccio/config@6.0.0-6-next.6
|
||||
- @verdaccio/local-storage@11.0.0-6-next.6
|
||||
|
||||
## 6.0.0-6-next.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1810ed0d]
|
||||
- Updated dependencies [648575aa]
|
||||
- @verdaccio/config@6.0.0-6-next.5
|
||||
- @verdaccio/utils@6.0.0-6-next.4
|
||||
|
||||
## 6.0.0-6-next.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5c5057fc]
|
||||
- @verdaccio/config@6.0.0-6-next.4
|
||||
- @verdaccio/logger@6.0.0-6-next.4
|
||||
- @verdaccio/local-storage@11.0.0-6-next.5
|
||||
- @verdaccio/streams@11.0.0-alpha.3
|
||||
|
||||
## 6.0.0-6-next.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [cb2281a5]
|
||||
- @verdaccio/local-storage@11.0.0-6-next.5
|
||||
|
||||
## 5.0.0-alpha.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fecbb9be: chore: add release step to private regisry on merge changeset pr
|
||||
- Updated dependencies [fecbb9be]
|
||||
- @verdaccio/local-storage@10.0.0-alpha.4
|
||||
- @verdaccio/config@5.0.0-alpha.3
|
||||
- @verdaccio/commons-api@10.0.0-alpha.3
|
||||
- @verdaccio/streams@10.0.0-alpha.3
|
||||
- @verdaccio/logger@5.0.0-alpha.3
|
||||
- @verdaccio/utils@5.0.0-alpha.3
|
||||
|
||||
## 5.0.0-alpha.3
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 54c58d1e: feat: add server rate limit protection to all request
|
||||
|
||||
To modify custom values, use the server settings property.
|
||||
|
||||
```markdown
|
||||
server:
|
||||
|
||||
## https://www.npmjs.com/package/express-rate-limit#configuration-options
|
||||
|
||||
rateLimit:
|
||||
windowMs: 1000
|
||||
max: 10000
|
||||
```
|
||||
|
||||
The values are intended to be high, if you want to improve security of your server consider
|
||||
using different values.
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [54c58d1e]
|
||||
- @verdaccio/config@5.0.0-alpha.2
|
||||
- @verdaccio/commons-api@10.0.0-alpha.2
|
||||
- @verdaccio/local-storage@10.0.0-alpha.3
|
||||
- @verdaccio/streams@10.0.0-alpha.2
|
||||
- @verdaccio/logger@5.0.0-alpha.2
|
||||
- @verdaccio/utils@5.0.0-alpha.2
|
||||
|
||||
## 5.0.0-alpha.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [2a327c4b]
|
||||
- @verdaccio/local-storage@10.0.0-alpha.2
|
||||
|
||||
## 5.0.0-alpha.1
|
||||
|
||||
### Major Changes
|
||||
|
||||
- d87fa026: feat!: experiments config renamed to flags
|
||||
|
||||
- The `experiments` configuration is renamed to `flags`. The functionality is exactly the same.
|
||||
|
||||
```js
|
||||
flags: token: false;
|
||||
search: false;
|
||||
```
|
||||
|
||||
- The `self_path` property from the config file is being removed in favor of `config_file` full path.
|
||||
- Refactor `config` module, better types and utilities
|
||||
|
||||
- da1ee9c8: - Replace signature handler for legacy tokens by removing deprecated crypto.createDecipher by createCipheriv
|
||||
|
||||
- Introduce environment variables for legacy tokens
|
||||
|
||||
### Code Improvements
|
||||
|
||||
- Add debug library for improve developer experience
|
||||
|
||||
### Breaking change
|
||||
|
||||
- The new signature invalidates all previous tokens generated by Verdaccio 4 or previous versions.
|
||||
- The secret key must have 32 characters long.
|
||||
|
||||
### New environment variables
|
||||
|
||||
- `VERDACCIO_LEGACY_ALGORITHM`: Allows to define the specific algorithm for the token signature which by default is `aes-256-ctr`
|
||||
- `VERDACCIO_LEGACY_ENCRYPTION_KEY`: By default, the token stores in the database, but using this variable allows to get it from memory
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 26b494cb: feat: add typescript project references settings
|
||||
|
||||
Reading https://ebaytech.berlin/optimizing-multi-package-apps-with-typescript-project-references-d5c57a3b4440 I realized I can use project references to solve the issue to pre-compile modules on develop mode.
|
||||
|
||||
It allows to navigate (IDE) trough the packages without need compile the packages.
|
||||
|
||||
Add two `tsconfig`, one using the previous existing configuration that is able to produce declaration files (`tsconfig.build`) and a new one `tsconfig` which is enables [_projects references_](https://www.typescriptlang.org/docs/handbook/project-references.html).
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- ae52ba35: refactor: migrate request to node-fetch at hooks package
|
||||
- b57b4338: Enable prerelease mode with **changesets**
|
||||
- 31af0164: ESLint Warnings Fixed
|
||||
|
||||
Related to issue #1461
|
||||
|
||||
- max-len: most of the sensible max-len errors are fixed
|
||||
- no-unused-vars: most of these types of errors are fixed by deleting not needed declarations
|
||||
- @typescript-eslint/no-unused-vars: same as above
|
||||
|
||||
- Updated dependencies [d87fa026]
|
||||
- Updated dependencies [da1ee9c8]
|
||||
- Updated dependencies [26b494cb]
|
||||
- Updated dependencies [b57b4338]
|
||||
- Updated dependencies [add778d5]
|
||||
- Updated dependencies [31af0164]
|
||||
- @verdaccio/config@5.0.0-alpha.1
|
||||
- @verdaccio/commons-api@10.0.0-alpha.1
|
||||
- @verdaccio/local-storage@10.0.0-alpha.1
|
||||
- @verdaccio/streams@10.0.0-alpha.1
|
||||
- @verdaccio/logger@5.0.0-alpha.1
|
||||
- @verdaccio/utils@5.0.0-alpha.1
|
||||
|
||||
## 5.0.0-alpha.1
|
||||
|
||||
### Major Changes
|
||||
|
||||
- d87fa0268: feat!: experiments config renamed to flags
|
||||
|
||||
- The `experiments` configuration is renamed to `flags`. The functionality is exactly the same.
|
||||
|
||||
```js
|
||||
flags: token: false;
|
||||
search: false;
|
||||
```
|
||||
|
||||
- The `self_path` property from the config file is being removed in favor of `config_file` full path.
|
||||
- Refactor `config` module, better types and utilities
|
||||
|
||||
- da1ee9c82: - Replace signature handler for legacy tokens by removing deprecated crypto.createDecipher by createCipheriv
|
||||
|
||||
- Introduce environment variables for legacy tokens
|
||||
|
||||
### Code Improvements
|
||||
|
||||
- Add debug library for improve developer experience
|
||||
|
||||
### Breaking change
|
||||
|
||||
- The new signature invalidates all previous tokens generated by Verdaccio 4 or previous versions.
|
||||
- The secret key must have 32 characters long.
|
||||
|
||||
### New environment variables
|
||||
|
||||
- `VERDACCIO_LEGACY_ALGORITHM`: Allows to define the specific algorithm for the token signature which by default is `aes-256-ctr`
|
||||
- `VERDACCIO_LEGACY_ENCRYPTION_KEY`: By default, the token stores in the database, but using this variable allows to get it from memory
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 26b494cbd: feat: add typescript project references settings
|
||||
|
||||
Reading https://ebaytech.berlin/optimizing-multi-package-apps-with-typescript-project-references-d5c57a3b4440 I realized I can use project references to solve the issue to pre-compile modules on develop mode.
|
||||
|
||||
It allows to navigate (IDE) trough the packages without need compile the packages.
|
||||
|
||||
Add two `tsconfig`, one using the previous existing configuration that is able to produce declaration files (`tsconfig.build`) and a new one `tsconfig` which is enables [_projects references_](https://www.typescriptlang.org/docs/handbook/project-references.html).
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- ae52ba352: refactor: migrate request to node-fetch at hooks package
|
||||
- b57b43388: Enable prerelease mode with **changesets**
|
||||
- 31af01641: ESLint Warnings Fixed
|
||||
|
||||
Related to issue #1461
|
||||
|
||||
- max-len: most of the sensible max-len errors are fixed
|
||||
- no-unused-vars: most of these types of errors are fixed by deleting not needed declarations
|
||||
- @typescript-eslint/no-unused-vars: same as above
|
||||
|
||||
- Updated dependencies [d87fa0268]
|
||||
- Updated dependencies [da1ee9c82]
|
||||
- Updated dependencies [26b494cbd]
|
||||
- Updated dependencies [b57b43388]
|
||||
- Updated dependencies [add778d55]
|
||||
- Updated dependencies [31af01641]
|
||||
- @verdaccio/config@5.0.0-alpha.1
|
||||
- @verdaccio/commons-api@10.0.0-alpha.0
|
||||
- @verdaccio/local-storage@10.0.0-alpha.0
|
||||
- @verdaccio/streams@10.0.0-alpha.0
|
||||
- @verdaccio/logger@5.0.0-alpha.1
|
||||
- @verdaccio/utils@5.0.0-alpha.1
|
||||
|
|
@ -3,10 +3,9 @@ const config = require('../../jest/config');
|
|||
module.exports = Object.assign({}, config, {
|
||||
coverageThreshold: {
|
||||
global: {
|
||||
branches: 79,
|
||||
functions: 94,
|
||||
lines: 87,
|
||||
statements: 87,
|
||||
branches: 0,
|
||||
functions: 0,
|
||||
lines: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "@verdaccio/search",
|
||||
"version": "7.0.0-next.2",
|
||||
"description": "verdaccio search utitlity tools",
|
||||
"main": "./build/dist.js",
|
||||
"version": "7.0.0-next.0",
|
||||
"description": "verdaccio search proxy",
|
||||
"main": "./build/index.js",
|
||||
"types": "build/index.d.ts",
|
||||
"author": {
|
||||
"name": "Juan Picado",
|
||||
|
|
@ -26,21 +26,30 @@
|
|||
"verdaccio"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
"node": ">=18"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf ./build",
|
||||
"test": "vitest run",
|
||||
"test": "jest",
|
||||
"type-check": "tsc --noEmit -p tsconfig.build.json",
|
||||
"build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json",
|
||||
"build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps",
|
||||
"build": "esbuild src/index.ts --bundle --outfile=build/dist.js --platform=node --target=node12 && pnpm run build:types"
|
||||
"watch": "pnpm build:js -- --watch",
|
||||
"build": "pnpm run build:js && pnpm run build:types"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": "4.3.4",
|
||||
"lodash": "4.17.21",
|
||||
"@verdaccio/config": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/core": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/logger": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/proxy": "workspace:7.0.0-next-7.11"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@verdaccio/types": "workspace:12.0.0-next.2",
|
||||
"@orama/orama": "1.2.4",
|
||||
"debug": "4.3.4",
|
||||
"esbuild": "0.14.10"
|
||||
"mockdate": "3.0.5",
|
||||
"nock": "13.5.1",
|
||||
"node-mocks-http": "1.14.1"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
export { default as SearchMemoryIndexer } from './indexer';
|
||||
export { Search as default } from './search';
|
||||
export * from './search-utils';
|
||||
|
|
|
|||
15
packages/search/src/search-utils.ts
Normal file
15
packages/search/src/search-utils.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { orderBy } from 'lodash';
|
||||
|
||||
import { searchUtils } from '@verdaccio/core';
|
||||
|
||||
export function removeDuplicates(results: searchUtils.SearchPackageItem[]) {
|
||||
const pkgNames: any[] = [];
|
||||
const orderByResults = orderBy(results, ['verdaccioPrivate', 'asc']);
|
||||
return orderByResults.filter((pkg) => {
|
||||
if (pkgNames.includes(pkg?.package?.name)) {
|
||||
return false;
|
||||
}
|
||||
pkgNames.push(pkg?.package?.name);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
104
packages/search/src/search.ts
Normal file
104
packages/search/src/search.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import buildDebug from 'debug';
|
||||
import _ from 'lodash';
|
||||
import { PassThrough } from 'stream';
|
||||
|
||||
import { searchUtils } from '@verdaccio/core';
|
||||
import { IProxy, ProxyInstanceList, ProxySearchParams, setupUpLinks } from '@verdaccio/proxy';
|
||||
import { Config, Logger } from '@verdaccio/types';
|
||||
|
||||
import { removeDuplicates } from './search-utils';
|
||||
|
||||
const debug = buildDebug('verdaccio:search');
|
||||
|
||||
class Search {
|
||||
public readonly uplinks: ProxyInstanceList;
|
||||
public readonly logger: Logger;
|
||||
constructor(config: Config, logger: Logger) {
|
||||
this.logger = logger.child({ module: 'proxy' });
|
||||
this.uplinks = setupUpLinks(config, this.logger);
|
||||
}
|
||||
|
||||
private getProxyList() {
|
||||
const uplinksList = Object.keys(this.uplinks);
|
||||
|
||||
return uplinksList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle search on packages and proxies.
|
||||
* Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
|
||||
* to a stream, once the proxies request has finished search in local storage for all packages
|
||||
* (privated and cached).
|
||||
*/
|
||||
public async search(options: ProxySearchParams): Promise<searchUtils.SearchPackageItem[]> {
|
||||
const results: searchUtils.SearchPackageItem[] = [];
|
||||
const upLinkList = this.getProxyList();
|
||||
// const transformResults = new TransFormResults({ objectMode: true });
|
||||
const streamPassThrough = new PassThrough({ objectMode: true });
|
||||
debug('uplinks found %s', upLinkList.length);
|
||||
const searchUplinksStreams = upLinkList.map((uplinkId: string) => {
|
||||
const uplink = this.uplinks[uplinkId];
|
||||
if (!uplink) {
|
||||
// this line should never happens
|
||||
this.logger.error({ uplinkId }, 'uplink @upLinkId not found');
|
||||
}
|
||||
return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
|
||||
});
|
||||
|
||||
try {
|
||||
debug('searching on %s uplinks...', searchUplinksStreams?.length);
|
||||
// only process those streams end successfully, if all request fails
|
||||
// just include local storage results (if local fails then return 500)
|
||||
await Promise.allSettled([...searchUplinksStreams]);
|
||||
streamPassThrough.end();
|
||||
|
||||
for await (const chunk of streamPassThrough) {
|
||||
if (_.isArray(chunk)) {
|
||||
(chunk as searchUtils.SearchItem[])
|
||||
.filter((pkgItem) => {
|
||||
debug(`streaming remote pkg name ${pkgItem?.package?.name}`);
|
||||
return true;
|
||||
})
|
||||
.forEach((pkgItem) => {
|
||||
// @ts-ignore
|
||||
return results.push({
|
||||
...pkgItem,
|
||||
verdaccioPkgCached: false,
|
||||
verdaccioPrivate: false,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
debug('searching all uplinks done');
|
||||
} catch (err: any) {
|
||||
this.logger.error({ err: err?.message }, ' error on uplinks search @{err}');
|
||||
throw err;
|
||||
}
|
||||
|
||||
return removeDuplicates(results);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume the upstream and pipe it to a transformable stream.
|
||||
*/
|
||||
private consumeSearchStream(
|
||||
uplinkId: string,
|
||||
uplink: IProxy,
|
||||
options: ProxySearchParams,
|
||||
searchPassThrough: PassThrough
|
||||
): Promise<any> {
|
||||
return uplink.search({ ...options }).then((bodyStream) => {
|
||||
bodyStream.pipe(searchPassThrough, { end: false });
|
||||
bodyStream.on('error', (err: any): void => {
|
||||
this.logger.error(
|
||||
{ uplinkId, err: err },
|
||||
'search error for uplink @{uplinkId}: @{err?.message}'
|
||||
);
|
||||
searchPassThrough.end();
|
||||
});
|
||||
return new Promise((resolve) => bodyStream.on('end', resolve));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export { Search };
|
||||
273
packages/search/test/partials/search.json
Normal file
273
packages/search/test/partials/search.json
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
{
|
||||
"objects": [
|
||||
{
|
||||
"package": {
|
||||
"name": "verdaccio",
|
||||
"scope": "unscoped",
|
||||
"version": "5.29.2",
|
||||
"description": "A lightweight private npm proxy registry",
|
||||
"keywords": [
|
||||
"private",
|
||||
"package",
|
||||
"repository",
|
||||
"registry",
|
||||
"enterprise",
|
||||
"modules",
|
||||
"proxy",
|
||||
"server",
|
||||
"verdaccio"
|
||||
],
|
||||
"date": "2024-02-21T19:56:45.379Z",
|
||||
"links": {
|
||||
"npm": "https://www.npmjs.com/package/verdaccio",
|
||||
"homepage": "https://verdaccio.org",
|
||||
"repository": "https://github.com/verdaccio/verdaccio",
|
||||
"bugs": "https://github.com/verdaccio/verdaccio/issues"
|
||||
},
|
||||
"author": {
|
||||
"name": "Verdaccio Maintainers",
|
||||
"email": "test@test.com",
|
||||
"username": "verdaccio.npm"
|
||||
},
|
||||
"publisher": { "username": "verdaccio.npm", "email": "test@test.com" },
|
||||
"maintainers": [
|
||||
{ "username": "jotadeveloper", "email": "test@test.com" },
|
||||
{ "username": "ayusharma", "email": "test@test.com" },
|
||||
{ "username": "trentearl", "email": "test@test.com" },
|
||||
{ "username": "jmwilkinson", "email": "test@test.com" },
|
||||
{ "username": "sergiohgz", "email": "test@test.com" },
|
||||
{ "username": "verdaccio.npm", "email": "test@test.com" }
|
||||
]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.28923397536716566,
|
||||
"detail": {
|
||||
"quality": 0.39403701233442867,
|
||||
"popularity": 0.1553034428576298,
|
||||
"maintenance": 0.3333333333333333
|
||||
}
|
||||
},
|
||||
"searchScore": 100000.26
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "@verdaccio/file-locking",
|
||||
"scope": "verdaccio",
|
||||
"version": "10.3.1",
|
||||
"description": "library that handle file locking",
|
||||
"keywords": ["verdaccio", "lock", "fs"],
|
||||
"date": "2023-03-29T18:48:01.509Z",
|
||||
"links": {
|
||||
"npm": "https://www.npmjs.com/package/%40verdaccio%2Ffile-locking",
|
||||
"homepage": "https://verdaccio.org",
|
||||
"repository": "https://github.com/verdaccio/monorepo",
|
||||
"bugs": "https://github.com/verdaccio/monorepo/issues"
|
||||
},
|
||||
"author": {
|
||||
"name": "Juan Picado",
|
||||
"email": "test@test.com",
|
||||
"username": "jotadeveloper"
|
||||
},
|
||||
"publisher": { "username": "verdaccio.npm", "email": "test@test.com" },
|
||||
"maintainers": [
|
||||
{ "username": "sergiohgz", "email": "test@test.com" },
|
||||
{ "username": "verdaccio.npm", "email": "test@test.com" },
|
||||
{ "username": "jotadeveloper", "email": "test@test.com" },
|
||||
{ "username": "ayusharma", "email": "test@test.com" }
|
||||
]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.4347300973147867,
|
||||
"detail": {
|
||||
"quality": 0.8773185907360584,
|
||||
"popularity": 0.15732960498192844,
|
||||
"maintenance": 0.33276902385798346
|
||||
}
|
||||
},
|
||||
"searchScore": 0.0010177421
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "verdaccio-service-construct",
|
||||
"scope": "unscoped",
|
||||
"version": "0.0.602",
|
||||
"keywords": ["cdk"],
|
||||
"date": "2024-03-09T01:14:46.812Z",
|
||||
"links": { "npm": "https://www.npmjs.com/package/verdaccio-service-construct" },
|
||||
"author": {
|
||||
"name": "Ayush Goyal",
|
||||
"email": "ayush987goyal@gmail.com",
|
||||
"username": "ayush987goyal"
|
||||
},
|
||||
"publisher": { "username": "ayush987goyal", "email": "ayush987goyal@gmail.com" },
|
||||
"maintainers": [{ "username": "ayush987goyal", "email": "ayush987goyal@gmail.com" }]
|
||||
},
|
||||
"flags": { "insecure": 0, "unstable": true },
|
||||
"score": {
|
||||
"final": 0.22683523291517088,
|
||||
"detail": {
|
||||
"quality": 0.32276139168703444,
|
||||
"popularity": 0.038114710692553955,
|
||||
"maintenance": 0.3333333333333333
|
||||
}
|
||||
},
|
||||
"searchScore": 3.968321e-8
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "verdaccio-theme-hilio",
|
||||
"scope": "unscoped",
|
||||
"version": "1.14.5",
|
||||
"description": "Verdaccio User Interface",
|
||||
"keywords": ["verdaccio", "verdaccio-plugin", "verdaccio-theme"],
|
||||
"date": "2021-01-12T05:48:12.643Z",
|
||||
"links": {
|
||||
"npm": "https://www.npmjs.com/package/verdaccio-theme-hilio",
|
||||
"homepage": "https://verdaccio.org",
|
||||
"repository": "https://github.com/verdaccio/ui",
|
||||
"bugs": "https://github.com/verdaccio/ui/issues"
|
||||
},
|
||||
"author": { "name": "Verdaccio Core Team", "email": "test@test.com" },
|
||||
"publisher": { "username": "joebnb", "email": "joebnb@qq.com" },
|
||||
"maintainers": [{ "username": "joebnb", "email": "joebnb@qq.com" }]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.20823236027862208,
|
||||
"detail": {
|
||||
"quality": 0.6388395536236707,
|
||||
"popularity": 0.022545843955562445,
|
||||
"maintenance": 0.02482699659164002
|
||||
}
|
||||
},
|
||||
"searchScore": 3.8766167e-8
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "@hamstudy/verdaccio-aws-s3-storage-sse",
|
||||
"scope": "hamstudy",
|
||||
"version": "10.3.2",
|
||||
"description": "AWS S3 storage implementation for Verdaccio - fork that adds support for SSE-C, SSE-S3, and AWS:KMS",
|
||||
"keywords": ["verdaccio", "plugin", "storage", "aws"],
|
||||
"date": "2022-10-04T21:39:04.907Z",
|
||||
"links": {
|
||||
"npm": "https://www.npmjs.com/package/%40hamstudy%2Fverdaccio-aws-s3-storage-sse",
|
||||
"homepage": "https://verdaccio.org",
|
||||
"repository": "https://github.com/taxilian/verdaccio-monorepo",
|
||||
"bugs": "https://github.com/verdaccio/monorepo/issues"
|
||||
},
|
||||
"author": {
|
||||
"name": "Richard Bateman",
|
||||
"email": "taxilian@gmail.com",
|
||||
"username": "taxilian"
|
||||
},
|
||||
"publisher": { "username": "taxilian", "email": "taxilian@gmail.com" },
|
||||
"maintainers": [
|
||||
{ "username": "rumbcam", "email": "kd7rmx@batemansr.us" },
|
||||
{ "username": "taxilian", "email": "taxilian@gmail.com" },
|
||||
{ "username": "bloveridge", "email": "bloveridge@gmail.com" }
|
||||
]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.220856041974831,
|
||||
"detail": {
|
||||
"quality": 0.6217078604565084,
|
||||
"popularity": 0.0008629747032998004,
|
||||
"maintenance": 0.09726183626206726
|
||||
}
|
||||
},
|
||||
"searchScore": 3.8267025e-8
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "testing-verdaccio",
|
||||
"scope": "unscoped",
|
||||
"version": "1.1.2",
|
||||
"description": "this is just for Assignment puropse",
|
||||
"date": "2023-03-20T07:58:49.325Z",
|
||||
"links": { "npm": "https://www.npmjs.com/package/testing-verdaccio" },
|
||||
"publisher": { "username": "samyak3009", "email": "samyak3009@gmail.com" },
|
||||
"maintainers": [{ "username": "samyak3009", "email": "samyak3009@gmail.com" }]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.23106132265338952,
|
||||
"detail": {
|
||||
"quality": 0.39172042177373956,
|
||||
"popularity": 0.0012551184665870981,
|
||||
"maintenance": 0.3231597275941777
|
||||
}
|
||||
},
|
||||
"searchScore": 3.383671e-8
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "verdaccio-staryauthgroup",
|
||||
"scope": "unscoped",
|
||||
"version": "0.0.1",
|
||||
"description": "A verdaccio plugin to control auth group",
|
||||
"keywords": ["verdaccio,auth,plugin,verdaccio-]"],
|
||||
"date": "2023-07-12T01:59:29.928Z",
|
||||
"links": { "npm": "https://www.npmjs.com/package/verdaccio-staryauthgroup" },
|
||||
"author": { "name": "weihuago4", "email": "weihuago4@gmail.com" },
|
||||
"publisher": { "username": "liuweihua", "email": "weihualau@126.com" },
|
||||
"maintainers": [{ "username": "liuweihua", "email": "weihualau@126.com" }]
|
||||
},
|
||||
"flags": { "insecure": 0, "unstable": true },
|
||||
"score": {
|
||||
"final": 0.2222439493540912,
|
||||
"detail": {
|
||||
"quality": 0.4337548917356935,
|
||||
"popularity": 0.0006994438979026641,
|
||||
"maintenance": 0.262493361340335
|
||||
}
|
||||
},
|
||||
"searchScore": 3.2598397e-8
|
||||
},
|
||||
{
|
||||
"package": {
|
||||
"name": "verdaccio-auth-gitlab",
|
||||
"scope": "unscoped",
|
||||
"version": "2.0.0-beta.11",
|
||||
"description": "Verdaccio authentication plugin by gitlab personal access token or oauth token or ci job token.",
|
||||
"keywords": [
|
||||
"verdaccio",
|
||||
"authentication",
|
||||
"auth",
|
||||
"plugin",
|
||||
"gitlab",
|
||||
"personal",
|
||||
"access",
|
||||
"oauth",
|
||||
"ci",
|
||||
"job",
|
||||
"token"
|
||||
],
|
||||
"date": "2022-12-07T11:00:06.767Z",
|
||||
"links": {
|
||||
"npm": "https://www.npmjs.com/package/verdaccio-auth-gitlab",
|
||||
"homepage": "https://github.com/pfdgithub/verdaccio-auth-gitlab",
|
||||
"repository": "https://github.com/pfdgithub/verdaccio-auth-gitlab",
|
||||
"bugs": "https://github.com/pfdgithub/verdaccio-auth-gitlab/issues"
|
||||
},
|
||||
"publisher": { "username": "pfdnpm", "email": "pfdfree@gmail.com" },
|
||||
"maintainers": [{ "username": "pfdnpm", "email": "pfdfree@gmail.com" }]
|
||||
},
|
||||
"flags": { "insecure": 0 },
|
||||
"score": {
|
||||
"final": 0.2194863817767726,
|
||||
"detail": {
|
||||
"quality": 0.4542787957560298,
|
||||
"popularity": 0.0025512045517420002,
|
||||
"maintenance": 0.2351709184481542
|
||||
}
|
||||
},
|
||||
"searchScore": 2.5283079e-8
|
||||
}
|
||||
],
|
||||
"total": 351,
|
||||
"time": "Sat Mar 09 2024 15:20:36 GMT+0000 (Coordinated Universal Time)"
|
||||
}
|
||||
102
packages/search/test/search.test.ts
Normal file
102
packages/search/test/search.test.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import nock from 'nock';
|
||||
|
||||
import { Config, getDefaultConfig } from '@verdaccio/config';
|
||||
import { logger, setup } from '@verdaccio/logger';
|
||||
|
||||
import { Search } from '../src/search';
|
||||
|
||||
setup({});
|
||||
|
||||
const domain = 'https://registry.npmjs.org';
|
||||
|
||||
describe('search', () => {
|
||||
const response = require('./partials/search.json');
|
||||
test('search', async () => {
|
||||
nock(domain).get('/-/v1/search').reply(200, response);
|
||||
const abort = new AbortController();
|
||||
const config = new Config(getDefaultConfig());
|
||||
const searchInstance = new Search(config, logger);
|
||||
const results = await searchInstance.search({
|
||||
query: { text: 'verdaccio', maintenance: 0, popularity: 0, quality: 0, size: 0 },
|
||||
abort,
|
||||
url: '/-/v1/search',
|
||||
});
|
||||
expect(results).toHaveLength(8);
|
||||
|
||||
expect(results[0]).toEqual({
|
||||
package: {
|
||||
name: 'verdaccio',
|
||||
scope: 'unscoped',
|
||||
version: '5.29.2',
|
||||
description: 'A lightweight private npm proxy registry',
|
||||
keywords: [
|
||||
'private',
|
||||
'package',
|
||||
'repository',
|
||||
'registry',
|
||||
'enterprise',
|
||||
'modules',
|
||||
'proxy',
|
||||
'server',
|
||||
'verdaccio',
|
||||
],
|
||||
date: '2024-02-21T19:56:45.379Z',
|
||||
links: {
|
||||
npm: 'https://www.npmjs.com/package/verdaccio',
|
||||
homepage: 'https://verdaccio.org',
|
||||
repository: 'https://github.com/verdaccio/verdaccio',
|
||||
bugs: 'https://github.com/verdaccio/verdaccio/issues',
|
||||
},
|
||||
author: {
|
||||
name: 'Verdaccio Maintainers',
|
||||
email: 'test@test.com',
|
||||
username: 'verdaccio.npm',
|
||||
},
|
||||
publisher: {
|
||||
username: 'verdaccio.npm',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
maintainers: [
|
||||
{
|
||||
username: 'jotadeveloper',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
{
|
||||
username: 'ayusharma',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
{
|
||||
username: 'trentearl',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
{
|
||||
username: 'jmwilkinson',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
{
|
||||
username: 'sergiohgz',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
{
|
||||
username: 'verdaccio.npm',
|
||||
email: 'test@test.com',
|
||||
},
|
||||
],
|
||||
},
|
||||
flags: {
|
||||
insecure: 0,
|
||||
},
|
||||
score: {
|
||||
final: 0.28923397536716566,
|
||||
detail: {
|
||||
quality: 0.39403701233442867,
|
||||
popularity: 0.1553034428576298,
|
||||
maintenance: 0.3333333333333333,
|
||||
},
|
||||
},
|
||||
searchScore: 100000.26,
|
||||
verdaccioPkgCached: false,
|
||||
verdaccioPrivate: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -10,12 +10,6 @@
|
|||
{
|
||||
"path": "../config"
|
||||
},
|
||||
{
|
||||
"path": "../core/core"
|
||||
},
|
||||
{
|
||||
"path": "../logger/logger"
|
||||
},
|
||||
{
|
||||
"path": "../utils"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ async function tarballRoute(fastify: FastifyInstance) {
|
|||
const { package: pkg, filename } = request.params;
|
||||
debug('stream tarball for %s@%s', pkg, filename);
|
||||
const abort = new AbortController();
|
||||
const stream = (await fastify.storage.getTarballNext(pkg, filename, {
|
||||
const stream = (await fastify.storage.getTarball(pkg, filename, {
|
||||
signal: abort.signal,
|
||||
// enableRemote: true,
|
||||
})) as any;
|
||||
|
|
@ -46,7 +46,7 @@ async function tarballRoute(fastify: FastifyInstance) {
|
|||
const { scope, name, filename } = request.params;
|
||||
const scopedPackage = `${scope}/${name}`;
|
||||
debug('stream scope tarball for %s@%s', scopedPackage, filename);
|
||||
const stream = (await fastify.storage.getTarballNext(scopedPackage, filename, {
|
||||
const stream = (await fastify.storage.getTarball(scopedPackage, filename, {
|
||||
signal: abort.signal,
|
||||
// enableRemote: true,
|
||||
})) as any;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,32 @@
|
|||
# @verdaccio/store
|
||||
# Verdaccio Search API
|
||||
|
||||
[](https://opencollective.com/verdaccio)
|
||||
[](https://stackshare.io/verdaccio)
|
||||
[](https://github.com/verdaccio/verdaccio/blob/master/LICENSE)
|
||||
[](https://crowdin.com/project/verdaccio)
|
||||
[](https://www.tickgit.com/browse?repo=github.com/verdaccio/verdaccio)
|
||||
The **Search** class in Verdaccio provides a convenient API for searching packages across both configured proxies and local storage. It enables efficient package discovery and retrieval by aggregating search results from multiple upstream sources.
|
||||
|
||||
[](https://twitter.com/verdaccio_npm)
|
||||
[](https://github.com/verdaccio/verdaccio/stargazers)
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @verdaccio/search
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```ts
|
||||
import { Config } from '@verdaccio/config';
|
||||
import { logger } from '@verdaccio/logger';
|
||||
import { Search } from '@verdaccio/search';
|
||||
|
||||
const config = new Config(configYaml);
|
||||
// Instantiate Search class
|
||||
const search = new Search(config, logger);
|
||||
|
||||
// Define search parameters
|
||||
const searchParams = {
|
||||
// specify search parameters as needed
|
||||
};
|
||||
|
||||
// Perform a search and retrieve the results
|
||||
const searchResults = await search.search(searchParams);
|
||||
```
|
||||
|
||||
## Donations
|
||||
|
||||
|
|
@ -15,58 +34,6 @@ Verdaccio is run by **volunteers**; nobody is working full-time on it. If you fi
|
|||
|
||||
**[Donate](https://opencollective.com/verdaccio)** 💵👍🏻 starting from _\$1/month_ or just one single contribution.
|
||||
|
||||
## Report a vulnerability
|
||||
|
||||
If you want to report a security vulnerability, please follow the steps which we have defined for you in our [security policy](https://github.com/verdaccio/verdaccio/security/policy).
|
||||
|
||||
## Open Collective Sponsors
|
||||
|
||||
Support this project by becoming a sponsor. Your logo will show up here with a link to your website. [[Become a sponsor](https://opencollective.com/verdaccio#sponsor)]
|
||||
|
||||
[](https://opencollective.com/verdaccio/sponsor/0/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/1/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/2/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/3/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/4/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/5/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/6/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/7/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/8/website)
|
||||
[](https://opencollective.com/verdaccio/sponsor/9/website)
|
||||
|
||||
## Open Collective Backers
|
||||
|
||||
Thank you to all our backers! 🙏 [[Become a backer](https://opencollective.com/verdaccio#backer)]
|
||||
|
||||
[](https://opencollective.com/verdaccio#backers)
|
||||
|
||||
## Special Thanks
|
||||
|
||||
Thanks to the following companies to help us to achieve our goals providing free open source licenses.
|
||||
|
||||
[](https://www.jetbrains.com/)
|
||||
[](https://crowdin.com/)
|
||||
[](https://balsamiq.com/)
|
||||
|
||||
## Contributors
|
||||
|
||||
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
|
||||
|
||||
[](../../graphs/contributors)
|
||||
|
||||
### FAQ / Contact / Troubleshoot
|
||||
|
||||
If you have any issue you can try the following options, do no desist to ask or check our issues database, perhaps someone has asked already what you are looking for.
|
||||
|
||||
- [Blog](https://verdaccio.org/blog/)
|
||||
- [Donations](https://opencollective.com/verdaccio)
|
||||
- [Reporting an issue](https://github.com/verdaccio/verdaccio/blob/master/CONTRIBUTING.md#reporting-a-bug)
|
||||
- [Running discussions](https://github.com/verdaccio/verdaccio/issues?q=is%3Aissue+is%3Aopen+label%3Adiscuss)
|
||||
- [Chat](http://chat.verdaccio.org/)
|
||||
- [Logos](https://verdaccio.org/docs/en/logo)
|
||||
- [Docker Examples](https://github.com/verdaccio/docker-examples)
|
||||
- [FAQ](https://github.com/verdaccio/verdaccio/issues?utf8=%E2%9C%93&q=is%3Aissue%20label%3Aquestion%20)
|
||||
|
||||
### License
|
||||
|
||||
Verdaccio is [MIT licensed](https://github.com/verdaccio/verdaccio/blob/master/LICENSE)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ module.exports = Object.assign({}, config, {
|
|||
global: {
|
||||
// FIXME: increase to 90
|
||||
branches: 62,
|
||||
functions: 86,
|
||||
lines: 76,
|
||||
functions: 84,
|
||||
lines: 74,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@
|
|||
"@verdaccio/loaders": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/local-storage": "workspace:12.0.0-next-7.11",
|
||||
"@verdaccio/logger": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/search": "workspace:7.0.0-next.0",
|
||||
"@verdaccio/proxy": "workspace:7.0.0-next-7.11",
|
||||
"@verdaccio/tarball": "workspace:12.0.0-next-7.11",
|
||||
"@verdaccio/url": "workspace:12.0.0-next-7.11",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
export { Storage } from './storage';
|
||||
export * from './lib/storage-utils';
|
||||
export * from './lib/search-utils';
|
||||
export * from './lib/versions-utils';
|
||||
export * from './lib/star-utils';
|
||||
export * from './type';
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
import buildDebug from 'debug';
|
||||
import _ from 'lodash';
|
||||
import { Transform } from 'stream';
|
||||
|
||||
import { searchUtils } from '@verdaccio/core';
|
||||
|
||||
const debug = buildDebug('verdaccio:storage:search:transform');
|
||||
|
||||
export class TransFormResults extends Transform {
|
||||
public constructor(options) {
|
||||
super(options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform either array of packages or a single package into a stream of packages.
|
||||
* From uplinks the chunks are array but from local packages are objects.
|
||||
* @param {string} chunk
|
||||
* @param {string} encoding
|
||||
* @param {function} done
|
||||
* @returns {void}
|
||||
* @override
|
||||
*/
|
||||
public _transform(chunk, _encoding, callback) {
|
||||
if (_.isArray(chunk)) {
|
||||
// from remotes we should expect chunks as arrays
|
||||
(chunk as searchUtils.SearchItem[])
|
||||
.filter((pkgItem) => {
|
||||
debug(`streaming remote pkg name ${pkgItem?.package?.name}`);
|
||||
return true;
|
||||
})
|
||||
.forEach((pkgItem) => {
|
||||
this.push({ ...pkgItem, verdaccioPkgCached: false, verdaccioPrivate: false });
|
||||
});
|
||||
return callback();
|
||||
} else {
|
||||
// local we expect objects
|
||||
debug(`streaming local pkg name ${chunk?.package?.name}`);
|
||||
this.push(chunk);
|
||||
return callback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
import { orderBy } from 'lodash';
|
||||
|
||||
import { pkgUtils, searchUtils } from '@verdaccio/core';
|
||||
import { Manifest, Version } from '@verdaccio/types';
|
||||
|
||||
export function removeDuplicates(results: searchUtils.SearchPackageItem[]) {
|
||||
const pkgNames: any[] = [];
|
||||
const orderByResults = orderBy(results, ['verdaccioPrivate', 'asc']);
|
||||
return orderByResults.filter((pkg) => {
|
||||
if (pkgNames.includes(pkg?.package?.name)) {
|
||||
return false;
|
||||
}
|
||||
pkgNames.push(pkg?.package?.name);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function mapManifestToSearchPackageBody(
|
||||
pkg: Manifest,
|
||||
searchItem: searchUtils.SearchItem
|
||||
): searchUtils.SearchPackageBody {
|
||||
const latest = pkgUtils.getLatest(pkg);
|
||||
const version: Version = pkg.versions[latest];
|
||||
const result: searchUtils.SearchPackageBody = {
|
||||
name: version.name,
|
||||
scope: '',
|
||||
description: version.description,
|
||||
version: latest,
|
||||
keywords: version.keywords,
|
||||
date: pkg.time[latest],
|
||||
// FIXME: type
|
||||
author: version.author as any,
|
||||
// FIXME: not possible fill this out from a private package
|
||||
publisher: {},
|
||||
// FIXME: type
|
||||
maintainers: version.maintainers as any,
|
||||
links: {
|
||||
npm: '',
|
||||
homepage: version.homepage,
|
||||
repository: version.repository,
|
||||
bugs: version.bugs,
|
||||
},
|
||||
};
|
||||
|
||||
if (typeof searchItem.package.scoped === 'string') {
|
||||
result.scope = searchItem.package.scoped;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import _ from 'lodash';
|
||||
import semver from 'semver';
|
||||
|
||||
import { errorUtils, pkgUtils, validatioUtils } from '@verdaccio/core';
|
||||
import { errorUtils, pkgUtils, searchUtils, validatioUtils } from '@verdaccio/core';
|
||||
import { API_ERROR, DIST_TAGS, HTTP_STATUS, USERS } from '@verdaccio/core';
|
||||
import { AttachMents, Manifest, Version, Versions } from '@verdaccio/types';
|
||||
import { generateRandomHexString, isNil, isObject } from '@verdaccio/utils';
|
||||
|
|
@ -360,3 +360,37 @@ export function hasDeprecatedVersions(pkgInfo: Manifest): boolean {
|
|||
export function isDeprecatedManifest(manifest: Manifest): boolean {
|
||||
return hasDeprecatedVersions(manifest) && Object.keys(manifest._attachments).length === 0;
|
||||
}
|
||||
|
||||
export function mapManifestToSearchPackageBody(
|
||||
pkg: Manifest,
|
||||
searchItem: searchUtils.SearchItem
|
||||
): searchUtils.SearchPackageBody {
|
||||
const latest = pkgUtils.getLatest(pkg);
|
||||
const version: Version = pkg.versions[latest];
|
||||
const result: searchUtils.SearchPackageBody = {
|
||||
name: version.name,
|
||||
scope: '',
|
||||
description: version.description,
|
||||
version: latest,
|
||||
keywords: version.keywords,
|
||||
date: pkg.time[latest],
|
||||
// FIXME: type
|
||||
author: version.author as any,
|
||||
// FIXME: not possible fill this out from a private package
|
||||
publisher: {},
|
||||
// FIXME: type
|
||||
maintainers: version.maintainers as any,
|
||||
links: {
|
||||
npm: '',
|
||||
homepage: version.homepage,
|
||||
repository: version.repository,
|
||||
bugs: version.bugs,
|
||||
},
|
||||
};
|
||||
|
||||
if (typeof searchItem.package.scoped === 'string') {
|
||||
result.scope = searchItem.package.scoped;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
import buildDebug from 'debug';
|
||||
import _ from 'lodash';
|
||||
import semver, { SemVer } from 'semver';
|
||||
|
||||
import { DIST_TAGS } from '@verdaccio/core';
|
||||
import { DIST_TAGS, searchUtils } from '@verdaccio/core';
|
||||
import { Manifest, StringValue, Version, Versions } from '@verdaccio/types';
|
||||
|
||||
const debug = buildDebug('verdaccio:storage:utils');
|
||||
|
||||
/**
|
||||
* Gets version from a package object taking into account semver weirdness.
|
||||
* @return {String} return the semantic version of a package
|
||||
|
|
@ -85,3 +88,49 @@ export function tagVersionNext(manifest: Manifest, version: string, tag: StringV
|
|||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the version is newer than the older version.
|
||||
* @param newVersion
|
||||
* @param oldVersion
|
||||
* @returns
|
||||
*/
|
||||
export function isNewerVersion(newVersion, oldVersion) {
|
||||
const comparisonResult = semver.compare(newVersion, oldVersion);
|
||||
|
||||
return comparisonResult === 1 || comparisonResult === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove duplicates from search results.
|
||||
* @param {Array} objects
|
||||
* @return {Array} filtered array
|
||||
*/
|
||||
export function removeLowerVersions(objects: searchUtils.SearchPackageItem[]) {
|
||||
const versionMap = new Map();
|
||||
|
||||
// Iterate through the array and keep the highest version for each name
|
||||
objects.forEach((item) => {
|
||||
const { name, version } = item.package;
|
||||
const key = name;
|
||||
|
||||
if (versionMap.has(name) === false || isNewerVersion(version, versionMap.get(name))) {
|
||||
debug('keeping %o@%o', name, version);
|
||||
versionMap.set(key, version);
|
||||
}
|
||||
});
|
||||
|
||||
// Filter objects based on the version map
|
||||
return objects.reduce((acc, item) => {
|
||||
const { name, version } = item.package;
|
||||
if (
|
||||
versionMap.has(name) &&
|
||||
versionMap.get(name) === version &&
|
||||
acc.find((i) => i.package.name === name) === undefined
|
||||
) {
|
||||
debug('adding %o@%o', name, version);
|
||||
acc.push(item);
|
||||
}
|
||||
return acc;
|
||||
}, [] as searchUtils.SearchPackageItem[]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import assert from 'assert';
|
|||
import buildDebug from 'debug';
|
||||
import _, { isEmpty, isNil } from 'lodash';
|
||||
import { basename } from 'path';
|
||||
import { PassThrough, Readable, Transform, pipeline as streamPipeline } from 'stream';
|
||||
import { PassThrough, Readable, Transform } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { default as URL } from 'url';
|
||||
|
||||
|
|
@ -23,7 +23,16 @@ import {
|
|||
} from '@verdaccio/core';
|
||||
import { asyncLoadPlugin } from '@verdaccio/loaders';
|
||||
import { logger } from '@verdaccio/logger';
|
||||
import { IProxy, ISyncUplinksOptions, ProxySearchParams, ProxyStorage } from '@verdaccio/proxy';
|
||||
import {
|
||||
IProxy,
|
||||
ISyncUplinksOptions,
|
||||
ProxyInstanceList,
|
||||
ProxySearchParams,
|
||||
ProxyStorage,
|
||||
setupUpLinks,
|
||||
updateVersionsHiddenUpLinkNext,
|
||||
} from '@verdaccio/proxy';
|
||||
import Search from '@verdaccio/search';
|
||||
import {
|
||||
convertDistRemoteToLocalTarballUrls,
|
||||
convertDistVersionToLocalTarballsUrl,
|
||||
|
|
@ -52,12 +61,9 @@ import {
|
|||
UpdateManifestOptions,
|
||||
cleanUpReadme,
|
||||
isDeprecatedManifest,
|
||||
mapManifestToSearchPackageBody,
|
||||
tagVersion,
|
||||
tagVersionNext,
|
||||
} from '.';
|
||||
import { TransFormResults } from './lib/TransFormResults';
|
||||
import { removeDuplicates } from './lib/search-utils';
|
||||
import { isPublishablePackage } from './lib/star-utils';
|
||||
import { isExecutingStarCommand } from './lib/star-utils';
|
||||
import {
|
||||
|
|
@ -66,14 +72,14 @@ import {
|
|||
generatePackageTemplate,
|
||||
generateRevision,
|
||||
getLatestReadme,
|
||||
mapManifestToSearchPackageBody,
|
||||
mergeUplinkTimeIntoLocalNext,
|
||||
mergeVersions,
|
||||
normalizeDistTags,
|
||||
normalizePackage,
|
||||
updateUpLinkMetadata,
|
||||
} from './lib/storage-utils';
|
||||
import { ProxyInstanceList, setupUpLinks, updateVersionsHiddenUpLinkNext } from './lib/uplink-util';
|
||||
import { getVersion } from './lib/versions-utils';
|
||||
import { getVersion, removeLowerVersions } from './lib/versions-utils';
|
||||
import { LocalStorage } from './local-storage';
|
||||
import { IGetPackageOptionsNext, StarManifestBody } from './type';
|
||||
|
||||
|
|
@ -90,10 +96,12 @@ class Storage {
|
|||
public readonly config: Config;
|
||||
public readonly logger: Logger;
|
||||
public readonly uplinks: ProxyInstanceList;
|
||||
private searchService: Search;
|
||||
public constructor(config: Config) {
|
||||
this.config = config;
|
||||
this.uplinks = setupUpLinks(config);
|
||||
this.logger = logger.child({ module: 'storage' });
|
||||
this.uplinks = setupUpLinks(config, this.logger);
|
||||
this.searchService = new Search(config, this.logger);
|
||||
this.filters = null;
|
||||
// @ts-ignore
|
||||
this.localStorage = null;
|
||||
|
|
@ -220,63 +228,19 @@ class Storage {
|
|||
/**
|
||||
* Handle search on packages and proxies.
|
||||
* Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
|
||||
* to a stream, once the proxies request has finished search in local storage for all packages
|
||||
* once the proxies request has finished search in local storage for all packages
|
||||
* (privated and cached).
|
||||
*/
|
||||
public async search(options: ProxySearchParams): Promise<searchUtils.SearchPackageItem[]> {
|
||||
const transformResults = new TransFormResults({ objectMode: true });
|
||||
const streamPassThrough = new PassThrough({ objectMode: true });
|
||||
const upLinkList = this.getProxyList();
|
||||
debug('uplinks found %s', upLinkList.length);
|
||||
const searchUplinksStreams = upLinkList.map((uplinkId: string) => {
|
||||
const uplink = this.uplinks[uplinkId];
|
||||
if (!uplink) {
|
||||
// this line should never happens
|
||||
this.logger.error({ uplinkId }, 'uplink @upLinkId not found');
|
||||
}
|
||||
return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
|
||||
});
|
||||
|
||||
try {
|
||||
debug('searching on %s uplinks...', searchUplinksStreams?.length);
|
||||
// only process those streams end successfully, if all request fails
|
||||
// just include local storage results (if local fails then return 500)
|
||||
await Promise.allSettled([...searchUplinksStreams]);
|
||||
debug('searching all uplinks done');
|
||||
} catch (err: any) {
|
||||
this.logger.error({ err: err?.message }, ' error on uplinks search @{err}');
|
||||
streamPassThrough.emit('error', err);
|
||||
}
|
||||
debug('search local');
|
||||
try {
|
||||
await this.searchCachedPackages(streamPassThrough, options.query as searchUtils.SearchQuery);
|
||||
} catch (err: any) {
|
||||
this.logger.error({ err: err?.message }, ' error on local search @{err}');
|
||||
streamPassThrough.emit('error', err);
|
||||
}
|
||||
const data: searchUtils.SearchPackageItem[] = [];
|
||||
const outPutStream = new PassThrough({ objectMode: true });
|
||||
streamPipeline(streamPassThrough, transformResults, outPutStream, (err: any) => {
|
||||
if (err) {
|
||||
this.logger.error({ err: err?.message }, ' error on search @{err}');
|
||||
throw errorUtils.getInternalError(err ? err.message : 'unknown search error');
|
||||
} else {
|
||||
debug('pipeline succeeded');
|
||||
}
|
||||
});
|
||||
|
||||
outPutStream.on('data', (chunk) => {
|
||||
data.push(chunk);
|
||||
});
|
||||
|
||||
return new Promise((resolve) => {
|
||||
outPutStream.on('finish', async () => {
|
||||
const searchFinalResults: searchUtils.SearchPackageItem[] = removeDuplicates(data);
|
||||
debug('search stream total results: %o', searchFinalResults.length);
|
||||
return resolve(searchFinalResults);
|
||||
});
|
||||
debug('search done');
|
||||
});
|
||||
debug('search on cache packages');
|
||||
const cachePackages = await this.getCachedPackages(options.query);
|
||||
debug('search found on cache packages %o', cachePackages.length);
|
||||
const remotePackages = await this.searchService.search(options);
|
||||
debug('search found on remote packages %o', remotePackages.length);
|
||||
const totalResults = [...cachePackages, ...remotePackages];
|
||||
const uniqueResults = removeLowerVersions(totalResults);
|
||||
debug('unique results %o', uniqueResults.length);
|
||||
return uniqueResults;
|
||||
}
|
||||
|
||||
private async getTarballFromUpstream(name: string, filename: string, { signal }) {
|
||||
|
|
@ -382,7 +346,7 @@ class Storage {
|
|||
// should not be the case
|
||||
const passThroughRemoteStream = new PassThrough();
|
||||
// ensure get the latest data
|
||||
const [updatedManifest] = await this.syncUplinksMetadataNext(name, cachedManifest, {
|
||||
const [updatedManifest] = await this.syncUplinksMetadata(name, cachedManifest, {
|
||||
uplinksLook: true,
|
||||
});
|
||||
const distFile = (updatedManifest as Manifest)._distfiles[filename];
|
||||
|
|
@ -425,7 +389,7 @@ class Storage {
|
|||
* @param param2
|
||||
* @returns
|
||||
*/
|
||||
public async getTarballNext(name: string, filename: string, { signal }): Promise<PassThrough> {
|
||||
public async getTarball(name: string, filename: string, { signal }): Promise<PassThrough> {
|
||||
debug('get tarball for package %o filename %o', name, filename);
|
||||
// TODO: check if isOpen is need it after all.
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
|
|
@ -676,33 +640,11 @@ class Storage {
|
|||
},
|
||||
this.config?.serverSettings?.pluginPrefix
|
||||
);
|
||||
debug('filters available %o', this.filters);
|
||||
debug('filters available %o', this.filters.length);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume the upstream and pipe it to a transformable stream.
|
||||
*/
|
||||
private consumeSearchStream(
|
||||
uplinkId: string,
|
||||
uplink: IProxy,
|
||||
options: ProxySearchParams,
|
||||
searchPassThrough: PassThrough
|
||||
): Promise<any> {
|
||||
return uplink.search({ ...options }).then((bodyStream) => {
|
||||
bodyStream.pipe(searchPassThrough, { end: false });
|
||||
bodyStream.on('error', (err: any): void => {
|
||||
logger.error(
|
||||
{ uplinkId, err: err },
|
||||
'search error for uplink @{uplinkId}: @{err?.message}'
|
||||
);
|
||||
searchPassThrough.end();
|
||||
});
|
||||
return new Promise((resolve) => bodyStream.on('end', resolve));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a wrapper that provide access to the package location.
|
||||
* @param {Object} pkgName package name.
|
||||
|
|
@ -734,27 +676,32 @@ class Storage {
|
|||
return await storage.readTarball(filename, { signal });
|
||||
}
|
||||
|
||||
private async searchCachedPackages(
|
||||
searchStream: PassThrough,
|
||||
query: searchUtils.SearchQuery
|
||||
): Promise<void> {
|
||||
debug('search on each package');
|
||||
this.logger.info(
|
||||
public async getCachedPackages(
|
||||
query?: searchUtils.SearchQuery
|
||||
): Promise<searchUtils.SearchPackageItem[]> {
|
||||
debug('search on each package', query);
|
||||
const results: searchUtils.SearchPackageItem[] = [];
|
||||
if (typeof query === 'undefined' || typeof query?.text === 'undefined') {
|
||||
debug('search query for cached not found');
|
||||
return results;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{ t: query.text, q: query.quality, p: query.popularity, m: query.maintenance, s: query.size },
|
||||
'search by text @{t}| maintenance @{m}| quality @{q}| popularity @{p}'
|
||||
);
|
||||
|
||||
if (typeof this.localStorage.getStoragePlugin().search === 'undefined') {
|
||||
this.logger.info('plugin search not implemented yet');
|
||||
searchStream.end();
|
||||
logger.info('plugin search not implemented yet');
|
||||
} else {
|
||||
debug('search on each package by plugin');
|
||||
debug('search on each package by plugin query');
|
||||
const items = await this.localStorage.getStoragePlugin().search(query);
|
||||
try {
|
||||
for (const searchItem of items) {
|
||||
const manifest = await this.getPackageLocalMetadata(searchItem.package.name);
|
||||
if (_.isEmpty(manifest?.versions) === false) {
|
||||
const searchPackage = mapManifestToSearchPackageBody(manifest, searchItem);
|
||||
debug('search local stream found %o', searchPackage.name);
|
||||
const searchPackageItem: searchUtils.SearchPackageItem = {
|
||||
package: searchPackage,
|
||||
score: searchItem.score,
|
||||
|
|
@ -764,16 +711,18 @@ class Storage {
|
|||
// FUTURE: find a better way to calculate the score
|
||||
searchScore: 1,
|
||||
};
|
||||
searchStream.write(searchPackageItem);
|
||||
results.push(searchPackageItem);
|
||||
} else {
|
||||
debug('local item without versions detected %s', searchItem.package.name);
|
||||
}
|
||||
}
|
||||
debug('search local stream end');
|
||||
searchStream.end();
|
||||
} catch (err) {
|
||||
this.logger.error({ err, query }, 'error on search by plugin @{err.message}');
|
||||
searchStream.emit('error', err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async removePackageByRevision(pkgName: string, revision: string): Promise<void> {
|
||||
|
|
@ -1457,7 +1406,7 @@ class Storage {
|
|||
private async checkPackageRemote(name: string, uplinksLook: boolean): Promise<Manifest | null> {
|
||||
try {
|
||||
// we provide a null manifest, thus the manifest returned will be the remote one
|
||||
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, null, {
|
||||
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, null, {
|
||||
uplinksLook,
|
||||
});
|
||||
|
||||
|
|
@ -1570,7 +1519,7 @@ class Storage {
|
|||
// if we can't get the local metadata, we try to get the remote metadata
|
||||
// if we do to have local metadata, we try to update it with the upstream registry
|
||||
debug('sync uplinks for %o', name);
|
||||
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, data, {
|
||||
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, data, {
|
||||
uplinksLook: options.uplinksLook,
|
||||
retry: options.retry,
|
||||
remoteAddress: options.requestOptions.remoteAddress,
|
||||
|
|
@ -1621,7 +1570,7 @@ class Storage {
|
|||
in that case the request returns empty body and we want ask next on the list if has fresh
|
||||
updates.
|
||||
*/
|
||||
public async syncUplinksMetadataNext(
|
||||
public async syncUplinksMetadata(
|
||||
name: string,
|
||||
localManifest: Manifest | null,
|
||||
options: Partial<ISyncUplinksOptions> = {}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import nock from 'nock';
|
||||
|
||||
import { Config, getDefaultConfig } from '@verdaccio/config';
|
||||
import { searchUtils } from '@verdaccio/core';
|
||||
import { fileUtils, searchUtils } from '@verdaccio/core';
|
||||
import { setup } from '@verdaccio/logger';
|
||||
import { removeDuplicates } from '@verdaccio/search';
|
||||
import { generatePackageMetadata } from '@verdaccio/test-helper';
|
||||
|
||||
import { Storage, removeDuplicates } from '../src';
|
||||
import { Storage } from '../src';
|
||||
|
||||
setup({});
|
||||
|
||||
|
|
@ -26,18 +28,37 @@ describe('search', () => {
|
|||
|
||||
expect(removeDuplicates([item, item])).toEqual([item]);
|
||||
});
|
||||
|
||||
});
|
||||
describe('search manager', () => {
|
||||
test('search items', async () => {
|
||||
// FIXME: fetch is already part of undici
|
||||
const domain = 'https://registry.npmjs.org';
|
||||
const url = '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio';
|
||||
const response = require('./fixtures/search.json');
|
||||
nock(domain).get(url).reply(200, response);
|
||||
const config = new Config(getDefaultConfig());
|
||||
const config = new Config({
|
||||
...getDefaultConfig(),
|
||||
storage: await fileUtils.createTempStorageFolder('fix-1'),
|
||||
});
|
||||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
const abort = new AbortController();
|
||||
const pkgName = 'verdaccio';
|
||||
const requestOptions = {
|
||||
host: 'localhost',
|
||||
protocol: 'http',
|
||||
headers: {},
|
||||
};
|
||||
// create private packages
|
||||
const bodyNewManifest = generatePackageMetadata(pkgName, '5.1.2');
|
||||
await storage.updateManifest(bodyNewManifest, {
|
||||
signal: new AbortController().signal,
|
||||
name: pkgName,
|
||||
uplinksLook: true,
|
||||
revision: '1',
|
||||
requestOptions,
|
||||
});
|
||||
|
||||
// @ts-expect-error
|
||||
const results = await storage.search({ url, query: { text: 'verdaccio' }, abort });
|
||||
expect(results).toHaveLength(4);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -290,7 +290,7 @@ describe('storage', () => {
|
|||
{
|
||||
storage: generateRandomStorage(),
|
||||
},
|
||||
'./fixtures/config/getTarballNext-getupstream.yaml',
|
||||
'./fixtures/config/getTarball-getupstream.yaml',
|
||||
__dirname
|
||||
)
|
||||
);
|
||||
|
|
@ -657,7 +657,7 @@ describe('storage', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('getTarballNext', () => {
|
||||
describe('getTarball', () => {
|
||||
test('should not found a package anywhere', (done) => {
|
||||
const config = new Config(
|
||||
configExample({
|
||||
|
|
@ -669,7 +669,7 @@ describe('storage', () => {
|
|||
storage.init(config).then(() => {
|
||||
const abort = new AbortController();
|
||||
storage
|
||||
.getTarballNext('some-tarball', 'some-tarball-1.0.0.tgz', {
|
||||
.getTarball('some-tarball', 'some-tarball-1.0.0.tgz', {
|
||||
signal: abort.signal,
|
||||
})
|
||||
.then((stream) => {
|
||||
|
|
@ -701,7 +701,7 @@ describe('storage', () => {
|
|||
{
|
||||
storage: generateRandomStorage(),
|
||||
},
|
||||
'./fixtures/config/getTarballNext-getupstream.yaml',
|
||||
'./fixtures/config/getTarball-getupstream.yaml',
|
||||
__dirname
|
||||
)
|
||||
);
|
||||
|
|
@ -709,7 +709,7 @@ describe('storage', () => {
|
|||
storage.init(config).then(() => {
|
||||
const abort = new AbortController();
|
||||
storage
|
||||
.getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
.getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
signal: abort.signal,
|
||||
})
|
||||
.then((stream) => {
|
||||
|
|
@ -745,7 +745,7 @@ describe('storage', () => {
|
|||
{
|
||||
storage: generateRandomStorage(),
|
||||
},
|
||||
'./fixtures/config/getTarballNext-getupstream.yaml',
|
||||
'./fixtures/config/getTarball-getupstream.yaml',
|
||||
__dirname
|
||||
)
|
||||
);
|
||||
|
|
@ -768,7 +768,7 @@ describe('storage', () => {
|
|||
.then(() => {
|
||||
const abort = new AbortController();
|
||||
storage
|
||||
.getTarballNext(pkgName, `${pkgName}-1.0.1.tgz`, {
|
||||
.getTarball(pkgName, `${pkgName}-1.0.1.tgz`, {
|
||||
signal: abort.signal,
|
||||
})
|
||||
.then((stream) => {
|
||||
|
|
@ -809,7 +809,7 @@ describe('storage', () => {
|
|||
{
|
||||
storage: storagePath,
|
||||
},
|
||||
'./fixtures/config/getTarballNext-getupstream.yaml',
|
||||
'./fixtures/config/getTarball-getupstream.yaml',
|
||||
__dirname
|
||||
)
|
||||
);
|
||||
|
|
@ -837,7 +837,7 @@ describe('storage', () => {
|
|||
.then(() => {
|
||||
const abort = new AbortController();
|
||||
storage
|
||||
.getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
.getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
signal: abort.signal,
|
||||
})
|
||||
.then((stream) => {
|
||||
|
|
@ -862,7 +862,7 @@ describe('storage', () => {
|
|||
{
|
||||
storage: generateRandomStorage(),
|
||||
},
|
||||
'./fixtures/config/getTarballNext-getupstream.yaml',
|
||||
'./fixtures/config/getTarball-getupstream.yaml',
|
||||
__dirname
|
||||
)
|
||||
);
|
||||
|
|
@ -885,7 +885,7 @@ describe('storage', () => {
|
|||
.then(() => {
|
||||
const abort = new AbortController();
|
||||
storage
|
||||
.getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
.getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
|
||||
signal: abort.signal,
|
||||
})
|
||||
.then((stream) => {
|
||||
|
|
@ -904,7 +904,7 @@ describe('storage', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('syncUplinksMetadataNext()', () => {
|
||||
describe('syncUplinksMetadata()', () => {
|
||||
describe('error handling', () => {
|
||||
test('should handle double failure on uplinks with timeout', async () => {
|
||||
const fooManifest = generatePackageMetadata('timeout', '8.0.0');
|
||||
|
|
@ -926,7 +926,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
await expect(
|
||||
storage.syncUplinksMetadataNext(fooManifest.name, null, {
|
||||
storage.syncUplinksMetadata(fooManifest.name, null, {
|
||||
retry: { limit: 3 },
|
||||
timeout: {
|
||||
request: 1000,
|
||||
|
|
@ -950,7 +950,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
await expect(
|
||||
storage.syncUplinksMetadataNext(fooManifest.name, null, {
|
||||
storage.syncUplinksMetadata(fooManifest.name, null, {
|
||||
retry: { limit: 0 },
|
||||
})
|
||||
).rejects.toThrow(API_ERROR.NO_PACKAGE);
|
||||
|
|
@ -970,7 +970,7 @@ describe('storage', () => {
|
|||
);
|
||||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
const [manifest] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest, {
|
||||
const [manifest] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
|
||||
retry: 0,
|
||||
});
|
||||
expect(manifest).toBe(fooManifest);
|
||||
|
|
@ -993,7 +993,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
|
||||
const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest);
|
||||
const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
|
||||
expect(response).not.toBeNull();
|
||||
expect((response as Manifest).name).toEqual(fooManifest.name);
|
||||
expect(Object.keys((response as Manifest).versions)).toEqual([
|
||||
|
|
@ -1034,7 +1034,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
|
||||
const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, null);
|
||||
const [response] = await storage.syncUplinksMetadata(fooManifest.name, null);
|
||||
// the latest from the remote manifest
|
||||
expect(response).not.toBeNull();
|
||||
expect((response as Manifest).name).toEqual(fooManifest.name);
|
||||
|
|
@ -1056,7 +1056,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
|
||||
const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest);
|
||||
const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
|
||||
expect(response).not.toBeNull();
|
||||
expect((response as Manifest).name).toEqual(fooManifest.name);
|
||||
expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
|
||||
|
|
@ -1080,7 +1080,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
|
||||
const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest, {
|
||||
const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
|
||||
uplinksLook: false,
|
||||
});
|
||||
|
||||
|
|
@ -1109,7 +1109,7 @@ describe('storage', () => {
|
|||
const storage = new Storage(config);
|
||||
await storage.init(config);
|
||||
|
||||
const [response] = await storage.syncUplinksMetadataNext('foo', null, {
|
||||
const [response] = await storage.syncUplinksMetadata('foo', null, {
|
||||
uplinksLook: true,
|
||||
});
|
||||
|
||||
|
|
@ -1120,7 +1120,7 @@ describe('storage', () => {
|
|||
});
|
||||
|
||||
describe('getLocalDatabase', () => {
|
||||
test('should return 0 local packages', async () => {
|
||||
test('should return no results', async () => {
|
||||
const config = new Config(
|
||||
configExample({
|
||||
...getDefaultConfig(),
|
||||
|
|
@ -1132,7 +1132,7 @@ describe('storage', () => {
|
|||
await expect(storage.getLocalDatabase()).resolves.toHaveLength(0);
|
||||
});
|
||||
|
||||
test('should return 1 local packages', async () => {
|
||||
test('should return single result', async () => {
|
||||
const config = new Config(
|
||||
configExample({
|
||||
...getDefaultConfig(),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import assert from 'assert';
|
||||
|
||||
import { getVersion, sortVersionsAndFilterInvalid, tagVersion } from '../src/index';
|
||||
import {
|
||||
getVersion,
|
||||
removeLowerVersions,
|
||||
sortVersionsAndFilterInvalid,
|
||||
tagVersion,
|
||||
} from '../src/index';
|
||||
|
||||
describe('versions-utils', () => {
|
||||
const dist = (version) => ({
|
||||
|
|
@ -106,4 +111,51 @@ describe('versions-utils', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeLowerVersions', () => {
|
||||
it('should remove lower semantic versions', () => {
|
||||
const inputArray = [
|
||||
{ package: { name: 'object1', version: '1.0.0' } },
|
||||
{ package: { name: 'object1', version: '2.0.0' } }, // Duplicate name 'object1'
|
||||
{ package: { name: 'object2', version: '2.0.0' } }, // Duplicate name 'object2'
|
||||
{ package: { name: 'object2', version: '2.0.0' } },
|
||||
{ package: { name: 'object3', version: '3.0.0' } },
|
||||
{ package: { name: 'object4', version: '1.0.0' } },
|
||||
];
|
||||
|
||||
const expectedOutput = [
|
||||
{ package: { name: 'object1', version: '2.0.0' } },
|
||||
{ package: { name: 'object2', version: '2.0.0' } },
|
||||
{ package: { name: 'object3', version: '3.0.0' } },
|
||||
{ package: { name: 'object4', version: '1.0.0' } },
|
||||
];
|
||||
|
||||
// @ts-expect-error
|
||||
const result = removeLowerVersions(inputArray);
|
||||
expect(result).toEqual(expectedOutput);
|
||||
});
|
||||
|
||||
it('should remove lower semantic versions 2', () => {
|
||||
const inputArray = [
|
||||
{ package: { name: 'object1', version: '1.0.0' } },
|
||||
{ package: { name: 'object1', version: '2.0.0' } }, // Duplicate name 'object1'
|
||||
{ package: { name: 'object2', version: '2.0.3' } }, // Duplicate name 'object2'
|
||||
{ package: { name: 'object2', version: '2.0.0' } },
|
||||
{ package: { name: 'object3', version: '3.0.0' } },
|
||||
{ package: { name: 'object4', version: '1.0.0' } },
|
||||
];
|
||||
|
||||
const expectedOutput = [
|
||||
{ package: { name: 'object1', version: '2.0.0' } },
|
||||
{ package: { name: 'object2', version: '2.0.3' } },
|
||||
{ package: { name: 'object3', version: '3.0.0' } },
|
||||
{ package: { name: 'object4', version: '1.0.0' } },
|
||||
];
|
||||
|
||||
// @ts-expect-error
|
||||
const result = removeLowerVersions(inputArray);
|
||||
|
||||
expect(result).toEqual(expectedOutput);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1451,6 +1451,40 @@ importers:
|
|||
version: 7.6.0
|
||||
|
||||
packages/search:
|
||||
dependencies:
|
||||
'@verdaccio/config':
|
||||
specifier: workspace:7.0.0-next-7.11
|
||||
version: link:../config
|
||||
'@verdaccio/core':
|
||||
specifier: workspace:7.0.0-next-7.11
|
||||
version: link:../core/core
|
||||
'@verdaccio/logger':
|
||||
specifier: workspace:7.0.0-next-7.11
|
||||
version: link:../logger/logger
|
||||
'@verdaccio/proxy':
|
||||
specifier: workspace:7.0.0-next-7.11
|
||||
version: link:../proxy
|
||||
debug:
|
||||
specifier: 4.3.4
|
||||
version: 4.3.4(supports-color@5.5.0)
|
||||
lodash:
|
||||
specifier: 4.17.21
|
||||
version: 4.17.21
|
||||
devDependencies:
|
||||
'@verdaccio/types':
|
||||
specifier: workspace:12.0.0-next.2
|
||||
version: link:../core/types
|
||||
mockdate:
|
||||
specifier: 3.0.5
|
||||
version: 3.0.5
|
||||
nock:
|
||||
specifier: 13.5.1
|
||||
version: 13.5.1
|
||||
node-mocks-http:
|
||||
specifier: 1.14.1
|
||||
version: 1.14.1
|
||||
|
||||
packages/search-indexer:
|
||||
devDependencies:
|
||||
'@orama/orama':
|
||||
specifier: 1.2.4
|
||||
|
|
@ -1635,6 +1669,9 @@ importers:
|
|||
'@verdaccio/proxy':
|
||||
specifier: workspace:7.0.0-next-7.11
|
||||
version: link:../proxy
|
||||
'@verdaccio/search':
|
||||
specifier: workspace:7.0.0-next.0
|
||||
version: link:../search
|
||||
'@verdaccio/tarball':
|
||||
specifier: workspace:12.0.0-next-7.11
|
||||
version: link:../core/tarball
|
||||
|
|
@ -11939,12 +11976,6 @@ packages:
|
|||
undici-types: 5.26.5
|
||||
dev: true
|
||||
|
||||
/@types/node@20.10.6:
|
||||
resolution: {integrity: sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==}
|
||||
dependencies:
|
||||
undici-types: 5.26.5
|
||||
dev: true
|
||||
|
||||
/@types/node@20.11.7:
|
||||
resolution: {integrity: sha512-GPmeN1C3XAyV5uybAf4cMLWT9fDWcmQhZVtMFu7OR32WjrqGG+Wnk2V1d0bmtUyE/Zy1QJ9BxyiTih9z8Oks8A==}
|
||||
dependencies:
|
||||
|
|
@ -22909,7 +22940,7 @@ packages:
|
|||
engines: {node: '>=14'}
|
||||
dependencies:
|
||||
'@types/express': 4.17.21
|
||||
'@types/node': 20.10.6
|
||||
'@types/node': 20.11.7
|
||||
accepts: 1.3.8
|
||||
content-disposition: 0.5.4
|
||||
depd: 1.1.2
|
||||
|
|
|
|||
Loading…
Reference in a new issue