diff --git a/.changeset/pre.json b/.changeset/pre.json index 92e3d4ff6..aab555dc2 100644 --- a/.changeset/pre.json +++ b/.changeset/pre.json @@ -39,7 +39,7 @@ "verdaccio-memory": "11.0.0", "@verdaccio/ui-theme": "6.0.0", "@verdaccio/proxy": "6.0.0", - "@verdaccio/search": "6.0.0", + "@verdaccio/search-indexer": "6.0.0", "@verdaccio/server": "6.0.0", "@verdaccio/server-fastify": "6.0.0", "@verdaccio/signature": "6.0.0", diff --git a/packages/api/src/package.ts b/packages/api/src/package.ts index f08a1b67e..62135ab77 100644 --- a/packages/api/src/package.ts +++ b/packages/api/src/package.ts @@ -67,7 +67,7 @@ export default function (route: Router, auth: Auth, storage: Storage): void { const { package: pkgName, filename } = req.params; const abort = new AbortController(); try { - const stream = (await storage.getTarballNext(pkgName, filename, { + const stream = (await storage.getTarball(pkgName, filename, { signal: abort.signal, // TODO: review why this param // enableRemote: true, diff --git a/packages/api/src/v1/search.ts b/packages/api/src/v1/search.ts index 4271c95c7..4d83306a6 100644 --- a/packages/api/src/v1/search.ts +++ b/packages/api/src/v1/search.ts @@ -50,12 +50,13 @@ export default function (route, auth: Auth, storage: Storage): void { from = parseInt(from, 10) || 0; try { + debug('storage search initiated'); data = await storage.search({ query, url, abort, }); - debug('stream finish'); + debug('storage items tota: %o', data.length); const checkAccessPromises: searchUtils.SearchItemPkg[] = await Promise.all( data.map((pkgItem) => { return checkAccess(pkgItem, auth, req.remote_user); diff --git a/packages/auth/test/auth.spec.ts b/packages/auth/test/auth.spec.ts index e54203eb7..af362ca2c 100644 --- a/packages/auth/test/auth.spec.ts +++ b/packages/auth/test/auth.spec.ts @@ -558,9 +558,10 @@ describe('AuthTest', () => { const getServer = async function (auth) { const app = express(); app.use(express.json({ strict: false, limit: '10mb' })); + + app.use(auth.apiJWTmiddleware()); // @ts-expect-error app.use(errorReportingMiddleware(logger)); - app.use(auth.apiJWTmiddleware()); app.get('/*', (req, res, next) => { if ((req as $RequestExtend).remote_user.error) { next(new Error((req as $RequestExtend).remote_user.error)); @@ -575,6 +576,7 @@ describe('AuthTest', () => { app.use(final); return app; }; + describe('legacy signature', () => { describe('error cases', () => { test('should handle invalid auth token', async () => { diff --git a/packages/plugins/local-storage/src/dir-utils.ts b/packages/plugins/local-storage/src/dir-utils.ts index 4b845c307..52113d20c 100644 --- a/packages/plugins/local-storage/src/dir-utils.ts +++ b/packages/plugins/local-storage/src/dir-utils.ts @@ -64,12 +64,6 @@ export async function searchOnStorage( ): Promise { const matchedStorages = Array.from(storages); const storageFolders = Array.from(storages.keys()); - // const getScopedFolders = async (pkgName) => { - // const scopedPackages = await getFolders(join(storagePath, pkgName), '*'); - // const listScoped = scopedPackages.map((scoped) => ({ - // name: `${pkgName}/${scoped}`, - // })); - // }; debug('search on %o', storagePath); debug('storage folders %o', matchedStorages.length); let results: searchUtils.SearchItemPkg[] = []; diff --git a/packages/plugins/local-storage/src/local-database.ts b/packages/plugins/local-storage/src/local-database.ts index 3dd5d8c57..7242ade53 100644 --- a/packages/plugins/local-storage/src/local-database.ts +++ b/packages/plugins/local-storage/src/local-database.ts @@ -136,13 +136,17 @@ class LocalDatabase extends pluginUtils.Plugin<{}> implements Storage { }); } + /** + * + * @param query + * @returns + */ public async search(query: searchUtils.SearchQuery): Promise { + debug('search query to %o', query.text); const results: searchUtils.SearchItem[] = []; const storagePath = this.getStoragePath(); - const packagesOnStorage = await this.filterByQuery( - await searchOnStorage(storagePath, this.storages), - query - ); + const localResults = await searchOnStorage(storagePath, this.storages); + const packagesOnStorage = await this.filterByQuery(localResults, query); debug('packages found %o', packagesOnStorage.length); for (let storage of packagesOnStorage) { // check if package is listed on the cache private database diff --git a/packages/plugins/local-storage/tests/dir-utils.test.ts b/packages/plugins/local-storage/tests/dir-utils.test.ts index 90b84d5b1..391cbc6fc 100644 --- a/packages/plugins/local-storage/tests/dir-utils.test.ts +++ b/packages/plugins/local-storage/tests/dir-utils.test.ts @@ -45,7 +45,6 @@ describe('searchOnFolders', () => { { name: 'pkg1', }, - { name: 'pkg2', }, diff --git a/packages/proxy/jest.config.js b/packages/proxy/jest.config.js index 373d019b8..9b301f64f 100644 --- a/packages/proxy/jest.config.js +++ b/packages/proxy/jest.config.js @@ -4,9 +4,9 @@ module.exports = Object.assign({}, config, { coverageThreshold: { global: { branches: 79, - functions: 94, - lines: 87, - statements: 87, + functions: 90, + lines: 86, + statements: 86, }, }, }); diff --git a/packages/proxy/src/index.ts b/packages/proxy/src/index.ts index 9bcd86890..8dfbb7d83 100644 --- a/packages/proxy/src/index.ts +++ b/packages/proxy/src/index.ts @@ -1 +1,2 @@ export * from './proxy'; +export * from './uplink-util'; diff --git a/packages/store/src/lib/uplink-util.ts b/packages/proxy/src/uplink-util.ts similarity index 75% rename from packages/store/src/lib/uplink-util.ts rename to packages/proxy/src/uplink-util.ts index 0fdd48b93..0f8afac64 100644 --- a/packages/store/src/lib/uplink-util.ts +++ b/packages/proxy/src/uplink-util.ts @@ -1,15 +1,15 @@ -import { logger } from '@verdaccio/logger'; -import { IProxy, ProxyStorage } from '@verdaccio/proxy'; -import { Config, Manifest } from '@verdaccio/types'; +import { Config, Logger, Manifest } from '@verdaccio/types'; + +import { IProxy, ProxyStorage } from './index'; export interface ProxyInstanceList { [key: string]: IProxy; } /** - * Set up the Up Storage for each link. + * Set up uplinks for each proxy configuration. */ -export function setupUpLinks(config: Config): ProxyInstanceList { +export function setupUpLinks(config: Config, logger: Logger): ProxyInstanceList { const uplinks: ProxyInstanceList = {}; for (const uplinkName in config.uplinks) { @@ -38,5 +38,5 @@ export function updateVersionsHiddenUpLinkNext(manifest: Manifest, upLink: IProx versions[version][Symbol.for('__verdaccio_uplink')] = upLink.upname; } - return { ...manifest, versions: versions }; + return { ...manifest, versions }; } diff --git a/packages/search-indexer/.babelrc b/packages/search-indexer/.babelrc new file mode 100644 index 000000000..633f93f42 --- /dev/null +++ b/packages/search-indexer/.babelrc @@ -0,0 +1,3 @@ +{ + "extends": "../../.babelrc" +} diff --git a/packages/search/README.md b/packages/search-indexer/README.md similarity index 100% rename from packages/search/README.md rename to packages/search-indexer/README.md diff --git a/packages/search-indexer/jest.config.js b/packages/search-indexer/jest.config.js new file mode 100644 index 000000000..373d019b8 --- /dev/null +++ b/packages/search-indexer/jest.config.js @@ -0,0 +1,12 @@ +const config = require('../../jest/config'); + +module.exports = Object.assign({}, config, { + coverageThreshold: { + global: { + branches: 79, + functions: 94, + lines: 87, + statements: 87, + }, + }, +}); diff --git a/packages/search-indexer/package.json b/packages/search-indexer/package.json new file mode 100644 index 000000000..838ca468f --- /dev/null +++ b/packages/search-indexer/package.json @@ -0,0 +1,49 @@ +{ + "name": "@verdaccio/search-indexer", + "version": "7.0.0-next.0", + "description": "verdaccio search indexer", + "main": "./build/dist.js", + "types": "build/index.d.ts", + "author": { + "name": "Juan Picado", + "email": "juanpicado19@gmail.com" + }, + "repository": { + "type": "https", + "url": "https://github.com/verdaccio/verdaccio" + }, + "license": "MIT", + "homepage": "https://verdaccio.org", + "keywords": [ + "private", + "package", + "repository", + "registry", + "enterprise", + "modules", + "proxy", + "server", + "verdaccio" + ], + "engines": { + "node": ">=18" + }, + "scripts": { + "clean": "rimraf ./build", + "test": "vitest run", + "type-check": "tsc --noEmit -p tsconfig.build.json", + "build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json", + "build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps", + "build": "esbuild src/index.ts --bundle --outfile=build/dist.js --platform=node --target=node12 && pnpm run build:types" + }, + "devDependencies": { + "@verdaccio/types": "workspace:12.0.0-next.2", + "@orama/orama": "1.2.4", + "debug": "4.3.4", + "esbuild": "0.14.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/verdaccio" + } +} diff --git a/packages/search-indexer/src/index.ts b/packages/search-indexer/src/index.ts new file mode 100644 index 000000000..901763c04 --- /dev/null +++ b/packages/search-indexer/src/index.ts @@ -0,0 +1 @@ +export { default as SearchMemoryIndexer } from './indexer'; diff --git a/packages/search/src/indexer.ts b/packages/search-indexer/src/indexer.ts similarity index 100% rename from packages/search/src/indexer.ts rename to packages/search-indexer/src/indexer.ts diff --git a/packages/search/test/index.spec.ts b/packages/search-indexer/test/index.spec.ts similarity index 100% rename from packages/search/test/index.spec.ts rename to packages/search-indexer/test/index.spec.ts diff --git a/packages/search-indexer/tsconfig.build.json b/packages/search-indexer/tsconfig.build.json new file mode 100644 index 000000000..fcc453bfe --- /dev/null +++ b/packages/search-indexer/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./build" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/packages/search-indexer/tsconfig.json b/packages/search-indexer/tsconfig.json new file mode 100644 index 000000000..ab7cee9bd --- /dev/null +++ b/packages/search-indexer/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.reference.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./build", + "noImplicitAny": false + }, + "include": ["src/**/*.ts", "types/*.d.ts"], + "references": [ + { + "path": "../config" + }, + { + "path": "../core/core" + }, + { + "path": "../logger/logger" + }, + { + "path": "../utils" + } + ] +} diff --git a/packages/search/types/patch.d.ts b/packages/search-indexer/types/patch.d.ts similarity index 100% rename from packages/search/types/patch.d.ts rename to packages/search-indexer/types/patch.d.ts diff --git a/packages/search/CHANGELOG.md b/packages/search/CHANGELOG.md deleted file mode 100644 index 4361c568b..000000000 --- a/packages/search/CHANGELOG.md +++ /dev/null @@ -1,763 +0,0 @@ -# @verdaccio/proxy - -## 7.0.0-next.2 - -### Major Changes - -- e7ebccb61: update major dependencies, remove old nodejs support - -### Minor Changes - -- daceb6d87: restore legacy support - -## 7.0.0-next.1 - -### Patch Changes - -- 35cc57b79: fix: keyword undefined errors - -## 7.0.0-next.0 - -### Major Changes - -- feat!: bump to v7 - -## 6.0.0 - -### Minor Changes - -- 15e58d988: feat: add search package utilities - -## 6.0.0-6-next.2 - -### Minor Changes - -- 15e58d98: feat: add search package utilities - -## 6.0.0-6-next.46 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.68 -- @verdaccio/config@6.0.0-6-next.68 -- @verdaccio/local-storage@11.0.0-6-next.38 -- @verdaccio/utils@6.0.0-6-next.36 -- @verdaccio/logger@6.0.0-6-next.36 - -## 6.0.0-6-next.45 - -### Patch Changes - -- Updated dependencies [16e38df8] - - @verdaccio/config@6.0.0-6-next.67 - - @verdaccio/core@6.0.0-6-next.67 - - @verdaccio/local-storage@11.0.0-6-next.37 - - @verdaccio/utils@6.0.0-6-next.35 - - @verdaccio/logger@6.0.0-6-next.35 - -## 6.0.0-6-next.44 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.66 -- @verdaccio/logger@6.0.0-6-next.34 -- @verdaccio/local-storage@11.0.0-6-next.36 -- @verdaccio/config@6.0.0-6-next.66 -- @verdaccio/utils@6.0.0-6-next.34 - -## 6.0.0-6-next.43 - -### Patch Changes - -- Updated dependencies [a1da1130] - - @verdaccio/core@6.0.0-6-next.65 - - @verdaccio/config@6.0.0-6-next.65 - - @verdaccio/local-storage@11.0.0-6-next.35 - - @verdaccio/utils@6.0.0-6-next.33 - - @verdaccio/logger@6.0.0-6-next.33 - -## 6.0.0-6-next.42 - -### Patch Changes - -- Updated dependencies [974cd8c1] - - @verdaccio/core@6.0.0-6-next.64 - - @verdaccio/config@6.0.0-6-next.64 - - @verdaccio/local-storage@11.0.0-6-next.34 - - @verdaccio/utils@6.0.0-6-next.32 - - @verdaccio/logger@6.0.0-6-next.32 - -## 6.0.0-6-next.41 - -### Patch Changes - -- Updated dependencies [ddb6a223] -- Updated dependencies [dc571aab] - - @verdaccio/config@6.0.0-6-next.63 - - @verdaccio/core@6.0.0-6-next.63 - - @verdaccio/local-storage@11.0.0-6-next.33 - - @verdaccio/utils@6.0.0-6-next.31 - - @verdaccio/logger@6.0.0-6-next.31 - -## 6.0.0-6-next.40 - -### Patch Changes - -- Updated dependencies [378e907d] - - @verdaccio/core@6.0.0-6-next.62 - - @verdaccio/logger@6.0.0-6-next.30 - - @verdaccio/local-storage@11.0.0-6-next.32 - - @verdaccio/config@6.0.0-6-next.62 - - @verdaccio/utils@6.0.0-6-next.30 - -## 6.0.0-6-next.39 - -### Patch Changes - -- Updated dependencies [d167f92e] - - @verdaccio/config@6.0.0-6-next.61 - - @verdaccio/local-storage@11.0.0-6-next.31 - - @verdaccio/core@6.0.0-6-next.61 - - @verdaccio/utils@6.0.0-6-next.29 - - @verdaccio/logger@6.0.0-6-next.29 - -## 6.0.0-6-next.38 - -### Minor Changes - -- 45c03819: refactor: render html middleware - -### Patch Changes - -- Updated dependencies [45c03819] - - @verdaccio/config@6.0.0-6-next.60 - - @verdaccio/local-storage@11.0.0-6-next.30 - - @verdaccio/core@6.0.0-6-next.60 - - @verdaccio/logger@6.0.0-6-next.28 - - @verdaccio/utils@6.0.0-6-next.28 - -## 6.0.0-6-next.37 - -### Patch Changes - -- Updated dependencies [65f88b82] - - @verdaccio/logger@6.0.0-6-next.27 - - @verdaccio/local-storage@11.0.0-6-next.29 - - @verdaccio/core@6.0.0-6-next.59 - - @verdaccio/config@6.0.0-6-next.59 - - @verdaccio/utils@6.0.0-6-next.27 - -## 6.0.0-6-next.36 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.58 -- @verdaccio/config@6.0.0-6-next.58 -- @verdaccio/local-storage@11.0.0-6-next.28 -- @verdaccio/utils@6.0.0-6-next.26 -- @verdaccio/logger@6.0.0-6-next.26 - -## 6.0.0-6-next.35 - -### Patch Changes - -- @verdaccio/local-storage@11.0.0-6-next.27 -- @verdaccio/core@6.0.0-6-next.57 -- @verdaccio/config@6.0.0-6-next.57 -- @verdaccio/logger@6.0.0-6-next.25 -- @verdaccio/utils@6.0.0-6-next.25 - -## 6.0.0-6-next.34 - -### Patch Changes - -- Updated dependencies [a1986e09] - - @verdaccio/utils@6.0.0-6-next.24 - - @verdaccio/config@6.0.0-6-next.56 - - @verdaccio/local-storage@11.0.0-6-next.26 - - @verdaccio/core@6.0.0-6-next.56 - - @verdaccio/logger@6.0.0-6-next.24 - -## 6.0.0-6-next.33 - -### Patch Changes - -- Updated dependencies [9718e033] - - @verdaccio/config@6.0.0-6-next.55 - - @verdaccio/core@6.0.0-6-next.55 - - @verdaccio/utils@6.0.0-6-next.23 - - @verdaccio/local-storage@11.0.0-6-next.25 - - @verdaccio/logger@6.0.0-6-next.23 - -## 6.0.0-6-next.32 - -### Patch Changes - -- Updated dependencies [ef88da3b] - - @verdaccio/config@6.0.0-6-next.54 - - @verdaccio/core@6.0.0-6-next.54 - - @verdaccio/logger@6.0.0-6-next.22 - - @verdaccio/local-storage@11.0.0-6-next.24 - - @verdaccio/utils@6.0.0-6-next.22 - -## 6.0.0-6-next.31 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.53 -- @verdaccio/logger@6.0.0-6-next.21 -- @verdaccio/local-storage@11.0.0-6-next.23 -- @verdaccio/config@6.0.0-6-next.53 -- @verdaccio/utils@6.0.0-6-next.21 - -## 6.0.0-6-next.30 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.52 -- @verdaccio/config@6.0.0-6-next.52 -- @verdaccio/logger@6.0.0-6-next.20 -- @verdaccio/local-storage@11.0.0-6-next.22 -- @verdaccio/utils@6.0.0-6-next.20 - -## 6.0.0-6-next.29 - -### Patch Changes - -- Updated dependencies [4b29d715] - - @verdaccio/config@6.0.0-6-next.51 - - @verdaccio/core@6.0.0-6-next.51 - - @verdaccio/local-storage@11.0.0-6-next.21 - - @verdaccio/logger@6.0.0-6-next.19 - - @verdaccio/utils@6.0.0-6-next.19 - -## 6.0.0-6-next.28 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.50 -- @verdaccio/config@6.0.0-6-next.50 -- @verdaccio/logger@6.0.0-6-next.18 -- @verdaccio/local-storage@11.0.0-6-next.20 -- @verdaccio/utils@6.0.0-6-next.18 - -## 6.0.0-6-next.27 - -### Patch Changes - -- @verdaccio/local-storage@11.0.0-6-next.19 -- @verdaccio/core@6.0.0-6-next.49 -- @verdaccio/config@6.0.0-6-next.49 -- @verdaccio/logger@6.0.0-6-next.17 -- @verdaccio/utils@6.0.0-6-next.17 - -## 6.0.0-6-next.26 - -### Patch Changes - -- Updated dependencies [43f32687] -- Updated dependencies [9fc2e796] -- Updated dependencies [62c24b63] - - @verdaccio/core@6.0.0-6-next.48 - - @verdaccio/config@6.0.0-6-next.48 - - @verdaccio/local-storage@11.0.0-6-next.18 - - @verdaccio/utils@6.0.0-6-next.16 - - @verdaccio/logger@6.0.0-6-next.16 - -## 6.0.0-6-next.25 - -### Patch Changes - -- @verdaccio/core@6.0.0-6-next.47 -- @verdaccio/config@6.0.0-6-next.47 -- @verdaccio/logger@6.0.0-6-next.15 -- @verdaccio/local-storage@11.0.0-6-next.17 -- @verdaccio/utils@6.0.0-6-next.15 - -## 6.0.0-6-next.24 - -### Patch Changes - -- Updated dependencies [b849128d] - - @verdaccio/core@6.0.0-6-next.8 - - @verdaccio/config@6.0.0-6-next.17 - - @verdaccio/logger@6.0.0-6-next.14 - - @verdaccio/local-storage@11.0.0-6-next.16 - - @verdaccio/utils@6.0.0-6-next.14 - -## 6.0.0-6-next.23 - -### Patch Changes - -- 351aeeaa: fix(deps): @verdaccio/utils should be a prod dep of local-storage -- Updated dependencies [351aeeaa] - - @verdaccio/core@6.0.0-6-next.7 - - @verdaccio/logger@6.0.0-6-next.13 - - @verdaccio/local-storage@11.0.0-6-next.15 - - @verdaccio/config@6.0.0-6-next.16 - - @verdaccio/utils@6.0.0-6-next.13 - -## 6.0.0-6-next.22 - -### Patch Changes - -- Updated dependencies [37274e4c] - - @verdaccio/local-storage@11.0.0-6-next.14 - - @verdaccio/core@6.0.0-6-next.6 - - @verdaccio/logger@6.0.0-6-next.12 - -## 6.0.0-6-next.21 - -### Major Changes - -- 292c0a37: feat!: replace deprecated request dependency by got - - This is a big refactoring of the core, fetching dependencies, improve code, more tests and better stability. This is essential for the next release, will take some time but would allow modularize more the core. - - ## Notes - - - Remove deprecated `request` by other `got`, retry improved, custom Agent ( got does not include it built-in) - - Remove `async` dependency from storage (used by core) it was linked with proxy somehow safe to remove now - - Refactor with promises instead callback wherever is possible - - ~Document the API~ - - Improve testing, integration tests - - Bugfix - - Clean up old validations - - Improve performance - - ## 💥 Breaking changes - - - Plugin API methods were callbacks based are returning promises, this will break current storage plugins, check documentation for upgrade. - - Write Tarball, Read Tarball methods parameters change, a new set of options like `AbortController` signals are being provided to the `addAbortSignal` can be internally used with Streams when a request is aborted. eg: `addAbortSignal(signal, fs.createReadStream(pathName));` - - `@verdaccio/streams` stream abort support is legacy is being deprecated removed - - Remove AWS and Google Cloud packages for future refactoring [#2574](https://github.com/verdaccio/verdaccio/pull/2574). - -### Patch Changes - -- Updated dependencies [292c0a37] -- Updated dependencies [a3a209b5] -- Updated dependencies [00d1d2a1] - - @verdaccio/config@6.0.0-6-next.15 - - @verdaccio/core@6.0.0-6-next.6 - - @verdaccio/logger@6.0.0-6-next.12 - - @verdaccio/local-storage@11.0.0-6-next.13 - - @verdaccio/utils@6.0.0-6-next.12 - -## 6.0.0-6-next.20 - -### Patch Changes - -- Updated dependencies [d43894e8] -- Updated dependencies [d08fe29d] - - @verdaccio/config@6.0.0-6-next.14 - - @verdaccio/local-storage@11.0.0-6-next.12 - - @verdaccio/core@6.0.0-6-next.5 - - @verdaccio/streams@11.0.0-6-next.5 - - @verdaccio/logger@6.0.0-6-next.11 - -## 6.0.0-6-next.19 - -### Major Changes - -- 82cb0f2b: feat!: config.logs throw an error, logging config not longer accept array or logs property - - ### 💥 Breaking change - - This is valid - - ```yaml - log: { type: stdout, format: pretty, level: http } - ``` - - This is invalid - - ```yaml - logs: { type: stdout, format: pretty, level: http } - ``` - - or - - ```yaml - logs: - - [{ type: stdout, format: pretty, level: http }] - ``` - -### Minor Changes - -- 5167bb52: feat: ui search support for remote, local and private packages - - The command `npm search` search globally and return all matches, with this improvement the user interface - is powered with the same capabilities. - - The UI also tag where is the origin the package with a tag, also provide the latest version and description of the package. - -### Patch Changes - -- Updated dependencies [82cb0f2b] -- Updated dependencies [5167bb52] - - @verdaccio/config@6.0.0-6-next.13 - - @verdaccio/core@6.0.0-6-next.5 - - @verdaccio/logger@6.0.0-6-next.11 - - @verdaccio/local-storage@11.0.0-6-next.12 - - @verdaccio/utils@6.0.0-6-next.11 - - @verdaccio/streams@11.0.0-6-next.5 - -## 6.0.0-6-next.18 - -### Patch Changes - -- Updated dependencies [b78f3525] - - @verdaccio/logger@6.0.0-6-next.10 - -## 6.0.0-6-next.17 - -### Patch Changes - -- Updated dependencies [730b5d8c] - - @verdaccio/logger@6.0.0-6-next.9 - -## 6.0.0-6-next.16 - -### Patch Changes - -- Updated dependencies [a828271d] -- Updated dependencies [24b9be02] -- Updated dependencies [e75c0a3b] -- Updated dependencies [b13a3fef] - - @verdaccio/local-storage@11.0.0-6-next.11 - - @verdaccio/utils@6.0.0-6-next.10 - - @verdaccio/core@6.0.0-6-next.4 - - @verdaccio/logger@6.0.0-6-next.8 - - @verdaccio/config@6.0.0-6-next.12 - - @verdaccio/streams@11.0.0-6-next.5 - -## 6.0.0-6-next.15 - -### Patch Changes - -- Updated dependencies [f86c31ed] - - @verdaccio/utils@6.0.0-6-next.9 - - @verdaccio/config@6.0.0-6-next.11 - - @verdaccio/local-storage@11.0.0-6-next.10 - -## 6.0.0-6-next.14 - -### Patch Changes - -- Updated dependencies [6c1eb021] - - @verdaccio/core@6.0.0-6-next.3 - - @verdaccio/logger@6.0.0-6-next.7 - - @verdaccio/config@6.0.0-6-next.10 - - @verdaccio/local-storage@11.0.0-6-next.10 - - @verdaccio/utils@6.0.0-6-next.8 - -## 6.0.0-6-next.13 - -### Minor Changes - -- b702ea36: abort search request support for proxy -- 154b2ecd: refactor: remove @verdaccio/commons-api in favor @verdaccio/core and remove duplications - -### Patch Changes - -- Updated dependencies [794af76c] -- Updated dependencies [154b2ecd] - - @verdaccio/config@6.0.0-6-next.9 - - @verdaccio/core@6.0.0-6-next.2 - - @verdaccio/streams@11.0.0-6-next.5 - - @verdaccio/logger@6.0.0-6-next.6 - - @verdaccio/utils@6.0.0-6-next.7 - - @verdaccio/local-storage@11.0.0-6-next.9 - -## 6.0.0-6-next.12 - -### Patch Changes - -- Updated dependencies [2c594910] - - @verdaccio/logger@6.0.0-6-next.5 - -## 6.0.0-6-next.11 - -### Major Changes - -- 459b6fa7: refactor: search v1 endpoint and local-database - - - refactor search `api v1` endpoint, improve performance - - remove usage of `async` dependency https://github.com/verdaccio/verdaccio/issues/1225 - - refactor method storage class - - create new module `core` to reduce the ammount of modules with utilities - - use `undici` instead `node-fetch` - - use `fastify` instead `express` for functional test - - ### Breaking changes - - - plugin storage API changes - - remove old search endpoint (return 404) - - filter local private packages at plugin level - - The storage api changes for methods `get`, `add`, `remove` as promise base. The `search` methods also changes and recieves a `query` object that contains all query params from the client. - - ```ts - export interface IPluginStorage extends IPlugin { - add(name: string): Promise; - remove(name: string): Promise; - get(): Promise; - init(): Promise; - getSecret(): Promise; - setSecret(secret: string): Promise; - getPackageStorage(packageInfo: string): IPackageStorage; - search(query: searchUtils.SearchQuery): Promise; - saveToken(token: Token): Promise; - deleteToken(user: string, tokenKey: string): Promise; - readTokens(filter: TokenFilter): Promise; - } - ``` - -### Patch Changes - -- Updated dependencies [459b6fa7] - - @verdaccio/config@6.0.0-6-next.8 - - @verdaccio/commons-api@11.0.0-6-next.4 - - @verdaccio/core@6.0.0-6-next.1 - - @verdaccio/local-storage@11.0.0-6-next.8 - - @verdaccio/streams@11.0.0-6-next.4 - - @verdaccio/utils@6.0.0-6-next.6 - - @verdaccio/logger@6.0.0-6-next.4 - -## 6.0.0-6-next.10 - -### Patch Changes - -- Updated dependencies [df0da3d6] - - @verdaccio/local-storage@11.0.0-6-next.7 - -## 6.0.0-6-next.9 - -### Patch Changes - -- Updated dependencies [d2c65da9] - - @verdaccio/utils@6.0.0-6-next.5 - - @verdaccio/config@6.0.0-6-next.7 - -## 6.0.0-6-next.8 - -### Patch Changes - -- Updated dependencies [1b217fd3] - - @verdaccio/config@6.0.0-6-next.6 - - @verdaccio/local-storage@11.0.0-6-next.6 - -## 6.0.0-6-next.7 - -### Patch Changes - -- Updated dependencies [1810ed0d] -- Updated dependencies [648575aa] - - @verdaccio/config@6.0.0-6-next.5 - - @verdaccio/utils@6.0.0-6-next.4 - -## 6.0.0-6-next.6 - -### Patch Changes - -- Updated dependencies [5c5057fc] - - @verdaccio/config@6.0.0-6-next.4 - - @verdaccio/logger@6.0.0-6-next.4 - - @verdaccio/local-storage@11.0.0-6-next.5 - - @verdaccio/streams@11.0.0-alpha.3 - -## 6.0.0-6-next.5 - -### Patch Changes - -- Updated dependencies [cb2281a5] - - @verdaccio/local-storage@11.0.0-6-next.5 - -## 5.0.0-alpha.4 - -### Patch Changes - -- fecbb9be: chore: add release step to private regisry on merge changeset pr -- Updated dependencies [fecbb9be] - - @verdaccio/local-storage@10.0.0-alpha.4 - - @verdaccio/config@5.0.0-alpha.3 - - @verdaccio/commons-api@10.0.0-alpha.3 - - @verdaccio/streams@10.0.0-alpha.3 - - @verdaccio/logger@5.0.0-alpha.3 - - @verdaccio/utils@5.0.0-alpha.3 - -## 5.0.0-alpha.3 - -### Minor Changes - -- 54c58d1e: feat: add server rate limit protection to all request - - To modify custom values, use the server settings property. - - ```markdown - server: - - ## https://www.npmjs.com/package/express-rate-limit#configuration-options - - rateLimit: - windowMs: 1000 - max: 10000 - ``` - - The values are intended to be high, if you want to improve security of your server consider - using different values. - -### Patch Changes - -- Updated dependencies [54c58d1e] - - @verdaccio/config@5.0.0-alpha.2 - - @verdaccio/commons-api@10.0.0-alpha.2 - - @verdaccio/local-storage@10.0.0-alpha.3 - - @verdaccio/streams@10.0.0-alpha.2 - - @verdaccio/logger@5.0.0-alpha.2 - - @verdaccio/utils@5.0.0-alpha.2 - -## 5.0.0-alpha.2 - -### Patch Changes - -- Updated dependencies [2a327c4b] - - @verdaccio/local-storage@10.0.0-alpha.2 - -## 5.0.0-alpha.1 - -### Major Changes - -- d87fa026: feat!: experiments config renamed to flags - - - The `experiments` configuration is renamed to `flags`. The functionality is exactly the same. - - ```js - flags: token: false; - search: false; - ``` - - - The `self_path` property from the config file is being removed in favor of `config_file` full path. - - Refactor `config` module, better types and utilities - -- da1ee9c8: - Replace signature handler for legacy tokens by removing deprecated crypto.createDecipher by createCipheriv - - - Introduce environment variables for legacy tokens - - ### Code Improvements - - - Add debug library for improve developer experience - - ### Breaking change - - - The new signature invalidates all previous tokens generated by Verdaccio 4 or previous versions. - - The secret key must have 32 characters long. - - ### New environment variables - - - `VERDACCIO_LEGACY_ALGORITHM`: Allows to define the specific algorithm for the token signature which by default is `aes-256-ctr` - - `VERDACCIO_LEGACY_ENCRYPTION_KEY`: By default, the token stores in the database, but using this variable allows to get it from memory - -### Minor Changes - -- 26b494cb: feat: add typescript project references settings - - Reading https://ebaytech.berlin/optimizing-multi-package-apps-with-typescript-project-references-d5c57a3b4440 I realized I can use project references to solve the issue to pre-compile modules on develop mode. - - It allows to navigate (IDE) trough the packages without need compile the packages. - - Add two `tsconfig`, one using the previous existing configuration that is able to produce declaration files (`tsconfig.build`) and a new one `tsconfig` which is enables [_projects references_](https://www.typescriptlang.org/docs/handbook/project-references.html). - -### Patch Changes - -- ae52ba35: refactor: migrate request to node-fetch at hooks package -- b57b4338: Enable prerelease mode with **changesets** -- 31af0164: ESLint Warnings Fixed - - Related to issue #1461 - - - max-len: most of the sensible max-len errors are fixed - - no-unused-vars: most of these types of errors are fixed by deleting not needed declarations - - @typescript-eslint/no-unused-vars: same as above - -- Updated dependencies [d87fa026] -- Updated dependencies [da1ee9c8] -- Updated dependencies [26b494cb] -- Updated dependencies [b57b4338] -- Updated dependencies [add778d5] -- Updated dependencies [31af0164] - - @verdaccio/config@5.0.0-alpha.1 - - @verdaccio/commons-api@10.0.0-alpha.1 - - @verdaccio/local-storage@10.0.0-alpha.1 - - @verdaccio/streams@10.0.0-alpha.1 - - @verdaccio/logger@5.0.0-alpha.1 - - @verdaccio/utils@5.0.0-alpha.1 - -## 5.0.0-alpha.1 - -### Major Changes - -- d87fa0268: feat!: experiments config renamed to flags - - - The `experiments` configuration is renamed to `flags`. The functionality is exactly the same. - - ```js - flags: token: false; - search: false; - ``` - - - The `self_path` property from the config file is being removed in favor of `config_file` full path. - - Refactor `config` module, better types and utilities - -- da1ee9c82: - Replace signature handler for legacy tokens by removing deprecated crypto.createDecipher by createCipheriv - - - Introduce environment variables for legacy tokens - - ### Code Improvements - - - Add debug library for improve developer experience - - ### Breaking change - - - The new signature invalidates all previous tokens generated by Verdaccio 4 or previous versions. - - The secret key must have 32 characters long. - - ### New environment variables - - - `VERDACCIO_LEGACY_ALGORITHM`: Allows to define the specific algorithm for the token signature which by default is `aes-256-ctr` - - `VERDACCIO_LEGACY_ENCRYPTION_KEY`: By default, the token stores in the database, but using this variable allows to get it from memory - -### Minor Changes - -- 26b494cbd: feat: add typescript project references settings - - Reading https://ebaytech.berlin/optimizing-multi-package-apps-with-typescript-project-references-d5c57a3b4440 I realized I can use project references to solve the issue to pre-compile modules on develop mode. - - It allows to navigate (IDE) trough the packages without need compile the packages. - - Add two `tsconfig`, one using the previous existing configuration that is able to produce declaration files (`tsconfig.build`) and a new one `tsconfig` which is enables [_projects references_](https://www.typescriptlang.org/docs/handbook/project-references.html). - -### Patch Changes - -- ae52ba352: refactor: migrate request to node-fetch at hooks package -- b57b43388: Enable prerelease mode with **changesets** -- 31af01641: ESLint Warnings Fixed - - Related to issue #1461 - - - max-len: most of the sensible max-len errors are fixed - - no-unused-vars: most of these types of errors are fixed by deleting not needed declarations - - @typescript-eslint/no-unused-vars: same as above - -- Updated dependencies [d87fa0268] -- Updated dependencies [da1ee9c82] -- Updated dependencies [26b494cbd] -- Updated dependencies [b57b43388] -- Updated dependencies [add778d55] -- Updated dependencies [31af01641] - - @verdaccio/config@5.0.0-alpha.1 - - @verdaccio/commons-api@10.0.0-alpha.0 - - @verdaccio/local-storage@10.0.0-alpha.0 - - @verdaccio/streams@10.0.0-alpha.0 - - @verdaccio/logger@5.0.0-alpha.1 - - @verdaccio/utils@5.0.0-alpha.1 diff --git a/packages/search/jest.config.js b/packages/search/jest.config.js index 373d019b8..2d4d07822 100644 --- a/packages/search/jest.config.js +++ b/packages/search/jest.config.js @@ -3,10 +3,9 @@ const config = require('../../jest/config'); module.exports = Object.assign({}, config, { coverageThreshold: { global: { - branches: 79, - functions: 94, - lines: 87, - statements: 87, + branches: 0, + functions: 0, + lines: 0, }, }, }); diff --git a/packages/search/package.json b/packages/search/package.json index f97dc7864..a93a8ba49 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -1,8 +1,8 @@ { "name": "@verdaccio/search", - "version": "7.0.0-next.2", - "description": "verdaccio search utitlity tools", - "main": "./build/dist.js", + "version": "7.0.0-next.0", + "description": "verdaccio search proxy", + "main": "./build/index.js", "types": "build/index.d.ts", "author": { "name": "Juan Picado", @@ -26,21 +26,30 @@ "verdaccio" ], "engines": { - "node": ">=12" + "node": ">=18" }, "scripts": { "clean": "rimraf ./build", - "test": "vitest run", + "test": "jest", "type-check": "tsc --noEmit -p tsconfig.build.json", "build:types": "tsc --emitDeclarationOnly -p tsconfig.build.json", "build:js": "babel src/ --out-dir build/ --copy-files --extensions \".ts,.tsx\" --source-maps", - "build": "esbuild src/index.ts --bundle --outfile=build/dist.js --platform=node --target=node12 && pnpm run build:types" + "watch": "pnpm build:js -- --watch", + "build": "pnpm run build:js && pnpm run build:types" + }, + "dependencies": { + "debug": "4.3.4", + "lodash": "4.17.21", + "@verdaccio/config": "workspace:7.0.0-next-7.11", + "@verdaccio/core": "workspace:7.0.0-next-7.11", + "@verdaccio/logger": "workspace:7.0.0-next-7.11", + "@verdaccio/proxy": "workspace:7.0.0-next-7.11" }, "devDependencies": { "@verdaccio/types": "workspace:12.0.0-next.2", - "@orama/orama": "1.2.4", - "debug": "4.3.4", - "esbuild": "0.14.10" + "mockdate": "3.0.5", + "nock": "13.5.1", + "node-mocks-http": "1.14.1" }, "funding": { "type": "opencollective", diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts index 901763c04..03597bbbb 100644 --- a/packages/search/src/index.ts +++ b/packages/search/src/index.ts @@ -1 +1,2 @@ -export { default as SearchMemoryIndexer } from './indexer'; +export { Search as default } from './search'; +export * from './search-utils'; diff --git a/packages/search/src/search-utils.ts b/packages/search/src/search-utils.ts new file mode 100644 index 000000000..4c098ec05 --- /dev/null +++ b/packages/search/src/search-utils.ts @@ -0,0 +1,15 @@ +import { orderBy } from 'lodash'; + +import { searchUtils } from '@verdaccio/core'; + +export function removeDuplicates(results: searchUtils.SearchPackageItem[]) { + const pkgNames: any[] = []; + const orderByResults = orderBy(results, ['verdaccioPrivate', 'asc']); + return orderByResults.filter((pkg) => { + if (pkgNames.includes(pkg?.package?.name)) { + return false; + } + pkgNames.push(pkg?.package?.name); + return true; + }); +} diff --git a/packages/search/src/search.ts b/packages/search/src/search.ts new file mode 100644 index 000000000..97e267270 --- /dev/null +++ b/packages/search/src/search.ts @@ -0,0 +1,104 @@ +import buildDebug from 'debug'; +import _ from 'lodash'; +import { PassThrough } from 'stream'; + +import { searchUtils } from '@verdaccio/core'; +import { IProxy, ProxyInstanceList, ProxySearchParams, setupUpLinks } from '@verdaccio/proxy'; +import { Config, Logger } from '@verdaccio/types'; + +import { removeDuplicates } from './search-utils'; + +const debug = buildDebug('verdaccio:search'); + +class Search { + public readonly uplinks: ProxyInstanceList; + public readonly logger: Logger; + constructor(config: Config, logger: Logger) { + this.logger = logger.child({ module: 'proxy' }); + this.uplinks = setupUpLinks(config, this.logger); + } + + private getProxyList() { + const uplinksList = Object.keys(this.uplinks); + + return uplinksList; + } + + /** + * Handle search on packages and proxies. + * Iterate all proxies configured and search in all endpoints in v2 and pipe all responses + * to a stream, once the proxies request has finished search in local storage for all packages + * (privated and cached). + */ + public async search(options: ProxySearchParams): Promise { + const results: searchUtils.SearchPackageItem[] = []; + const upLinkList = this.getProxyList(); + // const transformResults = new TransFormResults({ objectMode: true }); + const streamPassThrough = new PassThrough({ objectMode: true }); + debug('uplinks found %s', upLinkList.length); + const searchUplinksStreams = upLinkList.map((uplinkId: string) => { + const uplink = this.uplinks[uplinkId]; + if (!uplink) { + // this line should never happens + this.logger.error({ uplinkId }, 'uplink @upLinkId not found'); + } + return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough); + }); + + try { + debug('searching on %s uplinks...', searchUplinksStreams?.length); + // only process those streams end successfully, if all request fails + // just include local storage results (if local fails then return 500) + await Promise.allSettled([...searchUplinksStreams]); + streamPassThrough.end(); + + for await (const chunk of streamPassThrough) { + if (_.isArray(chunk)) { + (chunk as searchUtils.SearchItem[]) + .filter((pkgItem) => { + debug(`streaming remote pkg name ${pkgItem?.package?.name}`); + return true; + }) + .forEach((pkgItem) => { + // @ts-ignore + return results.push({ + ...pkgItem, + verdaccioPkgCached: false, + verdaccioPrivate: false, + }); + }); + } + } + debug('searching all uplinks done'); + } catch (err: any) { + this.logger.error({ err: err?.message }, ' error on uplinks search @{err}'); + throw err; + } + + return removeDuplicates(results); + } + + /** + * Consume the upstream and pipe it to a transformable stream. + */ + private consumeSearchStream( + uplinkId: string, + uplink: IProxy, + options: ProxySearchParams, + searchPassThrough: PassThrough + ): Promise { + return uplink.search({ ...options }).then((bodyStream) => { + bodyStream.pipe(searchPassThrough, { end: false }); + bodyStream.on('error', (err: any): void => { + this.logger.error( + { uplinkId, err: err }, + 'search error for uplink @{uplinkId}: @{err?.message}' + ); + searchPassThrough.end(); + }); + return new Promise((resolve) => bodyStream.on('end', resolve)); + }); + } +} + +export { Search }; diff --git a/packages/search/test/partials/search.json b/packages/search/test/partials/search.json new file mode 100644 index 000000000..cf697d89a --- /dev/null +++ b/packages/search/test/partials/search.json @@ -0,0 +1,273 @@ +{ + "objects": [ + { + "package": { + "name": "verdaccio", + "scope": "unscoped", + "version": "5.29.2", + "description": "A lightweight private npm proxy registry", + "keywords": [ + "private", + "package", + "repository", + "registry", + "enterprise", + "modules", + "proxy", + "server", + "verdaccio" + ], + "date": "2024-02-21T19:56:45.379Z", + "links": { + "npm": "https://www.npmjs.com/package/verdaccio", + "homepage": "https://verdaccio.org", + "repository": "https://github.com/verdaccio/verdaccio", + "bugs": "https://github.com/verdaccio/verdaccio/issues" + }, + "author": { + "name": "Verdaccio Maintainers", + "email": "test@test.com", + "username": "verdaccio.npm" + }, + "publisher": { "username": "verdaccio.npm", "email": "test@test.com" }, + "maintainers": [ + { "username": "jotadeveloper", "email": "test@test.com" }, + { "username": "ayusharma", "email": "test@test.com" }, + { "username": "trentearl", "email": "test@test.com" }, + { "username": "jmwilkinson", "email": "test@test.com" }, + { "username": "sergiohgz", "email": "test@test.com" }, + { "username": "verdaccio.npm", "email": "test@test.com" } + ] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.28923397536716566, + "detail": { + "quality": 0.39403701233442867, + "popularity": 0.1553034428576298, + "maintenance": 0.3333333333333333 + } + }, + "searchScore": 100000.26 + }, + { + "package": { + "name": "@verdaccio/file-locking", + "scope": "verdaccio", + "version": "10.3.1", + "description": "library that handle file locking", + "keywords": ["verdaccio", "lock", "fs"], + "date": "2023-03-29T18:48:01.509Z", + "links": { + "npm": "https://www.npmjs.com/package/%40verdaccio%2Ffile-locking", + "homepage": "https://verdaccio.org", + "repository": "https://github.com/verdaccio/monorepo", + "bugs": "https://github.com/verdaccio/monorepo/issues" + }, + "author": { + "name": "Juan Picado", + "email": "test@test.com", + "username": "jotadeveloper" + }, + "publisher": { "username": "verdaccio.npm", "email": "test@test.com" }, + "maintainers": [ + { "username": "sergiohgz", "email": "test@test.com" }, + { "username": "verdaccio.npm", "email": "test@test.com" }, + { "username": "jotadeveloper", "email": "test@test.com" }, + { "username": "ayusharma", "email": "test@test.com" } + ] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.4347300973147867, + "detail": { + "quality": 0.8773185907360584, + "popularity": 0.15732960498192844, + "maintenance": 0.33276902385798346 + } + }, + "searchScore": 0.0010177421 + }, + { + "package": { + "name": "verdaccio-service-construct", + "scope": "unscoped", + "version": "0.0.602", + "keywords": ["cdk"], + "date": "2024-03-09T01:14:46.812Z", + "links": { "npm": "https://www.npmjs.com/package/verdaccio-service-construct" }, + "author": { + "name": "Ayush Goyal", + "email": "ayush987goyal@gmail.com", + "username": "ayush987goyal" + }, + "publisher": { "username": "ayush987goyal", "email": "ayush987goyal@gmail.com" }, + "maintainers": [{ "username": "ayush987goyal", "email": "ayush987goyal@gmail.com" }] + }, + "flags": { "insecure": 0, "unstable": true }, + "score": { + "final": 0.22683523291517088, + "detail": { + "quality": 0.32276139168703444, + "popularity": 0.038114710692553955, + "maintenance": 0.3333333333333333 + } + }, + "searchScore": 3.968321e-8 + }, + { + "package": { + "name": "verdaccio-theme-hilio", + "scope": "unscoped", + "version": "1.14.5", + "description": "Verdaccio User Interface", + "keywords": ["verdaccio", "verdaccio-plugin", "verdaccio-theme"], + "date": "2021-01-12T05:48:12.643Z", + "links": { + "npm": "https://www.npmjs.com/package/verdaccio-theme-hilio", + "homepage": "https://verdaccio.org", + "repository": "https://github.com/verdaccio/ui", + "bugs": "https://github.com/verdaccio/ui/issues" + }, + "author": { "name": "Verdaccio Core Team", "email": "test@test.com" }, + "publisher": { "username": "joebnb", "email": "joebnb@qq.com" }, + "maintainers": [{ "username": "joebnb", "email": "joebnb@qq.com" }] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.20823236027862208, + "detail": { + "quality": 0.6388395536236707, + "popularity": 0.022545843955562445, + "maintenance": 0.02482699659164002 + } + }, + "searchScore": 3.8766167e-8 + }, + { + "package": { + "name": "@hamstudy/verdaccio-aws-s3-storage-sse", + "scope": "hamstudy", + "version": "10.3.2", + "description": "AWS S3 storage implementation for Verdaccio - fork that adds support for SSE-C, SSE-S3, and AWS:KMS", + "keywords": ["verdaccio", "plugin", "storage", "aws"], + "date": "2022-10-04T21:39:04.907Z", + "links": { + "npm": "https://www.npmjs.com/package/%40hamstudy%2Fverdaccio-aws-s3-storage-sse", + "homepage": "https://verdaccio.org", + "repository": "https://github.com/taxilian/verdaccio-monorepo", + "bugs": "https://github.com/verdaccio/monorepo/issues" + }, + "author": { + "name": "Richard Bateman", + "email": "taxilian@gmail.com", + "username": "taxilian" + }, + "publisher": { "username": "taxilian", "email": "taxilian@gmail.com" }, + "maintainers": [ + { "username": "rumbcam", "email": "kd7rmx@batemansr.us" }, + { "username": "taxilian", "email": "taxilian@gmail.com" }, + { "username": "bloveridge", "email": "bloveridge@gmail.com" } + ] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.220856041974831, + "detail": { + "quality": 0.6217078604565084, + "popularity": 0.0008629747032998004, + "maintenance": 0.09726183626206726 + } + }, + "searchScore": 3.8267025e-8 + }, + { + "package": { + "name": "testing-verdaccio", + "scope": "unscoped", + "version": "1.1.2", + "description": "this is just for Assignment puropse", + "date": "2023-03-20T07:58:49.325Z", + "links": { "npm": "https://www.npmjs.com/package/testing-verdaccio" }, + "publisher": { "username": "samyak3009", "email": "samyak3009@gmail.com" }, + "maintainers": [{ "username": "samyak3009", "email": "samyak3009@gmail.com" }] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.23106132265338952, + "detail": { + "quality": 0.39172042177373956, + "popularity": 0.0012551184665870981, + "maintenance": 0.3231597275941777 + } + }, + "searchScore": 3.383671e-8 + }, + { + "package": { + "name": "verdaccio-staryauthgroup", + "scope": "unscoped", + "version": "0.0.1", + "description": "A verdaccio plugin to control auth group", + "keywords": ["verdaccio,auth,plugin,verdaccio-]"], + "date": "2023-07-12T01:59:29.928Z", + "links": { "npm": "https://www.npmjs.com/package/verdaccio-staryauthgroup" }, + "author": { "name": "weihuago4", "email": "weihuago4@gmail.com" }, + "publisher": { "username": "liuweihua", "email": "weihualau@126.com" }, + "maintainers": [{ "username": "liuweihua", "email": "weihualau@126.com" }] + }, + "flags": { "insecure": 0, "unstable": true }, + "score": { + "final": 0.2222439493540912, + "detail": { + "quality": 0.4337548917356935, + "popularity": 0.0006994438979026641, + "maintenance": 0.262493361340335 + } + }, + "searchScore": 3.2598397e-8 + }, + { + "package": { + "name": "verdaccio-auth-gitlab", + "scope": "unscoped", + "version": "2.0.0-beta.11", + "description": "Verdaccio authentication plugin by gitlab personal access token or oauth token or ci job token.", + "keywords": [ + "verdaccio", + "authentication", + "auth", + "plugin", + "gitlab", + "personal", + "access", + "oauth", + "ci", + "job", + "token" + ], + "date": "2022-12-07T11:00:06.767Z", + "links": { + "npm": "https://www.npmjs.com/package/verdaccio-auth-gitlab", + "homepage": "https://github.com/pfdgithub/verdaccio-auth-gitlab", + "repository": "https://github.com/pfdgithub/verdaccio-auth-gitlab", + "bugs": "https://github.com/pfdgithub/verdaccio-auth-gitlab/issues" + }, + "publisher": { "username": "pfdnpm", "email": "pfdfree@gmail.com" }, + "maintainers": [{ "username": "pfdnpm", "email": "pfdfree@gmail.com" }] + }, + "flags": { "insecure": 0 }, + "score": { + "final": 0.2194863817767726, + "detail": { + "quality": 0.4542787957560298, + "popularity": 0.0025512045517420002, + "maintenance": 0.2351709184481542 + } + }, + "searchScore": 2.5283079e-8 + } + ], + "total": 351, + "time": "Sat Mar 09 2024 15:20:36 GMT+0000 (Coordinated Universal Time)" +} diff --git a/packages/search/test/search.test.ts b/packages/search/test/search.test.ts new file mode 100644 index 000000000..f8d50c153 --- /dev/null +++ b/packages/search/test/search.test.ts @@ -0,0 +1,102 @@ +import nock from 'nock'; + +import { Config, getDefaultConfig } from '@verdaccio/config'; +import { logger, setup } from '@verdaccio/logger'; + +import { Search } from '../src/search'; + +setup({}); + +const domain = 'https://registry.npmjs.org'; + +describe('search', () => { + const response = require('./partials/search.json'); + test('search', async () => { + nock(domain).get('/-/v1/search').reply(200, response); + const abort = new AbortController(); + const config = new Config(getDefaultConfig()); + const searchInstance = new Search(config, logger); + const results = await searchInstance.search({ + query: { text: 'verdaccio', maintenance: 0, popularity: 0, quality: 0, size: 0 }, + abort, + url: '/-/v1/search', + }); + expect(results).toHaveLength(8); + + expect(results[0]).toEqual({ + package: { + name: 'verdaccio', + scope: 'unscoped', + version: '5.29.2', + description: 'A lightweight private npm proxy registry', + keywords: [ + 'private', + 'package', + 'repository', + 'registry', + 'enterprise', + 'modules', + 'proxy', + 'server', + 'verdaccio', + ], + date: '2024-02-21T19:56:45.379Z', + links: { + npm: 'https://www.npmjs.com/package/verdaccio', + homepage: 'https://verdaccio.org', + repository: 'https://github.com/verdaccio/verdaccio', + bugs: 'https://github.com/verdaccio/verdaccio/issues', + }, + author: { + name: 'Verdaccio Maintainers', + email: 'test@test.com', + username: 'verdaccio.npm', + }, + publisher: { + username: 'verdaccio.npm', + email: 'test@test.com', + }, + maintainers: [ + { + username: 'jotadeveloper', + email: 'test@test.com', + }, + { + username: 'ayusharma', + email: 'test@test.com', + }, + { + username: 'trentearl', + email: 'test@test.com', + }, + { + username: 'jmwilkinson', + email: 'test@test.com', + }, + { + username: 'sergiohgz', + email: 'test@test.com', + }, + { + username: 'verdaccio.npm', + email: 'test@test.com', + }, + ], + }, + flags: { + insecure: 0, + }, + score: { + final: 0.28923397536716566, + detail: { + quality: 0.39403701233442867, + popularity: 0.1553034428576298, + maintenance: 0.3333333333333333, + }, + }, + searchScore: 100000.26, + verdaccioPkgCached: false, + verdaccioPrivate: false, + }); + }); +}); diff --git a/packages/search/tsconfig.json b/packages/search/tsconfig.json index ab7cee9bd..b13cc4428 100644 --- a/packages/search/tsconfig.json +++ b/packages/search/tsconfig.json @@ -10,12 +10,6 @@ { "path": "../config" }, - { - "path": "../core/core" - }, - { - "path": "../logger/logger" - }, { "path": "../utils" } diff --git a/packages/server/fastify/src/endpoints/tarball.ts b/packages/server/fastify/src/endpoints/tarball.ts index 1cfafd932..12c353f9a 100644 --- a/packages/server/fastify/src/endpoints/tarball.ts +++ b/packages/server/fastify/src/endpoints/tarball.ts @@ -16,7 +16,7 @@ async function tarballRoute(fastify: FastifyInstance) { const { package: pkg, filename } = request.params; debug('stream tarball for %s@%s', pkg, filename); const abort = new AbortController(); - const stream = (await fastify.storage.getTarballNext(pkg, filename, { + const stream = (await fastify.storage.getTarball(pkg, filename, { signal: abort.signal, // enableRemote: true, })) as any; @@ -46,7 +46,7 @@ async function tarballRoute(fastify: FastifyInstance) { const { scope, name, filename } = request.params; const scopedPackage = `${scope}/${name}`; debug('stream scope tarball for %s@%s', scopedPackage, filename); - const stream = (await fastify.storage.getTarballNext(scopedPackage, filename, { + const stream = (await fastify.storage.getTarball(scopedPackage, filename, { signal: abort.signal, // enableRemote: true, })) as any; diff --git a/packages/store/README.md b/packages/store/README.md index 9bff24e5e..5b119b34e 100644 --- a/packages/store/README.md +++ b/packages/store/README.md @@ -1,13 +1,32 @@ -# @verdaccio/store +# Verdaccio Search API -[![backers](https://opencollective.com/verdaccio/tiers/backer/badge.svg?label=Backer&color=brightgreen)](https://opencollective.com/verdaccio) -[![stackshare](https://img.shields.io/badge/Follow%20on-StackShare-blue.svg?logo=stackshare&style=flat)](https://stackshare.io/verdaccio) -[![MIT](https://img.shields.io/github/license/mashape/apistatus.svg)](https://github.com/verdaccio/verdaccio/blob/master/LICENSE) -[![Crowdin](https://d322cqt584bo4o.cloudfront.net/verdaccio/localized.svg)](https://crowdin.com/project/verdaccio) -[![TODOs](https://badgen.net/https/api.tickgit.com/badgen/github.com/verdaccio/verdaccio)](https://www.tickgit.com/browse?repo=github.com/verdaccio/verdaccio) +The **Search** class in Verdaccio provides a convenient API for searching packages across both configured proxies and local storage. It enables efficient package discovery and retrieval by aggregating search results from multiple upstream sources. -[![Twitter followers](https://img.shields.io/twitter/follow/verdaccio_npm.svg?style=social&label=Follow)](https://twitter.com/verdaccio_npm) -[![Github](https://img.shields.io/github/stars/verdaccio/verdaccio.svg?style=social&label=Stars)](https://github.com/verdaccio/verdaccio/stargazers) +## Installation + +```bash +npm install @verdaccio/search +``` + +## Usage + +```ts +import { Config } from '@verdaccio/config'; +import { logger } from '@verdaccio/logger'; +import { Search } from '@verdaccio/search'; + +const config = new Config(configYaml); +// Instantiate Search class +const search = new Search(config, logger); + +// Define search parameters +const searchParams = { + // specify search parameters as needed +}; + +// Perform a search and retrieve the results +const searchResults = await search.search(searchParams); +``` ## Donations @@ -15,58 +34,6 @@ Verdaccio is run by **volunteers**; nobody is working full-time on it. If you fi **[Donate](https://opencollective.com/verdaccio)** 💵👍🏻 starting from _\$1/month_ or just one single contribution. -## Report a vulnerability - -If you want to report a security vulnerability, please follow the steps which we have defined for you in our [security policy](https://github.com/verdaccio/verdaccio/security/policy). - -## Open Collective Sponsors - -Support this project by becoming a sponsor. Your logo will show up here with a link to your website. [[Become a sponsor](https://opencollective.com/verdaccio#sponsor)] - -[![sponsor](https://opencollective.com/verdaccio/sponsor/0/avatar.svg)](https://opencollective.com/verdaccio/sponsor/0/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/1/avatar.svg)](https://opencollective.com/verdaccio/sponsor/1/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/2/avatar.svg)](https://opencollective.com/verdaccio/sponsor/2/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/3/avatar.svg)](https://opencollective.com/verdaccio/sponsor/3/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/4/avatar.svg)](https://opencollective.com/verdaccio/sponsor/4/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/5/avatar.svg)](https://opencollective.com/verdaccio/sponsor/5/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/6/avatar.svg)](https://opencollective.com/verdaccio/sponsor/6/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/7/avatar.svg)](https://opencollective.com/verdaccio/sponsor/7/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/8/avatar.svg)](https://opencollective.com/verdaccio/sponsor/8/website) -[![sponsor](https://opencollective.com/verdaccio/sponsor/9/avatar.svg)](https://opencollective.com/verdaccio/sponsor/9/website) - -## Open Collective Backers - -Thank you to all our backers! 🙏 [[Become a backer](https://opencollective.com/verdaccio#backer)] - -[![backers](https://opencollective.com/verdaccio/backers.svg?width=890)](https://opencollective.com/verdaccio#backers) - -## Special Thanks - -Thanks to the following companies to help us to achieve our goals providing free open source licenses. - -[![jetbrain](assets/thanks/jetbrains/logo.png)](https://www.jetbrains.com/) -[![crowdin](assets/thanks/crowdin/logo.png)](https://crowdin.com/) -[![balsamiq](assets/thanks/balsamiq/logo.jpg)](https://balsamiq.com/) - -## Contributors - -This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. - -[![contributors](https://opencollective.com/verdaccio/contributors.svg?width=890&button=true)](../../graphs/contributors) - -### FAQ / Contact / Troubleshoot - -If you have any issue you can try the following options, do no desist to ask or check our issues database, perhaps someone has asked already what you are looking for. - -- [Blog](https://verdaccio.org/blog/) -- [Donations](https://opencollective.com/verdaccio) -- [Reporting an issue](https://github.com/verdaccio/verdaccio/blob/master/CONTRIBUTING.md#reporting-a-bug) -- [Running discussions](https://github.com/verdaccio/verdaccio/issues?q=is%3Aissue+is%3Aopen+label%3Adiscuss) -- [Chat](http://chat.verdaccio.org/) -- [Logos](https://verdaccio.org/docs/en/logo) -- [Docker Examples](https://github.com/verdaccio/docker-examples) -- [FAQ](https://github.com/verdaccio/verdaccio/issues?utf8=%E2%9C%93&q=is%3Aissue%20label%3Aquestion%20) - ### License Verdaccio is [MIT licensed](https://github.com/verdaccio/verdaccio/blob/master/LICENSE) diff --git a/packages/store/jest.config.js b/packages/store/jest.config.js index 6c12a1a5c..e5d1d5799 100644 --- a/packages/store/jest.config.js +++ b/packages/store/jest.config.js @@ -5,8 +5,8 @@ module.exports = Object.assign({}, config, { global: { // FIXME: increase to 90 branches: 62, - functions: 86, - lines: 76, + functions: 84, + lines: 74, }, }, }); diff --git a/packages/store/package.json b/packages/store/package.json index df6377998..c8094bced 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -44,6 +44,7 @@ "@verdaccio/loaders": "workspace:7.0.0-next-7.11", "@verdaccio/local-storage": "workspace:12.0.0-next-7.11", "@verdaccio/logger": "workspace:7.0.0-next-7.11", + "@verdaccio/search": "workspace:7.0.0-next.0", "@verdaccio/proxy": "workspace:7.0.0-next-7.11", "@verdaccio/tarball": "workspace:12.0.0-next-7.11", "@verdaccio/url": "workspace:12.0.0-next-7.11", diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index 9ab231558..c1eef8b2b 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -1,6 +1,5 @@ export { Storage } from './storage'; export * from './lib/storage-utils'; -export * from './lib/search-utils'; export * from './lib/versions-utils'; export * from './lib/star-utils'; export * from './type'; diff --git a/packages/store/src/lib/TransFormResults.ts b/packages/store/src/lib/TransFormResults.ts deleted file mode 100644 index c8dc54df8..000000000 --- a/packages/store/src/lib/TransFormResults.ts +++ /dev/null @@ -1,42 +0,0 @@ -import buildDebug from 'debug'; -import _ from 'lodash'; -import { Transform } from 'stream'; - -import { searchUtils } from '@verdaccio/core'; - -const debug = buildDebug('verdaccio:storage:search:transform'); - -export class TransFormResults extends Transform { - public constructor(options) { - super(options); - } - - /** - * Transform either array of packages or a single package into a stream of packages. - * From uplinks the chunks are array but from local packages are objects. - * @param {string} chunk - * @param {string} encoding - * @param {function} done - * @returns {void} - * @override - */ - public _transform(chunk, _encoding, callback) { - if (_.isArray(chunk)) { - // from remotes we should expect chunks as arrays - (chunk as searchUtils.SearchItem[]) - .filter((pkgItem) => { - debug(`streaming remote pkg name ${pkgItem?.package?.name}`); - return true; - }) - .forEach((pkgItem) => { - this.push({ ...pkgItem, verdaccioPkgCached: false, verdaccioPrivate: false }); - }); - return callback(); - } else { - // local we expect objects - debug(`streaming local pkg name ${chunk?.package?.name}`); - this.push(chunk); - return callback(); - } - } -} diff --git a/packages/store/src/lib/search-utils.ts b/packages/store/src/lib/search-utils.ts deleted file mode 100644 index 757d41f90..000000000 --- a/packages/store/src/lib/search-utils.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { orderBy } from 'lodash'; - -import { pkgUtils, searchUtils } from '@verdaccio/core'; -import { Manifest, Version } from '@verdaccio/types'; - -export function removeDuplicates(results: searchUtils.SearchPackageItem[]) { - const pkgNames: any[] = []; - const orderByResults = orderBy(results, ['verdaccioPrivate', 'asc']); - return orderByResults.filter((pkg) => { - if (pkgNames.includes(pkg?.package?.name)) { - return false; - } - pkgNames.push(pkg?.package?.name); - return true; - }); -} - -export function mapManifestToSearchPackageBody( - pkg: Manifest, - searchItem: searchUtils.SearchItem -): searchUtils.SearchPackageBody { - const latest = pkgUtils.getLatest(pkg); - const version: Version = pkg.versions[latest]; - const result: searchUtils.SearchPackageBody = { - name: version.name, - scope: '', - description: version.description, - version: latest, - keywords: version.keywords, - date: pkg.time[latest], - // FIXME: type - author: version.author as any, - // FIXME: not possible fill this out from a private package - publisher: {}, - // FIXME: type - maintainers: version.maintainers as any, - links: { - npm: '', - homepage: version.homepage, - repository: version.repository, - bugs: version.bugs, - }, - }; - - if (typeof searchItem.package.scoped === 'string') { - result.scope = searchItem.package.scoped; - } - - return result; -} diff --git a/packages/store/src/lib/storage-utils.ts b/packages/store/src/lib/storage-utils.ts index f7c322416..d36387f7f 100644 --- a/packages/store/src/lib/storage-utils.ts +++ b/packages/store/src/lib/storage-utils.ts @@ -1,7 +1,7 @@ import _ from 'lodash'; import semver from 'semver'; -import { errorUtils, pkgUtils, validatioUtils } from '@verdaccio/core'; +import { errorUtils, pkgUtils, searchUtils, validatioUtils } from '@verdaccio/core'; import { API_ERROR, DIST_TAGS, HTTP_STATUS, USERS } from '@verdaccio/core'; import { AttachMents, Manifest, Version, Versions } from '@verdaccio/types'; import { generateRandomHexString, isNil, isObject } from '@verdaccio/utils'; @@ -360,3 +360,37 @@ export function hasDeprecatedVersions(pkgInfo: Manifest): boolean { export function isDeprecatedManifest(manifest: Manifest): boolean { return hasDeprecatedVersions(manifest) && Object.keys(manifest._attachments).length === 0; } + +export function mapManifestToSearchPackageBody( + pkg: Manifest, + searchItem: searchUtils.SearchItem +): searchUtils.SearchPackageBody { + const latest = pkgUtils.getLatest(pkg); + const version: Version = pkg.versions[latest]; + const result: searchUtils.SearchPackageBody = { + name: version.name, + scope: '', + description: version.description, + version: latest, + keywords: version.keywords, + date: pkg.time[latest], + // FIXME: type + author: version.author as any, + // FIXME: not possible fill this out from a private package + publisher: {}, + // FIXME: type + maintainers: version.maintainers as any, + links: { + npm: '', + homepage: version.homepage, + repository: version.repository, + bugs: version.bugs, + }, + }; + + if (typeof searchItem.package.scoped === 'string') { + result.scope = searchItem.package.scoped; + } + + return result; +} diff --git a/packages/store/src/lib/versions-utils.ts b/packages/store/src/lib/versions-utils.ts index eb3c0881a..13c5c3221 100644 --- a/packages/store/src/lib/versions-utils.ts +++ b/packages/store/src/lib/versions-utils.ts @@ -1,9 +1,12 @@ +import buildDebug from 'debug'; import _ from 'lodash'; import semver, { SemVer } from 'semver'; -import { DIST_TAGS } from '@verdaccio/core'; +import { DIST_TAGS, searchUtils } from '@verdaccio/core'; import { Manifest, StringValue, Version, Versions } from '@verdaccio/types'; +const debug = buildDebug('verdaccio:storage:utils'); + /** * Gets version from a package object taking into account semver weirdness. * @return {String} return the semantic version of a package @@ -85,3 +88,49 @@ export function tagVersionNext(manifest: Manifest, version: string, tag: StringV } return data; } + +/** + * Check if the version is newer than the older version. + * @param newVersion + * @param oldVersion + * @returns + */ +export function isNewerVersion(newVersion, oldVersion) { + const comparisonResult = semver.compare(newVersion, oldVersion); + + return comparisonResult === 1 || comparisonResult === 0; +} + +/** + * Remove duplicates from search results. + * @param {Array} objects + * @return {Array} filtered array + */ +export function removeLowerVersions(objects: searchUtils.SearchPackageItem[]) { + const versionMap = new Map(); + + // Iterate through the array and keep the highest version for each name + objects.forEach((item) => { + const { name, version } = item.package; + const key = name; + + if (versionMap.has(name) === false || isNewerVersion(version, versionMap.get(name))) { + debug('keeping %o@%o', name, version); + versionMap.set(key, version); + } + }); + + // Filter objects based on the version map + return objects.reduce((acc, item) => { + const { name, version } = item.package; + if ( + versionMap.has(name) && + versionMap.get(name) === version && + acc.find((i) => i.package.name === name) === undefined + ) { + debug('adding %o@%o', name, version); + acc.push(item); + } + return acc; + }, [] as searchUtils.SearchPackageItem[]); +} diff --git a/packages/store/src/storage.ts b/packages/store/src/storage.ts index ae79de731..3739756b8 100644 --- a/packages/store/src/storage.ts +++ b/packages/store/src/storage.ts @@ -2,7 +2,7 @@ import assert from 'assert'; import buildDebug from 'debug'; import _, { isEmpty, isNil } from 'lodash'; import { basename } from 'path'; -import { PassThrough, Readable, Transform, pipeline as streamPipeline } from 'stream'; +import { PassThrough, Readable, Transform } from 'stream'; import { pipeline } from 'stream/promises'; import { default as URL } from 'url'; @@ -23,7 +23,16 @@ import { } from '@verdaccio/core'; import { asyncLoadPlugin } from '@verdaccio/loaders'; import { logger } from '@verdaccio/logger'; -import { IProxy, ISyncUplinksOptions, ProxySearchParams, ProxyStorage } from '@verdaccio/proxy'; +import { + IProxy, + ISyncUplinksOptions, + ProxyInstanceList, + ProxySearchParams, + ProxyStorage, + setupUpLinks, + updateVersionsHiddenUpLinkNext, +} from '@verdaccio/proxy'; +import Search from '@verdaccio/search'; import { convertDistRemoteToLocalTarballUrls, convertDistVersionToLocalTarballsUrl, @@ -52,12 +61,9 @@ import { UpdateManifestOptions, cleanUpReadme, isDeprecatedManifest, - mapManifestToSearchPackageBody, tagVersion, tagVersionNext, } from '.'; -import { TransFormResults } from './lib/TransFormResults'; -import { removeDuplicates } from './lib/search-utils'; import { isPublishablePackage } from './lib/star-utils'; import { isExecutingStarCommand } from './lib/star-utils'; import { @@ -66,14 +72,14 @@ import { generatePackageTemplate, generateRevision, getLatestReadme, + mapManifestToSearchPackageBody, mergeUplinkTimeIntoLocalNext, mergeVersions, normalizeDistTags, normalizePackage, updateUpLinkMetadata, } from './lib/storage-utils'; -import { ProxyInstanceList, setupUpLinks, updateVersionsHiddenUpLinkNext } from './lib/uplink-util'; -import { getVersion } from './lib/versions-utils'; +import { getVersion, removeLowerVersions } from './lib/versions-utils'; import { LocalStorage } from './local-storage'; import { IGetPackageOptionsNext, StarManifestBody } from './type'; @@ -90,10 +96,12 @@ class Storage { public readonly config: Config; public readonly logger: Logger; public readonly uplinks: ProxyInstanceList; + private searchService: Search; public constructor(config: Config) { this.config = config; - this.uplinks = setupUpLinks(config); this.logger = logger.child({ module: 'storage' }); + this.uplinks = setupUpLinks(config, this.logger); + this.searchService = new Search(config, this.logger); this.filters = null; // @ts-ignore this.localStorage = null; @@ -220,63 +228,19 @@ class Storage { /** * Handle search on packages and proxies. * Iterate all proxies configured and search in all endpoints in v2 and pipe all responses - * to a stream, once the proxies request has finished search in local storage for all packages + * once the proxies request has finished search in local storage for all packages * (privated and cached). */ public async search(options: ProxySearchParams): Promise { - const transformResults = new TransFormResults({ objectMode: true }); - const streamPassThrough = new PassThrough({ objectMode: true }); - const upLinkList = this.getProxyList(); - debug('uplinks found %s', upLinkList.length); - const searchUplinksStreams = upLinkList.map((uplinkId: string) => { - const uplink = this.uplinks[uplinkId]; - if (!uplink) { - // this line should never happens - this.logger.error({ uplinkId }, 'uplink @upLinkId not found'); - } - return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough); - }); - - try { - debug('searching on %s uplinks...', searchUplinksStreams?.length); - // only process those streams end successfully, if all request fails - // just include local storage results (if local fails then return 500) - await Promise.allSettled([...searchUplinksStreams]); - debug('searching all uplinks done'); - } catch (err: any) { - this.logger.error({ err: err?.message }, ' error on uplinks search @{err}'); - streamPassThrough.emit('error', err); - } - debug('search local'); - try { - await this.searchCachedPackages(streamPassThrough, options.query as searchUtils.SearchQuery); - } catch (err: any) { - this.logger.error({ err: err?.message }, ' error on local search @{err}'); - streamPassThrough.emit('error', err); - } - const data: searchUtils.SearchPackageItem[] = []; - const outPutStream = new PassThrough({ objectMode: true }); - streamPipeline(streamPassThrough, transformResults, outPutStream, (err: any) => { - if (err) { - this.logger.error({ err: err?.message }, ' error on search @{err}'); - throw errorUtils.getInternalError(err ? err.message : 'unknown search error'); - } else { - debug('pipeline succeeded'); - } - }); - - outPutStream.on('data', (chunk) => { - data.push(chunk); - }); - - return new Promise((resolve) => { - outPutStream.on('finish', async () => { - const searchFinalResults: searchUtils.SearchPackageItem[] = removeDuplicates(data); - debug('search stream total results: %o', searchFinalResults.length); - return resolve(searchFinalResults); - }); - debug('search done'); - }); + debug('search on cache packages'); + const cachePackages = await this.getCachedPackages(options.query); + debug('search found on cache packages %o', cachePackages.length); + const remotePackages = await this.searchService.search(options); + debug('search found on remote packages %o', remotePackages.length); + const totalResults = [...cachePackages, ...remotePackages]; + const uniqueResults = removeLowerVersions(totalResults); + debug('unique results %o', uniqueResults.length); + return uniqueResults; } private async getTarballFromUpstream(name: string, filename: string, { signal }) { @@ -382,7 +346,7 @@ class Storage { // should not be the case const passThroughRemoteStream = new PassThrough(); // ensure get the latest data - const [updatedManifest] = await this.syncUplinksMetadataNext(name, cachedManifest, { + const [updatedManifest] = await this.syncUplinksMetadata(name, cachedManifest, { uplinksLook: true, }); const distFile = (updatedManifest as Manifest)._distfiles[filename]; @@ -425,7 +389,7 @@ class Storage { * @param param2 * @returns */ - public async getTarballNext(name: string, filename: string, { signal }): Promise { + public async getTarball(name: string, filename: string, { signal }): Promise { debug('get tarball for package %o filename %o', name, filename); // TODO: check if isOpen is need it after all. // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -676,33 +640,11 @@ class Storage { }, this.config?.serverSettings?.pluginPrefix ); - debug('filters available %o', this.filters); + debug('filters available %o', this.filters.length); } return; } - /** - * Consume the upstream and pipe it to a transformable stream. - */ - private consumeSearchStream( - uplinkId: string, - uplink: IProxy, - options: ProxySearchParams, - searchPassThrough: PassThrough - ): Promise { - return uplink.search({ ...options }).then((bodyStream) => { - bodyStream.pipe(searchPassThrough, { end: false }); - bodyStream.on('error', (err: any): void => { - logger.error( - { uplinkId, err: err }, - 'search error for uplink @{uplinkId}: @{err?.message}' - ); - searchPassThrough.end(); - }); - return new Promise((resolve) => bodyStream.on('end', resolve)); - }); - } - /** * Retrieve a wrapper that provide access to the package location. * @param {Object} pkgName package name. @@ -734,27 +676,32 @@ class Storage { return await storage.readTarball(filename, { signal }); } - private async searchCachedPackages( - searchStream: PassThrough, - query: searchUtils.SearchQuery - ): Promise { - debug('search on each package'); - this.logger.info( + public async getCachedPackages( + query?: searchUtils.SearchQuery + ): Promise { + debug('search on each package', query); + const results: searchUtils.SearchPackageItem[] = []; + if (typeof query === 'undefined' || typeof query?.text === 'undefined') { + debug('search query for cached not found'); + return results; + } + + logger.info( { t: query.text, q: query.quality, p: query.popularity, m: query.maintenance, s: query.size }, 'search by text @{t}| maintenance @{m}| quality @{q}| popularity @{p}' ); if (typeof this.localStorage.getStoragePlugin().search === 'undefined') { - this.logger.info('plugin search not implemented yet'); - searchStream.end(); + logger.info('plugin search not implemented yet'); } else { - debug('search on each package by plugin'); + debug('search on each package by plugin query'); const items = await this.localStorage.getStoragePlugin().search(query); try { for (const searchItem of items) { const manifest = await this.getPackageLocalMetadata(searchItem.package.name); if (_.isEmpty(manifest?.versions) === false) { const searchPackage = mapManifestToSearchPackageBody(manifest, searchItem); + debug('search local stream found %o', searchPackage.name); const searchPackageItem: searchUtils.SearchPackageItem = { package: searchPackage, score: searchItem.score, @@ -764,16 +711,18 @@ class Storage { // FUTURE: find a better way to calculate the score searchScore: 1, }; - searchStream.write(searchPackageItem); + results.push(searchPackageItem); + } else { + debug('local item without versions detected %s', searchItem.package.name); } } debug('search local stream end'); - searchStream.end(); } catch (err) { this.logger.error({ err, query }, 'error on search by plugin @{err.message}'); - searchStream.emit('error', err); + throw err; } } + return results; } private async removePackageByRevision(pkgName: string, revision: string): Promise { @@ -1457,7 +1406,7 @@ class Storage { private async checkPackageRemote(name: string, uplinksLook: boolean): Promise { try { // we provide a null manifest, thus the manifest returned will be the remote one - const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, null, { + const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, null, { uplinksLook, }); @@ -1570,7 +1519,7 @@ class Storage { // if we can't get the local metadata, we try to get the remote metadata // if we do to have local metadata, we try to update it with the upstream registry debug('sync uplinks for %o', name); - const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, data, { + const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, data, { uplinksLook: options.uplinksLook, retry: options.retry, remoteAddress: options.requestOptions.remoteAddress, @@ -1621,7 +1570,7 @@ class Storage { in that case the request returns empty body and we want ask next on the list if has fresh updates. */ - public async syncUplinksMetadataNext( + public async syncUplinksMetadata( name: string, localManifest: Manifest | null, options: Partial = {} diff --git a/packages/store/test/fixtures/config/getTarballNext-getupstream.yaml b/packages/store/test/fixtures/config/getTarball-getupstream.yaml similarity index 100% rename from packages/store/test/fixtures/config/getTarballNext-getupstream.yaml rename to packages/store/test/fixtures/config/getTarball-getupstream.yaml diff --git a/packages/store/test/search.spec.ts b/packages/store/test/search.spec.ts index cc1c79321..9ce02f058 100644 --- a/packages/store/test/search.spec.ts +++ b/packages/store/test/search.spec.ts @@ -1,10 +1,12 @@ import nock from 'nock'; import { Config, getDefaultConfig } from '@verdaccio/config'; -import { searchUtils } from '@verdaccio/core'; +import { fileUtils, searchUtils } from '@verdaccio/core'; import { setup } from '@verdaccio/logger'; +import { removeDuplicates } from '@verdaccio/search'; +import { generatePackageMetadata } from '@verdaccio/test-helper'; -import { Storage, removeDuplicates } from '../src'; +import { Storage } from '../src'; setup({}); @@ -26,18 +28,37 @@ describe('search', () => { expect(removeDuplicates([item, item])).toEqual([item]); }); - + }); + describe('search manager', () => { test('search items', async () => { - // FIXME: fetch is already part of undici const domain = 'https://registry.npmjs.org'; const url = '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio'; const response = require('./fixtures/search.json'); nock(domain).get(url).reply(200, response); - const config = new Config(getDefaultConfig()); + const config = new Config({ + ...getDefaultConfig(), + storage: await fileUtils.createTempStorageFolder('fix-1'), + }); const storage = new Storage(config); await storage.init(config); const abort = new AbortController(); + const pkgName = 'verdaccio'; + const requestOptions = { + host: 'localhost', + protocol: 'http', + headers: {}, + }; + // create private packages + const bodyNewManifest = generatePackageMetadata(pkgName, '5.1.2'); + await storage.updateManifest(bodyNewManifest, { + signal: new AbortController().signal, + name: pkgName, + uplinksLook: true, + revision: '1', + requestOptions, + }); + // @ts-expect-error const results = await storage.search({ url, query: { text: 'verdaccio' }, abort }); expect(results).toHaveLength(4); }); diff --git a/packages/store/test/storage.spec.ts b/packages/store/test/storage.spec.ts index 203d7b0bf..1e767ad61 100644 --- a/packages/store/test/storage.spec.ts +++ b/packages/store/test/storage.spec.ts @@ -290,7 +290,7 @@ describe('storage', () => { { storage: generateRandomStorage(), }, - './fixtures/config/getTarballNext-getupstream.yaml', + './fixtures/config/getTarball-getupstream.yaml', __dirname ) ); @@ -657,7 +657,7 @@ describe('storage', () => { }); }); - describe('getTarballNext', () => { + describe('getTarball', () => { test('should not found a package anywhere', (done) => { const config = new Config( configExample({ @@ -669,7 +669,7 @@ describe('storage', () => { storage.init(config).then(() => { const abort = new AbortController(); storage - .getTarballNext('some-tarball', 'some-tarball-1.0.0.tgz', { + .getTarball('some-tarball', 'some-tarball-1.0.0.tgz', { signal: abort.signal, }) .then((stream) => { @@ -701,7 +701,7 @@ describe('storage', () => { { storage: generateRandomStorage(), }, - './fixtures/config/getTarballNext-getupstream.yaml', + './fixtures/config/getTarball-getupstream.yaml', __dirname ) ); @@ -709,7 +709,7 @@ describe('storage', () => { storage.init(config).then(() => { const abort = new AbortController(); storage - .getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, { + .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, { signal: abort.signal, }) .then((stream) => { @@ -745,7 +745,7 @@ describe('storage', () => { { storage: generateRandomStorage(), }, - './fixtures/config/getTarballNext-getupstream.yaml', + './fixtures/config/getTarball-getupstream.yaml', __dirname ) ); @@ -768,7 +768,7 @@ describe('storage', () => { .then(() => { const abort = new AbortController(); storage - .getTarballNext(pkgName, `${pkgName}-1.0.1.tgz`, { + .getTarball(pkgName, `${pkgName}-1.0.1.tgz`, { signal: abort.signal, }) .then((stream) => { @@ -809,7 +809,7 @@ describe('storage', () => { { storage: storagePath, }, - './fixtures/config/getTarballNext-getupstream.yaml', + './fixtures/config/getTarball-getupstream.yaml', __dirname ) ); @@ -837,7 +837,7 @@ describe('storage', () => { .then(() => { const abort = new AbortController(); storage - .getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, { + .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, { signal: abort.signal, }) .then((stream) => { @@ -862,7 +862,7 @@ describe('storage', () => { { storage: generateRandomStorage(), }, - './fixtures/config/getTarballNext-getupstream.yaml', + './fixtures/config/getTarball-getupstream.yaml', __dirname ) ); @@ -885,7 +885,7 @@ describe('storage', () => { .then(() => { const abort = new AbortController(); storage - .getTarballNext(pkgName, `${pkgName}-1.0.0.tgz`, { + .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, { signal: abort.signal, }) .then((stream) => { @@ -904,7 +904,7 @@ describe('storage', () => { }); }); - describe('syncUplinksMetadataNext()', () => { + describe('syncUplinksMetadata()', () => { describe('error handling', () => { test('should handle double failure on uplinks with timeout', async () => { const fooManifest = generatePackageMetadata('timeout', '8.0.0'); @@ -926,7 +926,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); await expect( - storage.syncUplinksMetadataNext(fooManifest.name, null, { + storage.syncUplinksMetadata(fooManifest.name, null, { retry: { limit: 3 }, timeout: { request: 1000, @@ -950,7 +950,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); await expect( - storage.syncUplinksMetadataNext(fooManifest.name, null, { + storage.syncUplinksMetadata(fooManifest.name, null, { retry: { limit: 0 }, }) ).rejects.toThrow(API_ERROR.NO_PACKAGE); @@ -970,7 +970,7 @@ describe('storage', () => { ); const storage = new Storage(config); await storage.init(config); - const [manifest] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest, { + const [manifest] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, { retry: 0, }); expect(manifest).toBe(fooManifest); @@ -993,7 +993,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); - const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest); + const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest); expect(response).not.toBeNull(); expect((response as Manifest).name).toEqual(fooManifest.name); expect(Object.keys((response as Manifest).versions)).toEqual([ @@ -1034,7 +1034,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); - const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, null); + const [response] = await storage.syncUplinksMetadata(fooManifest.name, null); // the latest from the remote manifest expect(response).not.toBeNull(); expect((response as Manifest).name).toEqual(fooManifest.name); @@ -1056,7 +1056,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); - const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest); + const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest); expect(response).not.toBeNull(); expect((response as Manifest).name).toEqual(fooManifest.name); expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0'); @@ -1080,7 +1080,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); - const [response] = await storage.syncUplinksMetadataNext(fooManifest.name, fooManifest, { + const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, { uplinksLook: false, }); @@ -1109,7 +1109,7 @@ describe('storage', () => { const storage = new Storage(config); await storage.init(config); - const [response] = await storage.syncUplinksMetadataNext('foo', null, { + const [response] = await storage.syncUplinksMetadata('foo', null, { uplinksLook: true, }); @@ -1120,7 +1120,7 @@ describe('storage', () => { }); describe('getLocalDatabase', () => { - test('should return 0 local packages', async () => { + test('should return no results', async () => { const config = new Config( configExample({ ...getDefaultConfig(), @@ -1132,7 +1132,7 @@ describe('storage', () => { await expect(storage.getLocalDatabase()).resolves.toHaveLength(0); }); - test('should return 1 local packages', async () => { + test('should return single result', async () => { const config = new Config( configExample({ ...getDefaultConfig(), diff --git a/packages/store/test/versions.spec.ts b/packages/store/test/versions.spec.ts index cf9cbb16a..d2c9ad786 100644 --- a/packages/store/test/versions.spec.ts +++ b/packages/store/test/versions.spec.ts @@ -1,6 +1,11 @@ import assert from 'assert'; -import { getVersion, sortVersionsAndFilterInvalid, tagVersion } from '../src/index'; +import { + getVersion, + removeLowerVersions, + sortVersionsAndFilterInvalid, + tagVersion, +} from '../src/index'; describe('versions-utils', () => { const dist = (version) => ({ @@ -106,4 +111,51 @@ describe('versions-utils', () => { }); }); }); + + describe('removeLowerVersions', () => { + it('should remove lower semantic versions', () => { + const inputArray = [ + { package: { name: 'object1', version: '1.0.0' } }, + { package: { name: 'object1', version: '2.0.0' } }, // Duplicate name 'object1' + { package: { name: 'object2', version: '2.0.0' } }, // Duplicate name 'object2' + { package: { name: 'object2', version: '2.0.0' } }, + { package: { name: 'object3', version: '3.0.0' } }, + { package: { name: 'object4', version: '1.0.0' } }, + ]; + + const expectedOutput = [ + { package: { name: 'object1', version: '2.0.0' } }, + { package: { name: 'object2', version: '2.0.0' } }, + { package: { name: 'object3', version: '3.0.0' } }, + { package: { name: 'object4', version: '1.0.0' } }, + ]; + + // @ts-expect-error + const result = removeLowerVersions(inputArray); + expect(result).toEqual(expectedOutput); + }); + + it('should remove lower semantic versions 2', () => { + const inputArray = [ + { package: { name: 'object1', version: '1.0.0' } }, + { package: { name: 'object1', version: '2.0.0' } }, // Duplicate name 'object1' + { package: { name: 'object2', version: '2.0.3' } }, // Duplicate name 'object2' + { package: { name: 'object2', version: '2.0.0' } }, + { package: { name: 'object3', version: '3.0.0' } }, + { package: { name: 'object4', version: '1.0.0' } }, + ]; + + const expectedOutput = [ + { package: { name: 'object1', version: '2.0.0' } }, + { package: { name: 'object2', version: '2.0.3' } }, + { package: { name: 'object3', version: '3.0.0' } }, + { package: { name: 'object4', version: '1.0.0' } }, + ]; + + // @ts-expect-error + const result = removeLowerVersions(inputArray); + + expect(result).toEqual(expectedOutput); + }); + }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32ae6e32f..e0f3bab26 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1451,6 +1451,40 @@ importers: version: 7.6.0 packages/search: + dependencies: + '@verdaccio/config': + specifier: workspace:7.0.0-next-7.11 + version: link:../config + '@verdaccio/core': + specifier: workspace:7.0.0-next-7.11 + version: link:../core/core + '@verdaccio/logger': + specifier: workspace:7.0.0-next-7.11 + version: link:../logger/logger + '@verdaccio/proxy': + specifier: workspace:7.0.0-next-7.11 + version: link:../proxy + debug: + specifier: 4.3.4 + version: 4.3.4(supports-color@5.5.0) + lodash: + specifier: 4.17.21 + version: 4.17.21 + devDependencies: + '@verdaccio/types': + specifier: workspace:12.0.0-next.2 + version: link:../core/types + mockdate: + specifier: 3.0.5 + version: 3.0.5 + nock: + specifier: 13.5.1 + version: 13.5.1 + node-mocks-http: + specifier: 1.14.1 + version: 1.14.1 + + packages/search-indexer: devDependencies: '@orama/orama': specifier: 1.2.4 @@ -1635,6 +1669,9 @@ importers: '@verdaccio/proxy': specifier: workspace:7.0.0-next-7.11 version: link:../proxy + '@verdaccio/search': + specifier: workspace:7.0.0-next.0 + version: link:../search '@verdaccio/tarball': specifier: workspace:12.0.0-next-7.11 version: link:../core/tarball @@ -11939,12 +11976,6 @@ packages: undici-types: 5.26.5 dev: true - /@types/node@20.10.6: - resolution: {integrity: sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==} - dependencies: - undici-types: 5.26.5 - dev: true - /@types/node@20.11.7: resolution: {integrity: sha512-GPmeN1C3XAyV5uybAf4cMLWT9fDWcmQhZVtMFu7OR32WjrqGG+Wnk2V1d0bmtUyE/Zy1QJ9BxyiTih9z8Oks8A==} dependencies: @@ -22909,7 +22940,7 @@ packages: engines: {node: '>=14'} dependencies: '@types/express': 4.17.21 - '@types/node': 20.10.6 + '@types/node': 20.11.7 accepts: 1.3.8 content-disposition: 0.5.4 depd: 1.1.2