feat: replace internal web search indexer (#3818)

* feat: replace internal search indexer

* chore: add import

* chore: fix tests

* add search dependency

* remove lunr-mutable-indexes

.pnp.cjs

@@ -74,6 +74,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
             ["@verdaccio/local-storage", "npm:10.3.3"],\
             ["@verdaccio/logger-7", "npm:6.0.0-6-next.13"],\
             ["@verdaccio/middleware", "npm:6.0.0-6-next.47"],\
+            ["@verdaccio/search", "npm:6.0.0-6-next.2"],\
             ["@verdaccio/signature", "npm:6.0.0-6-next.2"],\
             ["@verdaccio/streams", "npm:10.2.1"],\
             ["@verdaccio/tarball", "npm:11.0.0-6-next.37"],\
@@ -119,7 +120,6 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
             ["lockfile-lint", "npm:4.10.1"],\
             ["lodash", "npm:4.17.21"],\
             ["lru-cache", "npm:7.18.3"],\
-            ["lunr-mutable-indexes", "npm:2.3.2"],\
             ["mime", "npm:3.0.0"],\
             ["mkdirp", "npm:1.0.4"],\
             ["mv", "npm:2.1.1"],\
@@ -5328,6 +5328,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
           "linkType": "HARD"\
         }]\
       ]],\
+      ["@verdaccio/search", [\
+        ["npm:6.0.0-6-next.2", {\
+          "packageLocation": "./.yarn/cache/@verdaccio-search-npm-6.0.0-6-next.2-3678897c37-33f85e5e15.zip/node_modules/@verdaccio/search/",\
+          "packageDependencies": [\
+            ["@verdaccio/search", "npm:6.0.0-6-next.2"]\
+          ],\
+          "linkType": "HARD"\
+        }]\
+      ]],\
       ["@verdaccio/signature", [\
         ["npm:6.0.0-6-next.2", {\
           "packageLocation": "./.yarn/cache/@verdaccio-signature-npm-6.0.0-6-next.2-84876e53f2-6e5331ee23.zip/node_modules/@verdaccio/signature/",\
@@ -10835,25 +10844,6 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
           "linkType": "HARD"\
         }]\
       ]],\
-      ["lunr", [\
-        ["npm:2.3.9", {\
-          "packageLocation": "./.yarn/cache/lunr-npm-2.3.9-fa3aa9c2d6-176719e24f.zip/node_modules/lunr/",\
-          "packageDependencies": [\
-            ["lunr", "npm:2.3.9"]\
-          ],\
-          "linkType": "HARD"\
-        }]\
-      ]],\
-      ["lunr-mutable-indexes", [\
-        ["npm:2.3.2", {\
-          "packageLocation": "./.yarn/cache/lunr-mutable-indexes-npm-2.3.2-11e17e9706-792ec9a7f0.zip/node_modules/lunr-mutable-indexes/",\
-          "packageDependencies": [\
-            ["lunr-mutable-indexes", "npm:2.3.2"],\
-            ["lunr", "npm:2.3.9"]\
-          ],\
-          "linkType": "HARD"\
-        }]\
-      ]],\
       ["make-dir", [\
         ["npm:2.1.0", {\
           "packageLocation": "./.yarn/cache/make-dir-npm-2.1.0-1ddaf205e7-043548886b.zip/node_modules/make-dir/",\
@@ -14053,6 +14043,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
             ["@verdaccio/local-storage", "npm:10.3.3"],\
             ["@verdaccio/logger-7", "npm:6.0.0-6-next.13"],\
             ["@verdaccio/middleware", "npm:6.0.0-6-next.47"],\
+            ["@verdaccio/search", "npm:6.0.0-6-next.2"],\
             ["@verdaccio/signature", "npm:6.0.0-6-next.2"],\
             ["@verdaccio/streams", "npm:10.2.1"],\
             ["@verdaccio/tarball", "npm:11.0.0-6-next.37"],\
@@ -14098,7 +14089,6 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
             ["lockfile-lint", "npm:4.10.1"],\
             ["lodash", "npm:4.17.21"],\
             ["lru-cache", "npm:7.18.3"],\
-            ["lunr-mutable-indexes", "npm:2.3.2"],\
             ["mime", "npm:3.0.0"],\
             ["mkdirp", "npm:1.0.4"],\
             ["mv", "npm:2.1.1"],\

package.json

@@ -24,6 +24,7 @@
     "@verdaccio/local-storage": "10.3.3",
     "@verdaccio/logger-7": "6.0.0-6-next.13",
     "@verdaccio/middleware": "6.0.0-6-next.47",
+    "@verdaccio/search": "6.0.0-6-next.2",
     "@verdaccio/signature": "6.0.0-6-next.2",
     "@verdaccio/streams": "10.2.1",
     "@verdaccio/tarball": "11.0.0-6-next.37",
@@ -48,7 +49,6 @@
     "kleur": "4.1.5",
     "lodash": "4.17.21",
     "lru-cache": "7.18.3",
-    "lunr-mutable-indexes": "2.3.2",
     "mime": "3.0.0",
     "mkdirp": "1.0.4",
     "mv": "2.1.1",
@@ -62,6 +62,7 @@
   "devDependencies": {
     "@babel/cli": "7.21.5",
     "@babel/core": "7.21.8",
+    "@babel/eslint-parser": "7.21.8",
     "@babel/node": "7.20.7",
     "@babel/plugin-proposal-class-properties": "7.18.6",
     "@babel/plugin-proposal-decorators": "7.21.0",
@@ -101,7 +102,6 @@
     "@typescript-eslint/parser": "5.59.5",
     "@verdaccio-scope/verdaccio-auth-foo": "0.0.2",
     "@verdaccio/types": "11.0.0-6-next.25",
-    "@babel/eslint-parser": "7.21.8",
     "babel-jest": "29.5.0",
     "babel-plugin-dynamic-import-node": "2.3.3",
     "cross-env": "7.0.3",

src/api/index.ts

@@ -40,7 +40,7 @@ export function loadTheme(config) {
   }
 }
 
-const defineAPI = function (config: IConfig, storage: Storage): any {
+const defineAPI = async function (config: IConfig, storage: Storage): Promise<any> {
   const auth = new Auth(config);
   const app: Application = express();
 
@@ -105,7 +105,8 @@ const defineAPI = function (config: IConfig, storage: Storage): any {
       res.locals.app_version = version ?? '';
       next();
     });
-    app.use('/', webMiddleware(config, auth, storage));
+    const midl = await webMiddleware(config, auth, storage);
+    app.use('/', midl);
   } else {
     app.get('/', function (req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer) {
       next(ErrorCode.getNotFound(API_ERROR.WEB_DISABLED));
@@ -140,5 +141,5 @@ export default (async function (configHash: any): Promise<any> {
   const storage = new Storage(config);
   // waits until init calls have been initialized
   await storage.init(config, filters);
-  return defineAPI(config, storage);
+  return await defineAPI(config, storage);
 });

src/api/web/api/search.ts

@@ -1,10 +1,10 @@
 import { Router } from 'express';
 
-import { Package } from '@verdaccio/types';
+import { SearchMemoryIndexer } from '@verdaccio/search';
+import { Manifest } from '@verdaccio/types';
 
 import Auth from '../../../lib/auth';
 import { DIST_TAGS } from '../../../lib/constants';
-import Search from '../../../lib/search';
 import Storage from '../../../lib/storage';
 import { $NextFunctionVer, $RequestExtend, $ResponseExtend } from '../../../types';
 
@@ -13,16 +13,21 @@ function addSearchWebApi(storage: Storage, auth: Auth): Router {
   // Search package
   route.get(
     '/search/:anything',
-    function (req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer): void {
-      const results: any = Search.query(req.params.anything);
-      // FUTURE: figure out here the correct type
+    async function (
+      req: $RequestExtend,
+      _res: $ResponseExtend,
+      next: $NextFunctionVer
+    ): Promise<void> {
+      const term = req.params.anything;
+      const indexer = (await SearchMemoryIndexer.query(term)) as any;
       const packages: any[] = [];
+      const results = indexer.hits;
 
       const getPackageInfo = function (i): void {
         storage.getPackage({
-          name: results[i].ref,
+          name: results[i].id,
           uplinksLook: false,
-          callback: (err, entry: Package): void => {
+          callback: (err, entry: Manifest): void => {
             if (!err && entry) {
               auth.allow_access(
                 { packageName: entry.name },

src/api/web/index.ts

@@ -3,15 +3,16 @@ import express from 'express';
 import _ from 'lodash';
 
 import { webMiddleware } from '@verdaccio/middleware';
+import { SearchMemoryIndexer } from '@verdaccio/search';
 
 import loadPlugin from '../../lib/plugin-loader';
-import Search from '../../lib/search';
 import webApi from './api';
 
 const debug = buildDebug('verdaccio:web');
 
 export function loadTheme(config) {
   if (_.isNil(config.theme) === false) {
+    debug('loading custom ui theme');
     return _.head(
       loadPlugin(
         config,
@@ -26,9 +27,10 @@ export function loadTheme(config) {
   }
 }
 
-export default (config, auth, storage) => {
+export default async (config, auth, storage) => {
   const pluginOptions = loadTheme(config) || require('@verdaccio/ui-theme')();
-  Search.configureStorage(storage);
+  SearchMemoryIndexer.configureStorage(storage);
+  await SearchMemoryIndexer.init();
   // eslint-disable-next-line new-cap
   const router = express.Router();
   // load application

src/lib/search.ts

@@ -1,117 +0,0 @@
-// eslint-disable no-invalid-this
-import lunrMutable from 'lunr-mutable-indexes';
-
-import { Version } from '@verdaccio/types';
-
-import { IWebSearch } from '../types';
-import LocalStorage from './local-storage';
-import Storage from './storage';
-
-/**
- * Handle the search Indexer.
- */
-class Search implements IWebSearch {
-  public index: lunrMutable.index;
-  // @ts-ignore
-  public storage: Storage;
-
-  /**
-   * Constructor.
-   */
-  public constructor() {
-    this.index = lunrMutable(function (): void {
-      // FIXME: there is no types for this library
-      /* eslint no-invalid-this:off */
-      // @ts-ignore
-      this.field('name', { boost: 10 });
-      // @ts-ignore
-      this.field('description', { boost: 4 });
-      // @ts-ignore
-      this.field('author', { boost: 6 });
-      // @ts-ignore
-      this.field('keywords', { boost: 7 });
-      // @ts-ignore
-      this.field('version');
-      // @ts-ignore
-      this.field('readme');
-    });
-  }
-
-  /**
-   * Performs a query to the indexer.
-   * If the keyword is a * it returns all local elements
-   * otherwise performs a search
-   * @param {*} q the keyword
-   * @return {Array} list of results.
-   */
-  public query(query: string): any[] {
-    const localStorage = this.storage.localStorage as LocalStorage;
-
-    const hasScope = query.startsWith('@');
-    // FIXME: lunr-mutable-indexes ignored '@' during indexing
-    if (hasScope) {
-      query = query.replace('@', '');
-    }
-
-    const results =
-      query === '*'
-        ? localStorage.storagePlugin.get((items): any => {
-            items.map(function (pkg): any {
-              return { ref: pkg, score: 1 };
-            });
-          })
-        : this.index.search(`*${query}*`);
-
-    return hasScope ? results.filter(({ ref }) => ref.startsWith('@')) : results;
-  }
-
-  /**
-   * Add a new element to index
-   * @param {*} pkg the package
-   */
-  public add(pkg: Version): void {
-    this.index.add({
-      id: pkg.name,
-      name: pkg.name,
-      description: pkg.description,
-      version: `v${pkg.version}`,
-      keywords: pkg.keywords,
-      author: pkg._npmUser ? pkg._npmUser.name : '???',
-    });
-  }
-
-  /**
-   * Remove an element from the index.
-   * @param {*} name the id element
-   */
-  public remove(name: string): void {
-    this.index.remove({ id: name });
-  }
-
-  /**
-   * Force a re-index.
-   */
-  public reindex(): void {
-    this.storage.getLocalDatabase((error, packages): void => {
-      if (error) {
-        // that function shouldn't produce any
-        throw error;
-      }
-      let i = packages.length;
-      while (i--) {
-        this.add(packages[i]);
-      }
-    });
-  }
-
-  /**
-   * Set up the {Storage}
-   * @param {*} storage An storage reference.
-   */
-  public configureStorage(storage: Storage): void {
-    this.storage = storage;
-    this.reindex();
-  }
-}
-
-export default new Search();

src/lib/storage-utils.ts

@@ -1,21 +1,16 @@
 import _ from 'lodash';
 
 import { pkgUtils } from '@verdaccio/core';
-import {
-  AbbreviatedManifest,
-  AbbreviatedVersions,
-  Manifest,
-  Package,
-  Version,
-} from '@verdaccio/types';
+import { SearchMemoryIndexer } from '@verdaccio/search';
+import { AbbreviatedManifest, AbbreviatedVersions, Manifest, Version } from '@verdaccio/types';
 import { generateRandomHexString } from '@verdaccio/utils';
 
 import { API_ERROR, DIST_TAGS, HTTP_STATUS, STORAGE, USERS } from './constants';
 import LocalStorage from './local-storage';
-import Search from './search';
-import { ErrorCode, isObject, normalizeDistTags, semverSort } from './utils';
+import { logger } from './logger';
+import { ErrorCode, isObject, normalizeDistTags } from './utils';
 
-export function generatePackageTemplate(name: string): Package {
+export function generatePackageTemplate(name: string): Manifest {
   return {
     // standard things
     name,
@@ -34,7 +29,7 @@ export function generatePackageTemplate(name: string): Package {
  * Normalize package properties, tags, revision id.
  * @param {Object} pkg package reference.
  */
-export function normalizePackage(pkg: Package): Package {
+export function normalizePackage(pkg: Manifest): Manifest {
   const pkgProperties = ['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks', 'time'];
 
   pkgProperties.forEach((key): void => {
@@ -65,7 +60,7 @@ export function generateRevision(rev: string): string {
   return (+_rev[0] || 0) + 1 + '-' + generateRandomHexString();
 }
 
-export function getLatestReadme(pkg: Package): string {
+export function getLatestReadme(pkg: Manifest): string {
   const versions = pkg['versions'] || {};
   const distTags = pkg[DIST_TAGS] || {};
   // FIXME: here is a bit tricky add the types
@@ -107,7 +102,7 @@ export const WHITELIST = [
   'users',
 ];
 
-export function cleanUpLinksRef(keepUpLinkData: boolean, result: Package): Package {
+export function cleanUpLinksRef(keepUpLinkData: boolean, result: Manifest): Manifest {
   const propertyToKeep = [...WHITELIST];
   if (keepUpLinkData === true) {
     propertyToKeep.push('_uplinks');
@@ -152,7 +147,9 @@ export function publishPackage(
       if (!_.isNull(err)) {
         return reject(err);
       } else if (!_.isUndefined(latest)) {
-        Search.add(latest);
+        SearchMemoryIndexer.add(latest).catch((reason) => {
+          logger.error('indexer has failed on add item');
+        });
       }
       return resolve();
     });

src/lib/storage.ts

@@ -4,6 +4,7 @@ import _ from 'lodash';
 import Stream from 'stream';
 
 import { validatioUtils } from '@verdaccio/core';
+import { SearchMemoryIndexer } from '@verdaccio/search';
 import { ReadTarball } from '@verdaccio/streams';
 import {
   Callback,
@@ -24,7 +25,6 @@ import { hasProxyTo } from './config-utils';
 import { API_ERROR, DIST_TAGS, HTTP_STATUS } from './constants';
 import LocalStorage from './local-storage';
 import { mergeVersions } from './metadata-utils';
-import Search from './search';
 import {
   checkPackageLocal,
   checkPackageRemote,
@@ -146,7 +146,9 @@ class Storage {
   public removePackage(name: string, callback: Callback): void {
     this.localStorage.removePackage(name, callback);
     // update the indexer
-    Search.remove(name);
+    SearchMemoryIndexer.remove(name).catch((reason) => {
+      logger.error('indexer has failed on remove item');
+    });
   }
 
   /**

src/types/index.ts

@@ -1,5 +1,4 @@
 import { NextFunction, Request, Response } from 'express';
-import lunrMutable from 'lunr-mutable-indexes';
 
 import { pluginUtils } from '@verdaccio/core';
 import {
@@ -89,17 +88,6 @@ interface IAuthMiddleware {
   webUIJWTmiddleware(): $NextFunctionVer;
 }
 
-export interface IWebSearch {
-  index: lunrMutable.index;
-  storage: Storage;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  query(query: string): any;
-  add(pkg: Version): void;
-  remove(name: string): void;
-  reindex(): void;
-  configureStorage(storage: Storage): void;
-}
-
 export interface ISyncUplinks {
   uplinksLook?: boolean;
   etag?: string;

test/unit/modules/search/search.spec.ts

@@ -1,87 +0,0 @@
-import Config from '../../../../src/lib/config';
-import { setup } from '../../../../src/lib/logger';
-import Search from '../../../../src/lib/search';
-import Storage from '../../../../src/lib/storage';
-import buildConfig from '../../partials/config';
-
-setup([]);
-
-let packages = [
-  {
-    name: 'test1',
-    description: 'description',
-    _npmUser: {
-      name: 'test_user',
-    },
-  },
-  {
-    name: 'test2',
-    description: 'description',
-    _npmUser: {
-      name: 'test_user',
-    },
-  },
-  {
-    name: 'test3',
-    description: 'description',
-    _npmUser: {
-      name: 'test_user',
-    },
-  },
-  {
-    name: '@verdaccio/scope',
-    description: 'scope',
-    _npmUser: {
-      name: 'scope_user',
-    },
-  },
-  {
-    name: '@any/scope',
-    description: 'scope',
-    _npmUser: {
-      name: 'scope_user',
-    },
-  },
-];
-
-describe('search', () => {
-  beforeAll(async function () {
-    let config = new Config(buildConfig());
-    const storage = new Storage(config);
-    await storage.init(config);
-    Search.configureStorage(storage);
-    packages.map(function (item) {
-      // @ts-ignore
-      Search.add(item);
-    });
-  });
-
-  test('search query item', () => {
-    let result = Search.query('t');
-    expect(result).toHaveLength(3);
-  });
-
-  test('search query with @scope', () => {
-    let result = Search.query('@');
-    expect(result).toHaveLength(2);
-    result = Search.query('@verdaccio');
-    expect(result).toHaveLength(1);
-  });
-
-  test('search remove item', () => {
-    let item = {
-      name: 'test6',
-      description: 'description',
-      _npmUser: {
-        name: 'test_user',
-      },
-    };
-    // @ts-ignore
-    Search.add(item);
-    let result = Search.query('test6');
-    expect(result).toHaveLength(1);
-    Search.remove(item.name);
-    result = Search.query('test6');
-    expect(result).toHaveLength(0);
-  });
-});

yarn.lock

@@ -3219,6 +3219,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@verdaccio/search@npm:6.0.0-6-next.2":
+  version: 6.0.0-6-next.2
+  resolution: "@verdaccio/search@npm:6.0.0-6-next.2"
+  checksum: 33f85e5e15f77826b4d8cded78b899234fa63e660c7d5530ae5230a507561e1ab31ea96b304735e05c990fa15a653ba7b89cdc7e7e6e7e221a376a48a4670aa0
+  languageName: node
+  linkType: hard
+
 "@verdaccio/signature@npm:6.0.0-6-next.2":
   version: 6.0.0-6-next.2
   resolution: "@verdaccio/signature@npm:6.0.0-6-next.2"
@@ -7800,22 +7807,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lunr-mutable-indexes@npm:2.3.2":
-  version: 2.3.2
-  resolution: "lunr-mutable-indexes@npm:2.3.2"
-  dependencies:
-    lunr: ">= 2.3.0 < 2.4.0"
-  checksum: 792ec9a7f02071e65a03d843555ce8ee65b4580e767bdf0f5a0c72fecbcaa928a3e71c0c001d9d80c3855fc070ae826470f5aaaf8f1863d3402db057e59c5176
-  languageName: node
-  linkType: hard
-
-"lunr@npm:>= 2.3.0 < 2.4.0":
-  version: 2.3.9
-  resolution: "lunr@npm:2.3.9"
-  checksum: 176719e24fcce7d3cf1baccce9dd5633cd8bdc1f41ebe6a180112e5ee99d80373fe2454f5d4624d437e5a8319698ca6837b9950566e15d2cae5f2a543a3db4b8
-  languageName: node
-  linkType: hard
-
 "make-dir@npm:^2.0.0, make-dir@npm:^2.1.0":
   version: 2.1.0
   resolution: "make-dir@npm:2.1.0"
@@ -10732,6 +10723,7 @@ __metadata:
     "@verdaccio/local-storage": 10.3.3
     "@verdaccio/logger-7": 6.0.0-6-next.13
     "@verdaccio/middleware": 6.0.0-6-next.47
+    "@verdaccio/search": 6.0.0-6-next.2
     "@verdaccio/signature": 6.0.0-6-next.2
     "@verdaccio/streams": 10.2.1
     "@verdaccio/tarball": 11.0.0-6-next.37
@@ -10777,7 +10769,6 @@ __metadata:
     lockfile-lint: 4.10.1
     lodash: 4.17.21
     lru-cache: 7.18.3
-    lunr-mutable-indexes: 2.3.2
     mime: 3.0.0
     mkdirp: 1.0.4
     mv: 2.1.1