legacy storages support

legacy storages support

rebase

.eslintignore

@@ -14,6 +14,10 @@ test/functional/store/*
 docker-examples/**/lib/**/*.js
 test/cli/e2e-yarn4/bin/yarn-4.0.0-rc.14.cjs
 yarn.js
+
+#  used for unit tests
+packages/store/test/fixtures/plugins/**/*
+
 # storybook
 packages/ui-components/storybook-static
 dist.js

packages/server/fastify/src/endpoints/publish.ts

@@ -13,7 +13,7 @@
 //     const { packageName } = request.params;
 //     const storage = fastify.storage;
 //     debug('pkg name %s ', packageName);
-//     // const data = await storage?.getPackageNext({
+//     // const data = await storage?.getPackage({
 //     //   name: packageName,
 //     //   req: request.raw,
 //     //   uplinksLook: true,

packages/store/src/lib/legacy-utils.ts

@@ -0,0 +1,59 @@
+const isPromiseFunction = (func: any) => {
+  return typeof func === 'function' && func.then !== undefined;
+};
+
+const isCallbackFunction = (func: any) => {
+  return typeof func === 'function' && func.length > 0 && func.toString().includes('callback');
+};
+
+export function checkFunctionIsPromise<T>(instance: T, methodName: keyof T): boolean {
+  const method = instance[methodName];
+
+  if (typeof method !== 'function') {
+    return false;
+  }
+
+  if (isPromiseFunction(method)) {
+    return true;
+  }
+
+  if (isCallbackFunction(method)) {
+    return false;
+  }
+
+  return true;
+}
+
+export function isPromise(obj: any): obj is Promise<any> {
+  return (
+    !!obj &&
+    (typeof obj === 'object' || typeof obj === 'function') &&
+    typeof obj.then === 'function'
+  );
+}
+
+export function promisifiedCallbackFunction<T>(
+  instance: T,
+  methodName: keyof T,
+  ...args: any[]
+): Promise<any> {
+  return new Promise((resolve, reject) => {
+    const method = instance[methodName];
+
+    if (typeof method !== 'function') {
+      reject(new Error(`${String(methodName)} is not a function on the given instance.`));
+      return;
+    }
+
+    // Append the callback to the args
+    args.push((error: any, result: any) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    });
+
+    method.apply(instance, args);
+  });
+}

packages/store/src/local-storage.ts

@@ -33,8 +33,12 @@ class LocalStorage {
     if (this.storagePlugin === null) {
       this.storagePlugin = await this.loadStorage(this.config, this.logger);
       debug('storage plugin init');
-      await this.storagePlugin.init();
+      if (typeof this.storagePlugin.init !== 'undefined') {
-      debug('storage plugin initialized');
+        await this.storagePlugin.init();
+        debug('storage plugin initialized');
+      } else {
+        debug('storage plugin does not require initialization');
+      }
     } else {
       this.logger.warn('storage plugin has been already initialized');
     }

packages/store/src/storage.ts

@@ -68,6 +68,7 @@ import {
   tagVersion,
   tagVersionNext,
 } from '.';
+import { checkFunctionIsPromise, promisifiedCallbackFunction } from './lib/legacy-utils';
 import { isPublishablePackage } from './lib/star-utils';
 import { isExecutingStarCommand } from './lib/star-utils';
 import {
@@ -201,7 +202,9 @@ class Storage {
     }
 
     try {
-      const cacheManifest = await storage.readPackage(name);
+      const cacheManifest: Manifest = await (checkFunctionIsPromise(storage, 'readPackage')
+        ? storage.readPackage(name)
+        : promisifiedCallbackFunction(storage, 'readPackage', name));
       if (!cacheManifest._attachments[filename]) {
         throw errorUtils.getNotFound('no such file available');
       }
@@ -447,7 +450,7 @@ class Storage {
     }
 
     // we have version, so we need to return specific version
-    const [convertedManifest] = await this.getPackageNext(options);
+    const [convertedManifest] = await this.getPackage(options);
 
     const version: Version | undefined = getVersion(convertedManifest.versions, queryVersion);
 
@@ -494,7 +497,7 @@ class Storage {
 
   public async getPackageManifest(options: IGetPackageOptionsNext): Promise<Manifest> {
     // convert dist remotes to local bars
-    const [manifest] = await this.getPackageNext(options);
+    const [manifest] = await this.getPackage(options);
 
     // If change access is requested (?write=true), then check if logged in user is allowed to change package
     if (options.byPassCache === true) {
@@ -584,10 +587,16 @@ class Storage {
   public async getLocalDatabase(): Promise<Version[]> {
     debug('get local database');
     const storage = this.localStorage.getStoragePlugin();
-    const database = await storage.get();
+
+    // backward compatibility with legacy storage plugins
+    const database = await (checkFunctionIsPromise(storage, 'get')
+      ? storage.get()
+      : promisifiedCallbackFunction(storage, 'get'));
+
     const packages: Version[] = [];
     for (const pkg of database) {
       debug('get local database %o', pkg);
+
       const manifest = await this.getPackageLocalMetadata(pkg);
       const latest = manifest[DIST_TAGS].latest;
       if (latest && manifest.versions[latest]) {
@@ -823,7 +832,10 @@ class Storage {
     }
 
     try {
-      const result: Manifest = await storage.readPackage(name);
+      const result: Manifest = checkFunctionIsPromise(storage, 'readPackage')
+        ? await storage.readPackage(name)
+        : await promisifiedCallbackFunction(storage, 'readPackage', name);
+
       return normalizePackage(result);
     } catch (err: any) {
       if (err.code === STORAGE.NO_SUCH_FILE_ERROR || err.code === HTTP_STATUS.NOT_FOUND) {
@@ -1049,7 +1061,7 @@ class Storage {
    * @param name
    * @returns
    */
-  private async getPackagelocalByNameNext(name: string): Promise<Manifest | null> {
+  private async getPackagelocalByName(name: string): Promise<Manifest | null> {
     try {
       return await this.getPackageLocalMetadata(name);
     } catch (err: any) {
@@ -1082,6 +1094,7 @@ class Storage {
     if (typeof storage === 'undefined') {
       throw errorUtils.getNotFound();
     }
+    // TODO: juan
     const hasPackage = await storage.hasPackage();
     debug('has package %o for %o', pkgName, hasPackage);
     return hasPackage;
@@ -1123,7 +1136,7 @@ class Storage {
 
     try {
       // we check if package exist already locally
-      const localManifest = await this.getPackagelocalByNameNext(name);
+      const localManifest = await this.getPackagelocalByName(name);
       // if continue, the version to be published does not exist
       if (localManifest?.versions[versionToPublish] != null) {
         debug('%s version %s already exists (locally)', name, versionToPublish);
@@ -1600,7 +1613,7 @@ class Storage {
    * @return {*}  {Promise<[Manifest, any[]]>}
    * @memberof AbstractStorage
    */
-  private async getPackageNext(options: IGetPackageOptionsNext): Promise<[Manifest, any[]]> {
+  private async getPackage(options: IGetPackageOptionsNext): Promise<[Manifest, any[]]> {
     const { name } = options;
     debug('get package for %o', name);
     let data: Manifest | null = null;
@@ -1719,7 +1732,7 @@ class Storage {
 
     if (found && syncManifest !== null) {
       // updates the local cache manifest with fresh data
-      let updatedCacheManifest = await this.updateVersionsNext(name, syncManifest);
+      let updatedCacheManifest = await this.updateVersions(name, syncManifest);
       // plugin filter applied to the manifest
       const [filteredManifest, filtersErrors] = await this.applyFilters(updatedCacheManifest);
       return [
@@ -1883,13 +1896,16 @@ class Storage {
    * @return {Function}
    */
   private async readCreatePackage(pkgName: string): Promise<Manifest> {
-    const storage: any = this.getPrivatePackageStorage(pkgName);
+    const storage = this.getPrivatePackageStorage(pkgName);
     if (_.isNil(storage)) {
       throw errorUtils.getInternalError('storage could not be found');
     }
 
     try {
-      const result: Manifest = await storage.readPackage(pkgName);
+      // backward compatibility for legacy plugins
+      const result: Manifest = await (checkFunctionIsPromise(storage, 'readPackage')
+        ? storage.readPackage(pkgName)
+        : promisifiedCallbackFunction(storage, 'readPackage', pkgName));
       return normalizePackage(result);
     } catch (err: any) {
       if (err.code === STORAGE.NO_SUCH_FILE_ERROR || err.code === HTTP_STATUS.NOT_FOUND) {
@@ -1910,13 +1926,13 @@ class Storage {
 
     The steps are the following.
     1. Get the latest version of the package from the cache.
-    2. If does not exist will return a 
+    2. If does not exist will return a
 
     @param name
     @param remoteManifest
     @returns return a merged manifest.
   */
-  public async updateVersionsNext(name: string, remoteManifest: Manifest): Promise<Manifest> {
+  public async updateVersions(name: string, remoteManifest: Manifest): Promise<Manifest> {
     debug(`updating versions for package %o`, name);
     let cacheManifest: Manifest = await this.readCreatePackage(name);
     let change = false;

packages/store/test/fixtures/config/storage/plugin-legacy.yaml

@@ -0,0 +1,18 @@
+packages:
+  '@scope/foo':
+    access: $all
+    publish: $authenticated
+  '@*/*':
+    access: $all
+    publish: $all
+    proxy: ver
+  'foo':
+    access: $all
+    publish: $authenticated
+  '*':
+    access: $all
+    publish: $all
+    proxy: npmjs
+store:
+  legacy-storage-plugin:
+plugins: /roo/does-not-exist

packages/store/test/fixtures/config/storage/plugin-publish-legacy.yaml

@@ -0,0 +1,16 @@
+uplinks:
+  ver:
+    url: https://registry.verdaccio.org/
+packages:
+  '@*/*':
+    access: $all
+    publish: $all
+    proxy: ver
+  'upstream':
+    access: $all
+    publish: $authenticated
+  '*':
+    access: $all
+    publish: $all
+    proxy: ver
+log: { type: stdout, format: pretty, level: info }

packages/store/test/fixtures/plugins/verdaccio-legacy-storage-plugin/lib/index.js

@@ -0,0 +1,15 @@
+'use strict';
+var __importDefault =
+  (this && this.__importDefault) ||
+  function (mod) {
+    return mod && mod.__esModule ? mod : { default: mod };
+  };
+Object.defineProperty(exports, '__esModule', { value: true });
+exports.default = void 0;
+var plugin_1 = require('./plugin');
+Object.defineProperty(exports, 'default', {
+  enumerable: true,
+  get: function () {
+    return __importDefault(plugin_1).default;
+  },
+});

packages/store/test/fixtures/plugins/verdaccio-legacy-storage-plugin/lib/local-storage.js

@@ -0,0 +1,160 @@
+'use strict';
+Object.defineProperty(exports, '__esModule', { value: true });
+const commons_api_1 = require('@verdaccio/commons-api');
+class StoragePluginManage {
+  logger;
+  packageName;
+  config;
+  constructor(config, packageName, logger) {
+    this.logger = logger;
+    this.packageName = packageName;
+    this.config = config;
+  }
+  /**
+   * Handle a metadata update and
+   * @param name
+   * @param updateHandler
+   * @param onWrite
+   * @param transformPackage
+   * @param onEnd
+   */
+  updatePackage(name, updateHandler, onWrite, transformPackage, onEnd) {
+    onEnd((0, commons_api_1.getInternalError)('Not implemented'));
+    /**
+     * Example of implementation:
+     this.customStore.get().then((pkg: Package) => {
+     updateHandler(pkg, function onUpdateFinish(err) {
+     if (err) {
+     onEnd(err);
+     } else {
+     onWrite(name, pkg, onEnd);
+     }
+     })
+     });
+     */
+  }
+  /**
+   * Delete a specific file (tarball or package.json)
+   * @param fileName
+   * @param callback
+   */
+  deletePackage(fileName, callback) {
+    callback((0, commons_api_1.getInternalError)('Not implemented'));
+    /**
+     * Example of implementation:
+     this.customStore.delete(fileName,  (err) => {
+     if (err) {
+     callback(err);
+     } else {
+     callback(null);
+     }
+     })
+     */
+  }
+  /**
+   * Delete a package (folder, path)
+   * This happens after all versions ar tarballs have been removed.
+   * @param callback
+   */
+  removePackage(callback) {
+    callback((0, commons_api_1.getInternalError)('Not implemented'));
+    /**
+     * Example of implementation:
+     this.customStore.removePackage((err) => {
+     if (err) {
+     callback(err);
+     } else {
+     callback(null);
+     }
+     })
+     */
+  }
+  /**
+   * Publish a new package (version).
+   * @param name
+   * @param data
+   * @param callback
+   */
+  createPackage(name, data, callback) {
+    callback((0, commons_api_1.getInternalError)('Not implemented'));
+    /**
+     * Example of implementation:
+     * this.customStore.create(name, data).then(err => {
+     if (err.notFound) {
+     callback(getNotFound());
+     } else if (err.alreadyExist) {
+     callback(getConflict());
+     } else {
+     callback(null);
+     }
+     })
+     */
+  }
+  /**
+   * Perform write anobject to the storage.
+   * Similar to updatePackage but without middleware handlers
+   * @param pkgName package name
+   * @param pkg package metadata
+   * @param callback
+   */
+  savePackage(pkgName, pkg, callback) {
+    callback((0, commons_api_1.getInternalError)('Not implemented'));
+    /*
+      Example of implementation:
+      this.cumstomStore.write(pkgName, pkgName).then(data => {
+        callback(null);
+      }).catch(err => {
+        callback(getInternalError(err.message));
+      })
+    */
+  }
+  /**
+   * Read a package from storage
+   * @param pkgName package name
+   * @param callback
+   */
+  readPackage(pkgName, callback) {
+    callback(null, {
+      name: pkgName,
+      'dist-tags': { latest: '1.0.0' },
+      versions: { '1.0.0': { pkgName } },
+    });
+    /**
+     * Example of implementation:
+     * this.customStorage.read(name, (err, pkg: Package) => {
+     if (err.fooError) {
+     callback(getInternalError(err))
+     } else if (err.barError) {
+     callback(getNotFound());
+     } else {
+     callback(null, pkg)
+     }
+     });
+     */
+  }
+  /**
+   * Create writtable stream (write a tarball)
+   * @param name
+   */
+  writeTarball(name) {
+    /**
+     * Example of implementation:
+     * const stream = new UploadTarball({});
+     return stream;
+     */
+    return Buffer.from('');
+  }
+  /**
+   * Create a readable stream (read a from a tarball)
+   * @param name
+   */
+  readTarball(name) {
+    /**
+     * Example of implementation:
+     * const stream = new ReadTarball({});
+     return stream;
+     */
+    return Buffer.from('');
+  }
+}
+exports.default = StoragePluginManage;

packages/store/test/fixtures/plugins/verdaccio-legacy-storage-plugin/lib/plugin.js

@@ -0,0 +1,125 @@
+'use strict';
+var __importDefault =
+  (this && this.__importDefault) ||
+  function (mod) {
+    return mod && mod.__esModule ? mod : { default: mod };
+  };
+Object.defineProperty(exports, '__esModule', { value: true });
+const commons_api_1 = require('@verdaccio/commons-api');
+const local_storage_1 = __importDefault(require('./local-storage'));
+
+class VerdaccioStoragePlugin {
+  config;
+  version;
+  logger;
+
+  constructor(config, options) {
+    this.config = config;
+    this.logger = options.logger;
+  }
+
+  /**
+   *
+   */
+  async getSecret() {
+    return 'your secret';
+  }
+
+  async setSecret(secret) {
+    return true;
+  }
+
+  /**
+   * Add a new element.
+   * @param {*} name
+   * @return {Error|*}
+   */
+  add(name, callback) {
+    callback((0, commons_api_1.getInternalError)('your own message here'));
+  }
+
+  /**
+   * Perform a search in your registry
+   * @param onPackage
+   * @param onEnd
+   * @param validateName
+   */
+  search(onPackage, onEnd, validateName) {
+    onEnd();
+    /**
+     * Example of implementation:
+     * try {
+     *  someApi.getPackages((items) => {
+     *   items.map(() => {
+     *     if (validateName(item.name)) {
+     *       onPackage(item);
+     *     }
+     *   });
+     *  onEnd();
+     * } catch(err) {
+     *   onEnd(err);
+     * }
+     * });
+     */
+  }
+
+  /**
+   * Remove an element from the database.
+   * @param {*} name
+   * @return {Error|*}
+   */
+  remove(name, callback) {
+    callback((0, commons_api_1.getInternalError)('your own message here'));
+    /**
+     * Example of implementation
+     database.getPackage(name, (item, err) => {
+     if (err) {
+     callback(getInternalError('your own message here'));
+     }
+
+     // if all goes well we return nothing
+     callback(null);
+     }
+     */
+  }
+
+  /**
+   * Return all database elements.
+   * @return {Array}
+   */
+  get(callback) {
+    callback(null, [{ name: 'your-package' }]);
+    /*
+      Example of implementation
+      database.getAll((allItems, err) => {
+        callback(err, allItems);
+      })
+    */
+  }
+
+  /**
+   * Create an instance of the `PackageStorage`
+   * @param packageInfo
+   */
+  getPackageStorage(packageInfo) {
+    return new local_storage_1.default(this.config, packageInfo, this.logger);
+  }
+
+  /**
+   * All methods for npm token support
+   * more info here https://github.com/verdaccio/verdaccio/pull/1427
+   */
+  saveToken(token) {
+    throw new Error('Method not implemented.');
+  }
+
+  deleteToken(user, tokenKey) {
+    throw new Error('Method not implemented.');
+  }
+
+  readTokens(filter) {
+    throw new Error('Method not implemented.');
+  }
+}
+
+exports.default = VerdaccioStoragePlugin;

packages/store/test/fixtures/plugins/verdaccio-legacy-storage-plugin/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "verdaccio-legacy-storage-plugin",
+  "version": "1.0.0",
+  "description": "",
+  "main": "lib/index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

packages/store/test/helpers.ts

@@ -1,14 +1,21 @@
+import { pseudoRandomBytes } from 'crypto';
 import buildDebug from 'debug';
+import fs from 'fs';
+import os from 'os';
 import path from 'path';
 
 import { parseConfigFile } from '@verdaccio/config';
+import { Config, getDefaultConfig } from '@verdaccio/config';
+import { ConfigYaml, PackageUsers } from '@verdaccio/types';
 
-const debug = buildDebug('verdaccio:mock:config');
+const debug = buildDebug('verdaccio:storage:test:helpers');
+
+export const domain = 'https://registry.npmjs.org';
 
 /**
  * Override the default.yaml configuration file with any new config provided.
  */
-function configExample(externalConfig: any = {}, configFile?: string, location?: string) {
+export function configExample(externalConfig: any = {}, configFile?: string, location?: string) {
   let config = {};
   if (location && configFile) {
     const locationFile = path.join(location, configFile);
@@ -19,4 +26,54 @@ function configExample(externalConfig: any = {}, configFile?: string, location?:
   return { ...externalConfig, ...config };
 }
 
-export { configExample };
+export function generateRandomStorage() {
+  const tempStorage = pseudoRandomBytes(5).toString('hex');
+  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), '/verdaccio-test'));
+
+  return path.join(tempRoot, tempStorage);
+}
+
+export const getConfig = (file, override: Partial<ConfigYaml> = {}): Config => {
+  const config = new Config(
+    configExample(
+      {
+        ...getDefaultConfig(),
+        storage: generateRandomStorage(),
+        ...override,
+      },
+      `./fixtures/config/${file}`,
+      __dirname
+    )
+  );
+  return config;
+};
+
+export const defaultRequestOptions = {
+  host: 'localhost',
+  protocol: 'http',
+  headers: {},
+};
+export const executeStarPackage = async (
+  storage,
+  options: {
+    users: PackageUsers;
+    username: string;
+    name: string;
+    _rev: string;
+    _id?: string;
+  }
+) => {
+  const { name, _rev, _id, users, username } = options;
+  const starManifest = {
+    _rev,
+    _id,
+    users,
+  };
+  return storage.updateManifest(starManifest, {
+    signal: new AbortController().signal,
+    name,
+    uplinksLook: true,
+    revision: '1',
+    requestOptions: { ...defaultRequestOptions, username },
+  });
+};

packages/store/test/legacy-utils.spec.ts

@@ -0,0 +1,66 @@
+import { checkFunctionIsPromise, promisifiedCallbackFunction } from '../src/lib/legacy-utils';
+
+describe('utils', () => {
+  class MyClass {
+    asyncFunction(): Promise<string> {
+      return new Promise((resolve) => {
+        setTimeout(() => {
+          resolve('I am a promise');
+        }, 1000);
+      });
+    }
+
+    readPackage(pkgName: string, callback: (error: any, result: string) => void): void {
+      setTimeout(() => {
+        if (pkgName === 'error') {
+          callback(new Error('Package not found'), null);
+        } else {
+          callback(null, `Package ${pkgName} data`);
+        }
+      }, 1000);
+    }
+  }
+
+  describe('checkFunctionIsPromise', () => {
+    let myInstance: MyClass;
+    beforeEach(() => {
+      myInstance = new MyClass();
+    });
+
+    test('should identify asyncFunction as "Promise"', () => {
+      expect(checkFunctionIsPromise(myInstance, 'asyncFunction')).toBeTruthy();
+    });
+
+    test('should throw an error if method is not a function', () => {
+      expect(checkFunctionIsPromise(myInstance, 'nonExistentFunction' as any)).toBeFalsy();
+    });
+  });
+
+  describe('promisifiedCallbackFunction', () => {
+    let myInstance: MyClass;
+
+    beforeEach(() => {
+      myInstance = new MyClass();
+    });
+
+    test('should reject if method is not a function', async () => {
+      await expect(
+        promisifiedCallbackFunction(myInstance, 'nonExistentFunction' as any)
+      ).rejects.toThrow('nonExistentFunction is not a function on the given instance.');
+    });
+
+    test('should resolve with the correct value for readPackage when no error', async () => {
+      await expect(
+        promisifiedCallbackFunction(myInstance, 'readPackage', 'examplePackage')
+      ).resolves.toBe('Package examplePackage data');
+    });
+
+    test('should reject with an error for readPackage when there is an error', async () => {
+      await expect(promisifiedCallbackFunction(myInstance, 'readPackage', 'error')).rejects.toThrow(
+        'Package not found'
+      );
+    });
+
+    // Additional tests to cover other cases can be added here
+  });
+});

packages/store/test/storage.plugin.spec.ts

@@ -0,0 +1,89 @@
+import MockDate from 'mockdate';
+import nock from 'nock';
+import path from 'path';
+
+import { Config, getDefaultConfig } from '@verdaccio/config';
+import { DIST_TAGS } from '@verdaccio/core';
+import { setup } from '@verdaccio/logger';
+import { generatePackageMetadata } from '@verdaccio/test-helper';
+import { Manifest } from '@verdaccio/types';
+
+import { Storage } from '../src';
+import { configExample, generateRandomStorage, getConfig } from './helpers';
+
+setup({ type: 'stdout', format: 'pretty', level: 'trace' });
+
+const pluginsPartialsFolder = path.join(__dirname, './fixtures/plugins');
+
+describe('storage plugin', () => {
+  beforeEach(() => {
+    nock.cleanAll();
+    nock.abortPendingRequests();
+    jest.clearAllMocks();
+  });
+
+  describe('Plugin Legacy Support', () => {
+    test('should return no results from a legacy plugin', async () => {
+      const configJSON = getConfig('storage/plugin-legacy.yaml');
+      const config = new Config(
+        configExample({
+          ...configJSON,
+          plugins: pluginsPartialsFolder,
+          storage: generateRandomStorage(),
+        })
+      );
+
+      const storage = new Storage(config);
+      await storage.init(config);
+      await expect(storage.getLocalDatabase()).resolves.toHaveLength(1);
+    });
+    test('create private package', async () => {
+      const mockDate = '2018-01-14T11:17:40.712Z';
+      MockDate.set(mockDate);
+      const configJSON = getConfig('storage/plugin-publish-legacy.yaml');
+      const pkgName = 'upstream';
+      const requestOptions = {
+        host: 'localhost',
+        protocol: 'http',
+        headers: {},
+      };
+      const config = new Config(
+        configExample({
+          ...getDefaultConfig(),
+          ...configJSON,
+          plugins: pluginsPartialsFolder,
+          storage: generateRandomStorage(),
+        })
+      );
+      const storage = new Storage(config);
+      await storage.init(config);
+      const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
+      await storage.updateManifest(bodyNewManifest, {
+        signal: new AbortController().signal,
+        name: pkgName,
+        uplinksLook: true,
+        revision: '1',
+        requestOptions,
+      });
+      const manifest = (await storage.getPackageByOptions({
+        name: pkgName,
+        uplinksLook: true,
+        requestOptions,
+      })) as Manifest;
+      expect(manifest.name).toEqual(pkgName);
+      expect(manifest._id).toEqual(pkgName);
+      expect(Object.keys(manifest.versions)).toEqual(['1.0.0']);
+      expect(manifest.time).toEqual({
+        '1.0.0': mockDate,
+        created: mockDate,
+        modified: mockDate,
+      });
+      expect(manifest[DIST_TAGS]).toEqual({ latest: '1.0.0' });
+      // verdaccio keeps latest version of readme on manifest level but not by version
+      expect(manifest.versions['1.0.0'].readme).not.toBeDefined();
+      expect(manifest.readme).toEqual('# test');
+      expect(manifest._attachments).toEqual({});
+      expect(typeof manifest._rev).toBeTruthy();
+    });
+  });
+});

packages/store/test/storage.spec.ts

@@ -1,101 +1,32 @@
-import { pseudoRandomBytes } from 'crypto';
-import fs from 'fs';
 import MockDate from 'mockdate';
 import nock from 'nock';
 import * as httpMocks from 'node-mocks-http';
-import os from 'os';
-import path from 'path';
 
 import { Config, getDefaultConfig } from '@verdaccio/config';
-import {
+import { API_ERROR, API_MESSAGE, DIST_TAGS, HEADERS, errorUtils, fileUtils } from '@verdaccio/core';
-  API_ERROR,
-  API_MESSAGE,
-  DIST_TAGS,
-  HEADERS,
-  HEADER_TYPE,
-  errorUtils,
-  fileUtils,
-} from '@verdaccio/core';
 import { setup } from '@verdaccio/logger';
 import {
-  addNewVersion,
   generateLocalPackageMetadata,
   generatePackageMetadata,
-  generateRemotePackageMetadata,
   getDeprecatedPackageMetadata,
 } from '@verdaccio/test-helper';
-import {
+import { AbbreviatedManifest, Author, Manifest, Version } from '@verdaccio/types';
-  AbbreviatedManifest,
-  Author,
-  ConfigYaml,
-  Manifest,
-  PackageUsers,
-  Version,
-} from '@verdaccio/types';
 
 import { Storage } from '../src';
-import manifestFooRemoteNpmjs from './fixtures/manifests/foo-npmjs.json';
+import {
-import { configExample } from './helpers';
+  configExample,
-
+  defaultRequestOptions,
-function generateRandomStorage() {
+  domain,
-  const tempStorage = pseudoRandomBytes(5).toString('hex');
+  executeStarPackage,
-  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), '/verdaccio-test'));
+  generateRandomStorage,
-
+  getConfig,
-  return path.join(tempRoot, tempStorage);
+} from './helpers';
-}
 
 const logger = setup({ type: 'stdout', format: 'pretty', level: 'trace' });
 
-const domain = 'https://registry.npmjs.org';
 const fakeHost = 'localhost:4873';
 const fooManifest = generatePackageMetadata('foo', '1.0.0');
 
-const getConfig = (file, override: Partial<ConfigYaml> = {}): Config => {
-  const config = new Config(
-    configExample(
-      {
-        ...getDefaultConfig(),
-        storage: generateRandomStorage(),
-        ...override,
-      },
-      `./fixtures/config/${file}`,
-      __dirname
-    )
-  );
-  return config;
-};
-
-const defaultRequestOptions = {
-  host: 'localhost',
-  protocol: 'http',
-  headers: {},
-};
-
-const executeStarPackage = async (
-  storage,
-  options: {
-    users: PackageUsers;
-    username: string;
-    name: string;
-    _rev: string;
-    _id?: string;
-  }
-) => {
-  const { name, _rev, _id, users, username } = options;
-  const starManifest = {
-    _rev,
-    _id,
-    users,
-  };
-  return storage.updateManifest(starManifest, {
-    signal: new AbortController().signal,
-    name,
-    uplinksLook: true,
-    revision: '1',
-    requestOptions: { ...defaultRequestOptions, username },
-  });
-};
-
 const executeChangeOwners = async (
   storage,
   options: {
@@ -128,7 +59,7 @@ describe('storage', () => {
     jest.clearAllMocks();
   });
 
-  describe('updateManifest', () => {
+  describe('publishing commands', () => {
     describe('publishing', () => {
       test('create private package', async () => {
         const mockDate = '2018-01-14T11:17:40.712Z';
@@ -181,6 +112,7 @@ describe('storage', () => {
       });
 
       // TODO: Review triggerUncaughtException exception on abort
+      // is not working as expected, throws but crash the test
       test.skip('abort creating a private package', async () => {
         const mockDate = '2018-01-14T11:17:40.712Z';
         MockDate.set(mockDate);
@@ -682,7 +614,6 @@ describe('storage', () => {
             _rev: bodyNewManifest._rev,
             _id: bodyNewManifest._id,
             name: pkgName,
-            // @ts-expect-error
             username: undefined,
             users: { fooUser: true },
           })
@@ -887,758 +818,53 @@ describe('storage', () => {
         }
       );
     });
-  });
+    describe('tokens', () => {
-
+      describe('saveToken', () => {
-  describe('getTarball', () => {
+        test('should retrieve tokens created', async () => {
-    test('should get a package from local storage', (done) => {
+          const config = new Config(
-      const pkgName = 'foo';
+            configExample({
-      const config = new Config(
-        configExample({
-          ...getDefaultConfig(),
-          storage: generateRandomStorage(),
-        })
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const ac = new AbortController();
-        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
-        storage
-          .updateManifest(bodyNewManifest, {
-            signal: ac.signal,
-            name: pkgName,
-            uplinksLook: false,
-            requestOptions: defaultRequestOptions,
-          })
-          .then(() => {
-            const abort = new AbortController();
-            storage
-              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
-                signal: abort.signal,
-              })
-              .then((stream) => {
-                stream.on('data', (dat) => {
-                  expect(dat).toBeDefined();
-                  expect(dat.length).toEqual(512);
-                });
-                stream.on('end', () => {
-                  done();
-                });
-                stream.on('error', () => {
-                  done('this should not happen');
-                });
-              });
-          });
-      });
-    });
-
-    test('should not found a package anywhere', (done) => {
-      const config = new Config(
-        configExample({
-          ...getDefaultConfig(),
-          storage: generateRandomStorage(),
-        })
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const abort = new AbortController();
-        storage
-          .getTarball('some-tarball', 'some-tarball-1.0.0.tgz', {
-            signal: abort.signal,
-          })
-          .then((stream) => {
-            stream.on('error', (err) => {
-              expect(err).toEqual(errorUtils.getNotFound(API_ERROR.NO_PACKAGE));
-              done();
-            });
-          });
-      });
-    });
-
-    test('should create a package if tarball is requested and does not exist locally', (done) => {
-      const pkgName = 'upstream';
-      const upstreamManifest = generateRemotePackageMetadata(
-        pkgName,
-        '1.0.0',
-        'https://registry.something.org'
-      );
-      nock('https://registry.verdaccio.org').get(`/${pkgName}`).reply(201, upstreamManifest);
-      nock('https://registry.something.org')
-        .get(`/${pkgName}/-/${pkgName}-1.0.0.tgz`)
-        // types does not match here with documentation
-        // @ts-expect-error
-        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
-          [HEADER_TYPE.CONTENT_LENGTH]: 277,
-        });
-      const config = new Config(
-        configExample(
-          {
-            storage: generateRandomStorage(),
-          },
-          './fixtures/config/getTarball-getupstream.yaml',
-          __dirname
-        )
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const abort = new AbortController();
-        storage
-          .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
-            signal: abort.signal,
-          })
-          .then((stream) => {
-            stream.on('data', (dat) => {
-              expect(dat).toBeDefined();
-            });
-            stream.on('end', () => {
-              done();
-            });
-            stream.on('error', () => {
-              done('this should not happen');
-            });
-          });
-      });
-    });
-
-    test('should serve fetch tarball from upstream without dist info local', (done) => {
-      const pkgName = 'upstream';
-      const upstreamManifest = addNewVersion(
-        generateRemotePackageMetadata(pkgName, '1.0.0') as Manifest,
-        '1.0.1'
-      );
-      nock('https://registry.verdaccio.org').get(`/${pkgName}`).reply(201, upstreamManifest);
-      nock('http://localhost:5555')
-        .get(`/${pkgName}/-/${pkgName}-1.0.1.tgz`)
-        // types does not match here with documentation
-        // @ts-expect-error
-        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
-          [HEADER_TYPE.CONTENT_LENGTH]: 277,
-        });
-      const config = new Config(
-        configExample(
-          {
-            storage: generateRandomStorage(),
-          },
-          './fixtures/config/getTarball-getupstream.yaml',
-          __dirname
-        )
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const ac = new AbortController();
-        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
-        storage
-          .updateManifest(bodyNewManifest, {
-            signal: ac.signal,
-            name: pkgName,
-            uplinksLook: true,
-            revision: '1',
-            requestOptions: {
-              host: 'localhost',
-              protocol: 'http',
-              headers: {},
-            },
-          })
-          .then(() => {
-            const abort = new AbortController();
-            storage
-              .getTarball(pkgName, `${pkgName}-1.0.1.tgz`, {
-                signal: abort.signal,
-              })
-              .then((stream) => {
-                stream.on('data', (dat) => {
-                  expect(dat).toBeDefined();
-                });
-                stream.on('end', () => {
-                  done();
-                });
-                stream.on('error', () => {
-                  done('this should not happen');
-                });
-              });
-          });
-      });
-    });
-
-    test('should serve fetch tarball from upstream without with info local', (done) => {
-      const pkgName = 'upstream';
-      const upstreamManifest = addNewVersion(
-        addNewVersion(generateRemotePackageMetadata(pkgName, '1.0.0') as Manifest, '1.0.1'),
-        '1.0.2'
-      );
-      nock('https://registry.verdaccio.org')
-        .get(`/${pkgName}`)
-        .times(10)
-        .reply(201, upstreamManifest);
-      nock('http://localhost:5555')
-        .get(`/${pkgName}/-/${pkgName}-1.0.0.tgz`)
-        // types does not match here with documentation
-        // @ts-expect-error
-        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
-          [HEADER_TYPE.CONTENT_LENGTH]: 277,
-        });
-      const storagePath = generateRandomStorage();
-      const config = new Config(
-        configExample(
-          {
-            storage: storagePath,
-          },
-          './fixtures/config/getTarball-getupstream.yaml',
-          __dirname
-        )
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const req = httpMocks.createRequest({
-          method: 'GET',
-          connection: { remoteAddress: fakeHost },
-          headers: {
-            host: fakeHost,
-            [HEADERS.FORWARDED_PROTO]: 'http',
-          },
-          url: '/',
-        });
-        return storage
-          .getPackageByOptions({
-            name: pkgName,
-            uplinksLook: true,
-            requestOptions: {
-              headers: req.headers as any,
-              protocol: req.protocol,
-              host: req.get('host') as string,
-            },
-          })
-          .then(() => {
-            const abort = new AbortController();
-            storage
-              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
-                signal: abort.signal,
-              })
-              .then((stream) => {
-                stream.on('data', (dat) => {
-                  expect(dat).toBeDefined();
-                });
-                stream.on('end', () => {
-                  done();
-                });
-                stream.once('error', () => {
-                  done('this should not happen');
-                });
-              });
-          });
-      });
-    });
-
-    test('should serve local cache', (done) => {
-      const pkgName = 'upstream';
-      const config = new Config(
-        configExample(
-          {
-            storage: generateRandomStorage(),
-          },
-          './fixtures/config/getTarball-getupstream.yaml',
-          __dirname
-        )
-      );
-      const storage = new Storage(config, logger);
-      storage.init(config).then(() => {
-        const ac = new AbortController();
-        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
-        storage
-          .updateManifest(bodyNewManifest, {
-            signal: ac.signal,
-            name: pkgName,
-            uplinksLook: true,
-            revision: '1',
-            requestOptions: {
-              host: 'localhost',
-              protocol: 'http',
-              headers: {},
-            },
-          })
-          .then(() => {
-            const abort = new AbortController();
-            storage
-              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
-                signal: abort.signal,
-              })
-              .then((stream) => {
-                stream.on('data', (dat) => {
-                  expect(dat).toBeDefined();
-                });
-                stream.on('end', () => {
-                  done();
-                });
-                stream.on('error', () => {
-                  done('this should not happen');
-                });
-              });
-          });
-      });
-    });
-  });
-
-  describe('syncUplinksMetadata()', () => {
-    describe('error handling', () => {
-      test('should handle double failure on uplinks with timeout', async () => {
-        const fooManifest = generatePackageMetadata('timeout', '8.0.0');
-        nock('https://registry.timeout.com')
-          .get(`/${fooManifest.name}`)
-          .delayConnection(8000)
-          .reply(201, manifestFooRemoteNpmjs);
-
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncDoubleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-        await expect(
-          storage.syncUplinksMetadata(fooManifest.name, null, {
-            retry: { limit: 3 },
-            timeout: {
-              request: 1000,
-            },
-          })
-        ).rejects.toThrow(API_ERROR.NO_PACKAGE);
-      }, 18000);
-
-      test('should handle one proxy fails', async () => {
-        const fooManifest = generatePackageMetadata('foo', '8.0.0');
-        nock('https://registry.verdaccio.org').get('/foo').replyWithError('service in holidays');
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-        await expect(
-          storage.syncUplinksMetadata(fooManifest.name, null, {
-            retry: { limit: 0 },
-          })
-        ).rejects.toThrow(API_ERROR.NO_PACKAGE);
-      });
-
-      test('should handle one proxy reply 304', async () => {
-        const fooManifest = generatePackageMetadata('foo-no-data', '8.0.0');
-        nock('https://registry.verdaccio.org').get('/foo-no-data').reply(304);
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-        const [manifest] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
-          retry: { limit: 0 },
-        });
-        expect(manifest).toBe(fooManifest);
-      });
-    });
-
-    describe('success scenarios', () => {
-      test('should handle one proxy success', async () => {
-        const fooManifest = generateLocalPackageMetadata('foo', '8.0.0');
-        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-
-        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
-        expect(response).not.toBeNull();
-        expect((response as Manifest).name).toEqual(fooManifest.name);
-        expect(Object.keys((response as Manifest).versions)).toEqual([
-          '8.0.0',
-          '1.0.0',
-          '0.0.3',
-          '0.0.4',
-          '0.0.5',
-          '0.0.6',
-          '0.0.7',
-        ]);
-        expect(Object.keys((response as Manifest).time)).toEqual([
-          'modified',
-          'created',
-          '8.0.0',
-          '1.0.0',
-          '0.0.3',
-          '0.0.4',
-          '0.0.5',
-          '0.0.6',
-          '0.0.7',
-        ]);
-        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
-        expect((response as Manifest).time['8.0.0']).toBeDefined();
-      });
-
-      test('should handle one proxy success with no local cache manifest', async () => {
-        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-
-        const [response] = await storage.syncUplinksMetadata(fooManifest.name, null);
-        // the latest from the remote manifest
-        expect(response).not.toBeNull();
-        expect((response as Manifest).name).toEqual(fooManifest.name);
-        expect((response as Manifest)[DIST_TAGS].latest).toEqual('0.0.7');
-      });
-
-      test('should handle no proxy found with local cache manifest', async () => {
-        const fooManifest = generatePackageMetadata('foo', '8.0.0');
-        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncNoUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-
-        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
-        expect(response).not.toBeNull();
-        expect((response as Manifest).name).toEqual(fooManifest.name);
-        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
-      });
-      test.todo('should handle double proxy with last one success');
-    });
-
-    describe('options', () => {
-      test('should handle disable uplinks via options.uplinksLook=false with cache', async () => {
-        const fooManifest = generatePackageMetadata('foo', '8.0.0');
-        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
-        const config = new Config(
-          configExample(
-            {
-              storage: generateRandomStorage(),
-            },
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
-            __dirname
-          )
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-
-        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
-          uplinksLook: false,
-        });
-
-        expect((response as Manifest).name).toEqual(fooManifest.name);
-        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
-      });
-
-      test('should handle disable uplinks via options.uplinksLook=false without cache', async () => {
-        const fooRemoteManifest = generateRemotePackageMetadata(
-          'foo',
-          '9.0.0',
-          'https://registry.verdaccio.org',
-          ['9.0.0', '9.0.1', '9.0.2', '9.0.3']
-        );
-        nock('https://registry.verdaccio.org').get('/foo').reply(201, fooRemoteManifest);
-        const config = new Config(
-          configExample(
-            {
               ...getDefaultConfig(),
               storage: generateRandomStorage(),
-            },
+            })
-            './fixtures/config/syncSingleUplinksMetadata.yaml',
+          );
-            __dirname
+          const storage = new Storage(config, logger);
-          )
+          await storage.init(config);
-        );
+          await storage.saveToken({
-        const storage = new Storage(config, logger);
+            user: 'foo',
-        await storage.init(config);
+            token: 'secret',
-
+            key: 'key',
-        const [response] = await storage.syncUplinksMetadata('foo', null, {
+            created: 'created',
-          uplinksLook: true,
+            readonly: true,
+          });
+          const tokens = await storage.readTokens({ user: 'foo' });
+          expect(tokens).toEqual([
+            { user: 'foo', token: 'secret', key: 'key', readonly: true, created: 'created' },
+          ]);
         });
 
-        expect((response as Manifest).name).toEqual('foo');
+        test('should delete a token created', async () => {
-        expect((response as Manifest)[DIST_TAGS].latest).toEqual('9.0.0');
+          const config = new Config(
-      });
+            configExample({
-    });
+              ...getDefaultConfig(),
-  });
+              storage: generateRandomStorage(),
-
+            })
-  describe('getLocalDatabase', () => {
+          );
-    test('should return no results', async () => {
+          const storage = new Storage(config, logger);
-      const config = new Config(
+          await storage.init(config);
-        configExample({
+          await storage.saveToken({
-          ...getDefaultConfig(),
+            user: 'foo',
-          storage: generateRandomStorage(),
+            token: 'secret',
-        })
+            key: 'key',
-      );
+            created: 'created',
-      const storage = new Storage(config, logger);
+            readonly: true,
-      await storage.init(config);
+          });
-      await expect(storage.getLocalDatabase()).resolves.toHaveLength(0);
+          const tokens = await storage.readTokens({ user: 'foo' });
-    });
+          expect(tokens).toHaveLength(1);
-
+          await storage.deleteToken('foo', 'key');
-    test('should return single result', async () => {
+          const tokens2 = await storage.readTokens({ user: 'foo' });
-      const config = new Config(
+          expect(tokens2).toHaveLength(0);
-        configExample({
-          ...getDefaultConfig(),
-          storage: generateRandomStorage(),
-        })
-      );
-      const req = httpMocks.createRequest({
-        method: 'GET',
-        connection: { remoteAddress: fakeHost },
-        headers: {
-          host: 'host',
-        },
-        url: '/',
-      });
-      const storage = new Storage(config, logger);
-      await storage.init(config);
-      const manifest = generatePackageMetadata('foo');
-      const ac = new AbortController();
-      await storage.updateManifest(manifest, {
-        signal: ac.signal,
-        name: 'foo',
-        uplinksLook: false,
-        requestOptions: {
-          headers: req.headers as any,
-          protocol: req.protocol,
-          host: req.get('host') as string,
-        },
-      });
-      const response = await storage.getLocalDatabase();
-      expect(response).toHaveLength(1);
-      expect(response[0]).toEqual(expect.objectContaining({ name: 'foo', version: '1.0.0' }));
-    });
-  });
-
-  describe('tokens', () => {
-    describe('saveToken', () => {
-      test('should retrieve tokens created', async () => {
-        const config = new Config(
-          configExample({
-            ...getDefaultConfig(),
-            storage: generateRandomStorage(),
-          })
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-        await storage.saveToken({
-          user: 'foo',
-          token: 'secret',
-          key: 'key',
-          created: 'created',
-          readonly: true,
         });
-        const tokens = await storage.readTokens({ user: 'foo' });
-        expect(tokens).toEqual([
-          { user: 'foo', token: 'secret', key: 'key', readonly: true, created: 'created' },
-        ]);
       });
-
-      test('should delete a token created', async () => {
-        const config = new Config(
-          configExample({
-            ...getDefaultConfig(),
-            storage: generateRandomStorage(),
-          })
-        );
-        const storage = new Storage(config, logger);
-        await storage.init(config);
-        await storage.saveToken({
-          user: 'foo',
-          token: 'secret',
-          key: 'key',
-          created: 'created',
-          readonly: true,
-        });
-        const tokens = await storage.readTokens({ user: 'foo' });
-        expect(tokens).toHaveLength(1);
-        await storage.deleteToken('foo', 'key');
-        const tokens2 = await storage.readTokens({ user: 'foo' });
-        expect(tokens2).toHaveLength(0);
-      });
-    });
-  });
-
-  describe('removeTarball', () => {
-    test('should fail on remove tarball of package does not exist', async () => {
-      const username = 'foouser';
-      const config = new Config(
-        configExample({
-          ...getDefaultConfig(),
-          storage: generateRandomStorage(),
-        })
-      );
-      const storage = new Storage(config, logger);
-      await storage.init(config);
-      await expect(storage.removeTarball('foo', 'foo-1.0.0.tgz', 'rev', username)).rejects.toThrow(
-        API_ERROR.NO_PACKAGE
-      );
-    });
-  });
-
-  describe('removePackage', () => {
-    test('should remove entirely a package', async () => {
-      const username = 'foouser';
-      const config = new Config(
-        configExample({
-          ...getDefaultConfig(),
-          storage: generateRandomStorage(),
-        })
-      );
-      const req = httpMocks.createRequest({
-        method: 'GET',
-        connection: { remoteAddress: fakeHost },
-        headers: {
-          host: fakeHost,
-          [HEADERS.FORWARDED_PROTO]: 'http',
-        },
-        url: '/',
-      });
-      const storage = new Storage(config, logger);
-      await storage.init(config);
-
-      const manifest = generatePackageMetadata('foo');
-      const ac = new AbortController();
-      // 1. publish a package
-      await storage.updateManifest(manifest, {
-        signal: ac.signal,
-        name: 'foo',
-        uplinksLook: false,
-        requestOptions: {
-          headers: req.headers as any,
-          protocol: req.protocol,
-          host: req.get('host') as string,
-        },
-      });
-      // 2. request package (should be available in the local cache)
-      const manifest1 = (await storage.getPackageByOptions({
-        name: 'foo',
-        uplinksLook: false,
-        requestOptions: {
-          headers: req.headers as any,
-          protocol: req.protocol,
-          host: req.get('host') as string,
-        },
-      })) as Manifest;
-      const _rev = manifest1._rev;
-      // 3. remove the tarball
-      await expect(
-        storage.removeTarball(manifest1.name, 'foo-1.0.0.tgz', _rev, username)
-      ).resolves.toBeDefined();
-      // 4. remove the package
-      await storage.removePackage(manifest1.name, _rev, username);
-      // 5. fails if package does not exist anymore in storage
-      await expect(
-        storage.getPackageByOptions({
-          name: 'foo',
-          uplinksLook: false,
-          requestOptions: {
-            headers: req.headers as any,
-            protocol: req.protocol,
-            host: req.get('host') as string,
-          },
-        })
-      ).rejects.toThrow('package does not exist on uplink: foo');
-    });
-
-    test('ok to remove package as non-owner without check', async () => {
-      const config = getConfig('publishWithOwnerDefault.yaml');
-      const storage = new Storage(config, logger);
-      await storage.init(config);
-      const owner = 'fooUser';
-      const options = { ...defaultRequestOptions, username: owner };
-
-      // 1. publish a package
-      const bodyNewManifest = generatePackageMetadata('foo', '1.0.0');
-      await storage.updateManifest(bodyNewManifest, {
-        signal: new AbortController().signal,
-        name: 'foo',
-        uplinksLook: true,
-        requestOptions: options,
-      });
-      // 2. request package (should be available in the local cache)
-      const manifest1 = (await storage.getPackageByOptions({
-        name: 'foo',
-        uplinksLook: false,
-        requestOptions: options,
-      })) as Manifest;
-      const _rev = manifest1._rev;
-      // 3. remove the tarball as other user
-      const nonOwner = 'barUser';
-      await expect(
-        storage.removeTarball(manifest1.name, 'foo-1.0.0.tgz', _rev, nonOwner)
-      ).resolves.toBeDefined();
-      // 4. remove the package as other user
-      await storage.removePackage(manifest1.name, _rev, nonOwner);
-      // 5. fails if package does not exist anymore in storage
-      await expect(
-        storage.getPackageByOptions({
-          name: 'foo',
-          uplinksLook: false,
-          requestOptions: options,
-        })
-      ).rejects.toThrow('package does not exist on uplink: foo');
-    });
-
-    test('should fail as non-owner with check', async () => {
-      const config = getConfig('publishWithOwnerAndCheck.yaml');
-      const storage = new Storage(config, logger);
-      await storage.init(config);
-      const owner = 'fooUser';
-      const options = { ...defaultRequestOptions, username: owner };
-
-      // 1. publish a package
-      const bodyNewManifest = generatePackageMetadata('foo', '1.0.0');
-      await storage.updateManifest(bodyNewManifest, {
-        signal: new AbortController().signal,
-        name: 'foo',
-        uplinksLook: true,
-        requestOptions: options,
-      });
-      // 2. request package (should be available in the local cache)
-      const manifest1 = (await storage.getPackageByOptions({
-        name: 'foo',
-        uplinksLook: false,
-        requestOptions: options,
-      })) as Manifest;
-      const _rev = manifest1._rev;
-      // 3. try removing the tarball
-      const nonOwner = 'barUser';
-      await expect(
-        storage.removeTarball(manifest1.name, 'foo-1.0.0.tgz', _rev, nonOwner)
-      ).rejects.toThrow();
-      // 4. try removing the package
-      await expect(storage.removePackage(manifest1.name, _rev, nonOwner)).rejects.toThrow();
     });
   });
 

packages/store/test/storage.sync.spec.ts

@@ -0,0 +1,242 @@
+import nock from 'nock';
+
+import { Config, getDefaultConfig } from '@verdaccio/config';
+import { API_ERROR, DIST_TAGS } from '@verdaccio/core';
+import { setup } from '@verdaccio/logger';
+import {
+  generateLocalPackageMetadata,
+  generatePackageMetadata,
+  generateRemotePackageMetadata,
+} from '@verdaccio/test-helper';
+import { Manifest } from '@verdaccio/types';
+
+import { Storage } from '../src';
+import manifestFooRemoteNpmjs from './fixtures/manifests/foo-npmjs.json';
+import { configExample, generateRandomStorage } from './helpers';
+
+setup({ type: 'stdout', format: 'pretty', level: 'trace' });
+
+const fooManifest = generatePackageMetadata('foo', '1.0.0');
+
+describe('storage', () => {
+  beforeEach(() => {
+    nock.cleanAll();
+    nock.abortPendingRequests();
+    jest.clearAllMocks();
+  });
+
+  describe('syncUplinksMetadata()', () => {
+    describe('error handling', () => {
+      test('should handle double failure on uplinks with timeout', async () => {
+        const fooManifest = generatePackageMetadata('timeout', '8.0.0');
+        nock('https://registry.timeout.com')
+          .get(`/${fooManifest.name}`)
+          .delayConnection(8000)
+          .reply(201, manifestFooRemoteNpmjs);
+
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncDoubleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+
+        const storage = new Storage(config);
+        await storage.init(config);
+        await expect(
+          storage.syncUplinksMetadata(fooManifest.name, null, {
+            retry: { limit: 3 },
+            timeout: {
+              request: 1000,
+            },
+          })
+        ).rejects.toThrow(API_ERROR.NO_PACKAGE);
+      }, 18000);
+
+      test('should handle one proxy fails', async () => {
+        const fooManifest = generatePackageMetadata('foo', '8.0.0');
+        nock('https://registry.verdaccio.org').get('/foo').replyWithError('service in holidays');
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+        await expect(
+          storage.syncUplinksMetadata(fooManifest.name, null, {
+            retry: { limit: 0 },
+          })
+        ).rejects.toThrow(API_ERROR.NO_PACKAGE);
+      });
+
+      test('should handle one proxy reply 304', async () => {
+        const fooManifest = generatePackageMetadata('foo-no-data', '8.0.0');
+        nock('https://registry.verdaccio.org').get('/foo-no-data').reply(304);
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+        const [manifest] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
+          retry: { limit: 0 },
+        });
+        expect(manifest).toBe(fooManifest);
+      });
+    });
+
+    describe('success scenarios', () => {
+      test('should handle one proxy success', async () => {
+        const fooManifest = generateLocalPackageMetadata('foo', '8.0.0');
+        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+
+        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
+        expect(response).not.toBeNull();
+        expect((response as Manifest).name).toEqual(fooManifest.name);
+        expect(Object.keys((response as Manifest).versions)).toEqual([
+          '8.0.0',
+          '1.0.0',
+          '0.0.3',
+          '0.0.4',
+          '0.0.5',
+          '0.0.6',
+          '0.0.7',
+        ]);
+        expect(Object.keys((response as Manifest).time)).toEqual([
+          'modified',
+          'created',
+          '8.0.0',
+          '1.0.0',
+          '0.0.3',
+          '0.0.4',
+          '0.0.5',
+          '0.0.6',
+          '0.0.7',
+        ]);
+        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
+        expect((response as Manifest).time['8.0.0']).toBeDefined();
+      });
+
+      test('should handle one proxy success with no local cache manifest', async () => {
+        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+
+        const [response] = await storage.syncUplinksMetadata(fooManifest.name, null);
+        // the latest from the remote manifest
+        expect(response).not.toBeNull();
+        expect((response as Manifest).name).toEqual(fooManifest.name);
+        expect((response as Manifest)[DIST_TAGS].latest).toEqual('0.0.7');
+      });
+
+      test('should handle no proxy found with local cache manifest', async () => {
+        const fooManifest = generatePackageMetadata('foo', '8.0.0');
+        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncNoUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+
+        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest);
+        expect(response).not.toBeNull();
+        expect((response as Manifest).name).toEqual(fooManifest.name);
+        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
+      });
+      test.todo('should handle double proxy with last one success');
+    });
+
+    describe('options', () => {
+      test('should handle disable uplinks via options.uplinksLook=false with cache', async () => {
+        const fooManifest = generatePackageMetadata('foo', '8.0.0');
+        nock('https://registry.verdaccio.org').get('/foo').reply(201, manifestFooRemoteNpmjs);
+        const config = new Config(
+          configExample(
+            {
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+
+        const [response] = await storage.syncUplinksMetadata(fooManifest.name, fooManifest, {
+          uplinksLook: false,
+        });
+
+        expect((response as Manifest).name).toEqual(fooManifest.name);
+        expect((response as Manifest)[DIST_TAGS].latest).toEqual('8.0.0');
+      });
+
+      test('should handle disable uplinks via options.uplinksLook=false without cache', async () => {
+        const fooRemoteManifest = generateRemotePackageMetadata(
+          'foo',
+          '9.0.0',
+          'https://registry.verdaccio.org',
+          ['9.0.0', '9.0.1', '9.0.2', '9.0.3']
+        );
+        nock('https://registry.verdaccio.org').get('/foo').reply(201, fooRemoteManifest);
+        const config = new Config(
+          configExample(
+            {
+              ...getDefaultConfig(),
+              storage: generateRandomStorage(),
+            },
+            './fixtures/config/syncSingleUplinksMetadata.yaml',
+            __dirname
+          )
+        );
+        const storage = new Storage(config);
+        await storage.init(config);
+
+        const [response] = await storage.syncUplinksMetadata('foo', null, {
+          uplinksLook: true,
+        });
+
+        expect((response as Manifest).name).toEqual('foo');
+        expect((response as Manifest)[DIST_TAGS].latest).toEqual('9.0.0');
+      });
+    });
+  });
+});

packages/store/test/storage.tarball.spec.ts

@@ -0,0 +1,331 @@
+import nock from 'nock';
+import * as httpMocks from 'node-mocks-http';
+import path from 'path';
+
+import { Config, getDefaultConfig } from '@verdaccio/config';
+import { API_ERROR, HEADERS, HEADER_TYPE, errorUtils } from '@verdaccio/core';
+import { setup } from '@verdaccio/logger';
+import {
+  addNewVersion,
+  generatePackageMetadata,
+  generateRemotePackageMetadata,
+} from '@verdaccio/test-helper';
+import { Manifest } from '@verdaccio/types';
+
+import { Storage } from '../src';
+import { configExample, defaultRequestOptions, generateRandomStorage } from './helpers';
+
+setup({ type: 'stdout', format: 'pretty', level: 'trace' });
+
+const fakeHost = 'localhost:4873';
+
+describe('storage', () => {
+  beforeEach(() => {
+    nock.cleanAll();
+    nock.abortPendingRequests();
+    jest.clearAllMocks();
+  });
+
+  describe('getTarball', () => {
+    test('should get a package from local storage', (done) => {
+      const pkgName = 'foo';
+      const config = new Config(
+        configExample({
+          ...getDefaultConfig(),
+          storage: generateRandomStorage(),
+        })
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const ac = new AbortController();
+        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
+        storage
+          .updateManifest(bodyNewManifest, {
+            signal: ac.signal,
+            name: pkgName,
+            uplinksLook: false,
+            requestOptions: defaultRequestOptions,
+          })
+          .then(() => {
+            const abort = new AbortController();
+            storage
+              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
+                signal: abort.signal,
+              })
+              .then((stream) => {
+                stream.on('data', (dat) => {
+                  expect(dat).toBeDefined();
+                  expect(dat.length).toEqual(512);
+                });
+                stream.on('end', () => {
+                  done();
+                });
+                stream.on('error', () => {
+                  done('this should not happen');
+                });
+              });
+          });
+      });
+    });
+
+    test('should not found a package anywhere', (done) => {
+      const config = new Config(
+        configExample({
+          ...getDefaultConfig(),
+          storage: generateRandomStorage(),
+        })
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const abort = new AbortController();
+        storage
+          .getTarball('some-tarball', 'some-tarball-1.0.0.tgz', {
+            signal: abort.signal,
+          })
+          .then((stream) => {
+            stream.on('error', (err) => {
+              expect(err).toEqual(errorUtils.getNotFound(API_ERROR.NO_PACKAGE));
+              done();
+            });
+          });
+      });
+    });
+
+    test('should create a package if tarball is requested and does not exist locally', (done) => {
+      const pkgName = 'upstream';
+      const upstreamManifest = generateRemotePackageMetadata(
+        pkgName,
+        '1.0.0',
+        'https://registry.something.org'
+      );
+      nock('https://registry.verdaccio.org').get(`/${pkgName}`).reply(201, upstreamManifest);
+      nock('https://registry.something.org')
+        .get(`/${pkgName}/-/${pkgName}-1.0.0.tgz`)
+        // types does not match here with documentation
+        // @ts-expect-error
+        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
+          [HEADER_TYPE.CONTENT_LENGTH]: 277,
+        });
+      const config = new Config(
+        configExample(
+          {
+            storage: generateRandomStorage(),
+          },
+          './fixtures/config/getTarball-getupstream.yaml',
+          __dirname
+        )
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const abort = new AbortController();
+        storage
+          .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
+            signal: abort.signal,
+          })
+          .then((stream) => {
+            stream.on('data', (dat) => {
+              expect(dat).toBeDefined();
+            });
+            stream.on('end', () => {
+              done();
+            });
+            stream.on('error', () => {
+              done('this should not happen');
+            });
+          });
+      });
+    });
+
+    test('should serve fetch tarball from upstream without dist info local', (done) => {
+      const pkgName = 'upstream';
+      const upstreamManifest = addNewVersion(
+        generateRemotePackageMetadata(pkgName, '1.0.0') as Manifest,
+        '1.0.1'
+      );
+      nock('https://registry.verdaccio.org').get(`/${pkgName}`).reply(201, upstreamManifest);
+      nock('http://localhost:5555')
+        .get(`/${pkgName}/-/${pkgName}-1.0.1.tgz`)
+        // types does not match here with documentation
+        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
+          [HEADER_TYPE.CONTENT_LENGTH]: 277,
+        });
+      const config = new Config(
+        configExample(
+          {
+            storage: generateRandomStorage(),
+          },
+          './fixtures/config/getTarball-getupstream.yaml',
+          __dirname
+        )
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const ac = new AbortController();
+        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
+        storage
+          .updateManifest(bodyNewManifest, {
+            signal: ac.signal,
+            name: pkgName,
+            uplinksLook: true,
+            revision: '1',
+            requestOptions: {
+              host: 'localhost',
+              protocol: 'http',
+              headers: {},
+            },
+          })
+          .then(() => {
+            const abort = new AbortController();
+            storage
+              .getTarball(pkgName, `${pkgName}-1.0.1.tgz`, {
+                signal: abort.signal,
+              })
+              .then((stream) => {
+                stream.on('data', (dat) => {
+                  expect(dat).toBeDefined();
+                });
+                stream.on('end', () => {
+                  done();
+                });
+                stream.on('error', () => {
+                  done('this should not happen');
+                });
+              });
+          });
+      });
+    });
+
+    test('should serve fetch tarball from upstream without with info local', (done) => {
+      const pkgName = 'upstream';
+      const upstreamManifest = addNewVersion(
+        addNewVersion(generateRemotePackageMetadata(pkgName, '1.0.0') as Manifest, '1.0.1'),
+        '1.0.2'
+      );
+      nock('https://registry.verdaccio.org')
+        .get(`/${pkgName}`)
+        .times(10)
+        .reply(201, upstreamManifest);
+      nock('http://localhost:5555')
+        .get(`/${pkgName}/-/${pkgName}-1.0.0.tgz`)
+        // types does not match here with documentation
+        .replyWithFile(201, path.join(__dirname, 'fixtures/tarball.tgz'), {
+          [HEADER_TYPE.CONTENT_LENGTH]: 277,
+        });
+      const storagePath = generateRandomStorage();
+      const config = new Config(
+        configExample(
+          {
+            storage: storagePath,
+          },
+          './fixtures/config/getTarball-getupstream.yaml',
+          __dirname
+        )
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const req = httpMocks.createRequest({
+          method: 'GET',
+          connection: { remoteAddress: fakeHost },
+          headers: {
+            host: fakeHost,
+            [HEADERS.FORWARDED_PROTO]: 'http',
+          },
+          url: '/',
+        });
+        return storage
+          .getPackageByOptions({
+            name: pkgName,
+            uplinksLook: true,
+            requestOptions: {
+              headers: req.headers as any,
+              protocol: req.protocol,
+              host: req.get('host') as string,
+            },
+          })
+          .then(() => {
+            const abort = new AbortController();
+            storage
+              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
+                signal: abort.signal,
+              })
+              .then((stream) => {
+                stream.on('data', (dat) => {
+                  expect(dat).toBeDefined();
+                });
+                stream.on('end', () => {
+                  done();
+                });
+                stream.once('error', () => {
+                  done('this should not happen');
+                });
+              });
+          });
+      });
+    });
+
+    test('should serve local cache', (done) => {
+      const pkgName = 'upstream';
+      const config = new Config(
+        configExample(
+          {
+            storage: generateRandomStorage(),
+          },
+          './fixtures/config/getTarball-getupstream.yaml',
+          __dirname
+        )
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const ac = new AbortController();
+        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
+        storage
+          .updateManifest(bodyNewManifest, {
+            signal: ac.signal,
+            name: pkgName,
+            uplinksLook: true,
+            revision: '1',
+            requestOptions: {
+              host: 'localhost',
+              protocol: 'http',
+              headers: {},
+            },
+          })
+          .then(() => {
+            const abort = new AbortController();
+            storage
+              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
+                signal: abort.signal,
+              })
+              .then((stream) => {
+                stream.on('data', (dat) => {
+                  expect(dat).toBeDefined();
+                });
+                stream.on('end', () => {
+                  done();
+                });
+                stream.on('error', () => {
+                  done('this should not happen');
+                });
+              });
+          });
+      });
+    });
+  });
+
+  describe('removeTarball', () => {
+    test('should fail on remove tarball of package does not exist', async () => {
+      const username = 'foouser';
+      const config = new Config(
+        configExample({
+          ...getDefaultConfig(),
+          storage: generateRandomStorage(),
+        })
+      );
+      const storage = new Storage(config);
+      await storage.init(config);
+      await expect(storage.removeTarball('foo', 'foo-1.0.0.tgz', 'rev', username)).rejects.toThrow(
+        API_ERROR.NO_PACKAGE
+      );
+    });
+  });
+});

packages/store/tsconfig.json

@@ -2,7 +2,8 @@
   "extends": "../../tsconfig.reference.json",
   "compilerOptions": {
     "rootDir": "./src",
-    "outDir": "./build"
+    "outDir": "./build",
+    "resolveJsonModule": true
   },
   "include": ["src/**/*"],
   "exclude": ["src/**/*.test.ts"],

packages/verdaccio/test/disabled_test/uplinks/cache.ts

@@ -19,7 +19,6 @@ const PKG_GH1312 = 'pkg-gh1312';
 
 function isCached(pkgName, tarballName) {
   const pathCached = path.join(__dirname, STORAGE, pkgName, tarballName);
-  console.log('isCached =>', pathCached);
 
   return fs.existsSync(pathCached);
 }