feat: plugin support to filter packages

Add a plugin that can filter all package metadata before being returned.
This enables blocking of packages from verdaccio.

IPluginStorageFilter are loaded like other plugins from the config.
Verdaccio will look for plugins in config.filters and pass this to
storage.init. This is the same design as other plugins and will be
dynamically found with the same rules. These plugins must impliment
a filter_metadata method, which is called serially (in the order
loaded from the config) for every metadata request. It gets a current
copy of a package metadata and may choose to modify it as required.
For example, this may be used to block a bad version of a package or
add a time delay from when new packages can be used from your
registry. Errors in a filter will cause a 404, similar to upLinkErrors
as it is not safe to recover gracefully from them. Note: When version
is removed, be careful about updating tags.

Fixes: #818

package.json

@@ -51,7 +51,7 @@
     "@commitlint/config-conventional": "7.5.0",
     "@octokit/rest": "16.25.0",
     "@verdaccio/babel-preset": "0.1.0",
-    "@verdaccio/types": "5.0.0-beta.4",
+    "@verdaccio/types": "5.0.2",
     "codecov": "3.3.0",
     "cross-env": "5.2.0",
     "eslint": "5.16.0",

src/api/index.js

@@ -20,7 +20,7 @@ import web from './web';
 
 import type { $Application } from 'express';
 import type { $ResponseExtend, $RequestExtend, $NextFunctionVer, IStorageHandler, IAuth } from '../../types';
-import type { Config as IConfig, IPluginMiddleware } from '@verdaccio/types';
+import type { Config as IConfig, IPluginMiddleware, IPluginStorageFilter } from '@verdaccio/types';
 import { setup, logger } from '../lib/logger';
 import { log, final, errorReportingMiddleware } from './middleware';
 
@@ -107,8 +107,14 @@ const defineAPI = function(config: IConfig, storage: IStorageHandler) {
 export default (async function(configHash: any) {
   setup(configHash.logs);
   const config: IConfig = new AppConfig(_.cloneDeep(configHash));
+  // register middleware plugins
+  const plugin_params = {
+    config: config,
+    logger: logger,
+  };
+  const filters = loadPlugin(config, config.filters || {}, plugin_params, (plugin: IPluginStorageFilter) => plugin.filter_metadata);
   const storage: IStorageHandler = new Storage(config);
   // waits until init calls have been initialized
-  await storage.init(config);
+  await storage.init(config, filters);
   return defineAPI(config, storage);
 });

src/lib/storage.js

@@ -17,7 +17,7 @@ import { setupUpLinks, updateVersionsHiddenUpLink } from './uplink-util';
 import { mergeVersions } from './metadata-utils';
 import { ErrorCode, normalizeDistTags, validateMetadata, isObject } from './utils';
 import type { IStorage, IProxy, IStorageHandler, ProxyList, StringValue, IGetPackageOptions, ISyncUplinks } from '../../types';
-import type { Versions, Package, Config, MergeTags, Version, DistFile, Callback, Logger } from '@verdaccio/types';
+import type { Versions, Package, Config, MergeTags, Version, DistFile, Callback, Logger, IPluginStorageFilter } from '@verdaccio/types';
 import type { IReadTarball, IUploadTarball } from '@verdaccio/streams';
 import { hasProxyTo } from './config-utils';
 import { logger } from '../lib/logger';
@@ -27,6 +27,7 @@ class Storage implements IStorageHandler {
   config: Config;
   logger: Logger;
   uplinks: ProxyList;
+  filters: Array<IPluginStorageFilter>;
 
   constructor(config: Config) {
     this.config = config;
@@ -34,7 +35,8 @@ class Storage implements IStorageHandler {
     this.logger = logger.child();
   }
 
-  init(config: Config) {
+  init(config: Config, filters: Array<IPluginStorageFilter> = []) {
+    this.filters = filters;
     this.localStorage = new LocalStorage(this.config, logger);
 
     return this.localStorage.getSecret(config);
@@ -503,11 +505,24 @@ class Storage implements IStorageHandler {
           return callback(null, packageInfo);
         }
 
-        self.localStorage.updateVersions(name, packageInfo, function(err, packageJsonLocal: Package) {
+        self.localStorage.updateVersions(name, packageInfo, async (err, packageJsonLocal: Package) => {
           if (err) {
             return callback(err);
           }
-          return callback(null, packageJsonLocal, upLinksErrors);
+          // Any error here will cause a 404, like an uplink error. This is likely the right thing to do
+          // as a broken filter is a security risk.
+          const filterErrors = [];
+          // This MUST be done serially and not in parallel as they modify packageJsonLocal
+          for (const filter of self.filters) {
+            try {
+              // These filters can assume it's save to modify packageJsonLocal and return it directly for
+              // performance (i.e. need not be pure)
+              packageJsonLocal = await filter.filter_metadata(packageJsonLocal);
+            } catch (err) {
+              filterErrors.push(err);
+            }
+          }
+          callback(null, packageJsonLocal, _.concat(upLinksErrors, filterErrors));
         });
       }
     );

test/unit/api/api.spec.js

@@ -42,6 +42,12 @@ describe('endpoint unit test', () => {
             file: './test-storage-api-spec/.htpasswd'
           }
         },
+        filters: {
+          '../partials/plugin/filter': {
+            pkg: 'npm_test',
+            version: '2.0.0'
+          }
+        },
         storage: store,
         self_path: store,
         uplinks: {
@@ -384,6 +390,37 @@ describe('endpoint unit test', () => {
           });
       });
 
+      test('be able to filter packages', (done) => {
+        request(app)
+          .get('/npm_test')
+          .set(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HTTP_STATUS.OK)
+          .end(function(err, res) {
+            if (err) {
+              return done(err);
+            }
+            // Filter out 2.0.0
+            expect(Object.keys(res.body.versions)).toEqual(['1.0.0']);
+            done();
+          });
+      });
+
+      test('should not found when a filter fails', (done) => {
+        request(app)
+           // Filter errors look like other uplink errors
+          .get('/trigger-filter-failure')
+          .set(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HTTP_STATUS.NOT_FOUND)
+          .end(function(err, res) {
+            if (err) {
+              return done(err);
+            }
+            done();
+          });
+      });
+
       test('should forbid access to remote package', (done) => {
 
         request(app)

test/unit/partials/mock-store/npm_test/package.json

@@ -0,0 +1,87 @@
+{
+  "_id": "npm_test",
+  "name": "npm_test",
+  "description": "",
+  "dist-tags": {
+    "latest": "1.0.0"
+  },
+  "versions": {
+    "1.0.0": {
+      "name": "npm_test",
+      "version": "1.0.0",
+      "description": "",
+      "main": "index.js",
+      "dependencies": {
+        "test": "^1.4.0"
+      },
+      "devDependencies": {
+
+      },
+      "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+      },
+      "keywords": [
+
+      ],
+      "author": "",
+      "license": "ISC",
+      "readme": "ERROR: No README data found!",
+      "_id": "npm_test@1.0.0",
+      "_npmVersion": "5.5.1",
+      "_nodeVersion": "9.3.0",
+      "_npmUser": {
+
+      },
+      "dist": {
+        "integrity": "sha512-tfzM1OFjWwg2d2Wke\/DV6icjeTZUVOZYLkbf8wmONRSAgMovL\/F+zyI24OhTtWyOXd1Kbj2YUMBvLpmpAjv8zg==",
+        "shasum": "3e4e6bd5097b295e520b947c9be3259a9509a673",
+        "tarball": "http:\/\/localhost:4873\/npm_test\/-\/npm_test-1.0.0.tgz"
+      }
+    },
+    "2.0.0": {
+      "name": "npm_test",
+      "version": "2.0.0",
+      "description": "",
+      "main": "index.js",
+      "dependencies": {
+        "test": "^2.4.0"
+      },
+      "devDependencies": {
+
+      },
+      "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+      },
+      "keywords": [
+
+      ],
+      "author": "",
+      "license": "ISC",
+      "readme": "ERROR: No README data found!",
+      "_id": "npm_test@2.0.0",
+      "_npmVersion": "5.5.1",
+      "_nodeVersion": "9.3.0",
+      "_npmUser": {
+
+      },
+      "dist": {
+        "integrity": "sha512-tzzM1OFjWwg2d2Wke\/DV6icjeTZUVOZYLkbf8wmONRSAgMovL\/F+zyI24OhTtWyOXd1Kbj2YUMBvLpmpAjv8zg==",
+        "shasum": "3a4e6bd5097b295e520b947c9be3259a9509a673",
+        "tarball": "http:\/\/localhost:4873\/npm_test\/-\/npm_test-2.0.0.tgz"
+      }
+    }
+  },
+  "readme": "ERROR: No README data found!",
+  "_attachments": {
+    "npm_test-1.0.0.tgz": {
+      "content_type": "application\/octet-stream",
+      "data": "H4sIAAAAAAAAE+2ST08CMRDFOe+nmPTAyawt7ELCVT149ihqmu4gI9I2bUGM4bvbbhGM4eYmxmR\/l6bvtW+mf6xUK\/mMlzaP5Ys3etAxnPNJVcE5PVHV0RPjkairsZiK0YALUU+mMOBdN3KOjQ\/SxVZ+m5PPAsfxn\/BRADAt18hmwDxpY0k+BfSBXSRni86T0ckUJS95Vhv0ypENByeLa0ntjHSDu\/iPvpZajIJWhD66qRwcC6Xlj6KsYm7U94cN2+sfe7KRS34LabuMCaiWBubsxjnjZqANJAO8RUULwmbOYDgE3FEAcSqzwvc345oUd\/\/QKnITlsadzvNKCrVv7+X27ooV++Kv36qnp6enSz4B8bhKUwAIAAA=",
+      "length": 281
+    },
+    "npm_test-2.0.0.tgz": {
+      "content_type": "application\/octet-stream",
+      "data": "H4sIAAAAAAAAE+2ST08CMRDFOe+nmPTAyawt7ELCVT149ihqmu4gI9I2bUGM4bvbbhGM4eYmxmR\/l6bvtW+mf6xUK\/mMlzaP5Ys3etAxnPNJVcE5PVHV0RPjkairsZiK0YALUU+mMOBdN3KOjQ\/SxVZ+m5PPAsfxn\/BRADAt18hmwDxpY0k+BfSBXSRni86T0ckUJS95Vhv0ypENByeLa0ntjHSDu\/iPvpZajIJWhD66qRwcC6Xlj6KsYm7U94cN2+sfe7KRS34LabuMCaiWBubsxjnjZqANJAO8RUULwmbOYDgE3FEAcSqzwvc345oUd\/\/QKnITlsadzvNKCrVv7+X27ooV++Kv36qnp6enSz4B8bhKUwAIAAA=",
+      "length": 281
+    }
+  }
+}

test/unit/partials/plugin/filter.js

@@ -0,0 +1,24 @@
+class FilterPlugin {
+  constructor(config) {
+    this._config = config;
+  }
+
+  filter_metadata(pkg) {
+    return new Promise((resolve) => {
+        // We use this to test what happens when a filter rejects
+        if(pkg.name === 'trigger-filter-failure') {
+          reject(new Error('Example filter failure'));
+          return;
+        }
+        // Example filter that removes a single blocked package
+        if (this._config.pkg === pkg.name) {
+          // In reality, we also want to remove references in attachments and dist-tags, etc. This is just a POC
+          delete pkg.versions[this._config.version];
+        }
+        resolve(pkg);
+      }
+    );
+  }
+}
+
+exports.default = FilterPlugin;