Merge pull request #1161 from mlucool/filters

Filter packages

package.json

@@ -51,7 +51,7 @@
     "@commitlint/config-conventional": "7.5.0",
     "@octokit/rest": "16.25.0",
     "@verdaccio/babel-preset": "0.1.0",
-    "@verdaccio/types": "5.0.0-beta.4",
+    "@verdaccio/types": "5.0.2",
     "codecov": "3.3.0",
     "cross-env": "5.2.0",
     "eslint": "5.16.0",

src/api/index.js

@@ -20,7 +20,7 @@ import web from './web';
 
 import type { $Application } from 'express';
 import type { $ResponseExtend, $RequestExtend, $NextFunctionVer, IStorageHandler, IAuth } from '../../types';
-import type { Config as IConfig, IPluginMiddleware } from '@verdaccio/types';
+import type { Config as IConfig, IPluginMiddleware, IPluginStorageFilter } from '@verdaccio/types';
 import { setup, logger } from '../lib/logger';
 import { log, final, errorReportingMiddleware } from './middleware';
 
@@ -107,8 +107,14 @@ const defineAPI = function(config: IConfig, storage: IStorageHandler) {
 export default (async function(configHash: any) {
   setup(configHash.logs);
   const config: IConfig = new AppConfig(_.cloneDeep(configHash));
+  // register middleware plugins
+  const plugin_params = {
+    config: config,
+    logger: logger,
+  };
+  const filters = loadPlugin(config, config.filters || {}, plugin_params, (plugin: IPluginStorageFilter) => plugin.filter_metadata);
   const storage: IStorageHandler = new Storage(config);
   // waits until init calls have been initialized
-  await storage.init(config);
+  await storage.init(config, filters);
   return defineAPI(config, storage);
 });

src/lib/storage.js

@@ -17,7 +17,7 @@ import { setupUpLinks, updateVersionsHiddenUpLink } from './uplink-util';
 import { mergeVersions } from './metadata-utils';
 import { ErrorCode, normalizeDistTags, validateMetadata, isObject } from './utils';
 import type { IStorage, IProxy, IStorageHandler, ProxyList, StringValue, IGetPackageOptions, ISyncUplinks } from '../../types';
-import type { Versions, Package, Config, MergeTags, Version, DistFile, Callback, Logger } from '@verdaccio/types';
+import type { Versions, Package, Config, MergeTags, Version, DistFile, Callback, Logger, IPluginStorageFilter } from '@verdaccio/types';
 import type { IReadTarball, IUploadTarball } from '@verdaccio/streams';
 import { hasProxyTo } from './config-utils';
 import { logger } from '../lib/logger';
@@ -27,6 +27,7 @@ class Storage implements IStorageHandler {
   config: Config;
   logger: Logger;
   uplinks: ProxyList;
+  filters: Array<IPluginStorageFilter>;
 
   constructor(config: Config) {
     this.config = config;
@@ -34,7 +35,8 @@ class Storage implements IStorageHandler {
     this.logger = logger.child();
   }
 
-  init(config: Config) {
+  init(config: Config, filters: Array<IPluginStorageFilter> = []) {
+    this.filters = filters;
     this.localStorage = new LocalStorage(this.config, logger);
 
     return this.localStorage.getSecret(config);
@@ -503,11 +505,24 @@ class Storage implements IStorageHandler {
           return callback(null, packageInfo);
         }
 
-        self.localStorage.updateVersions(name, packageInfo, function(err, packageJsonLocal: Package) {
+        self.localStorage.updateVersions(name, packageInfo, async (err, packageJsonLocal: Package) => {
           if (err) {
             return callback(err);
           }
-          return callback(null, packageJsonLocal, upLinksErrors);
+          // Any error here will cause a 404, like an uplink error. This is likely the right thing to do
+          // as a broken filter is a security risk.
+          const filterErrors = [];
+          // This MUST be done serially and not in parallel as they modify packageJsonLocal
+          for (const filter of self.filters) {
+            try {
+              // These filters can assume it's save to modify packageJsonLocal and return it directly for
+              // performance (i.e. need not be pure)
+              packageJsonLocal = await filter.filter_metadata(packageJsonLocal);
+            } catch (err) {
+              filterErrors.push(err);
+            }
+          }
+          callback(null, packageJsonLocal, _.concat(upLinksErrors, filterErrors));
         });
       }
     );

test/unit/api/api.spec.js

@@ -42,6 +42,12 @@ describe('endpoint unit test', () => {
             file: './test-storage-api-spec/.htpasswd'
           }
         },
+        filters: {
+          '../partials/plugin/filter': {
+            pkg: 'npm_test',
+            version: '2.0.0'
+          }
+        },
         storage: store,
         self_path: store,
         uplinks: {
@@ -384,6 +390,37 @@ describe('endpoint unit test', () => {
           });
       });
 
+      test('be able to filter packages', (done) => {
+        request(app)
+          .get('/npm_test')
+          .set(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HTTP_STATUS.OK)
+          .end(function(err, res) {
+            if (err) {
+              return done(err);
+            }
+            // Filter out 2.0.0
+            expect(Object.keys(res.body.versions)).toEqual(['1.0.0']);
+            done();
+          });
+      });
+
+      test('should not found when a filter fails', (done) => {
+        request(app)
+           // Filter errors look like other uplink errors
+          .get('/trigger-filter-failure')
+          .set(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HEADER_TYPE.CONTENT_TYPE, HEADERS.JSON_CHARSET)
+          .expect(HTTP_STATUS.NOT_FOUND)
+          .end(function(err, res) {
+            if (err) {
+              return done(err);
+            }
+            done();
+          });
+      });
+
       test('should forbid access to remote package', (done) => {
 
         request(app)

test/unit/partials/mock-store/npm_test/package.json

@@ -0,0 +1,87 @@
+{
+  "_id": "npm_test",
+  "name": "npm_test",
+  "description": "",
+  "dist-tags": {
+    "latest": "1.0.0"
+  },
+  "versions": {
+    "1.0.0": {
+      "name": "npm_test",
+      "version": "1.0.0",
+      "description": "",
+      "main": "index.js",
+      "dependencies": {
+        "test": "^1.4.0"
+      },
+      "devDependencies": {
+
+      },
+      "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+      },
+      "keywords": [
+
+      ],
+      "author": "",
+      "license": "ISC",
+      "readme": "ERROR: No README data found!",
+      "_id": "npm_test@1.0.0",
+      "_npmVersion": "5.5.1",
+      "_nodeVersion": "9.3.0",
+      "_npmUser": {
+
+      },
+      "dist": {
+        "integrity": "sha512-tfzM1OFjWwg2d2Wke\/DV6icjeTZUVOZYLkbf8wmONRSAgMovL\/F+zyI24OhTtWyOXd1Kbj2YUMBvLpmpAjv8zg==",
+        "shasum": "3e4e6bd5097b295e520b947c9be3259a9509a673",
+        "tarball": "http:\/\/localhost:4873\/npm_test\/-\/npm_test-1.0.0.tgz"
+      }
+    },
+    "2.0.0": {
+      "name": "npm_test",
+      "version": "2.0.0",
+      "description": "",
+      "main": "index.js",
+      "dependencies": {
+        "test": "^2.4.0"
+      },
+      "devDependencies": {
+
+      },
+      "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+      },
+      "keywords": [
+
+      ],
+      "author": "",
+      "license": "ISC",
+      "readme": "ERROR: No README data found!",
+      "_id": "npm_test@2.0.0",
+      "_npmVersion": "5.5.1",
+      "_nodeVersion": "9.3.0",
+      "_npmUser": {
+
+      },
+      "dist": {
+        "integrity": "sha512-tzzM1OFjWwg2d2Wke\/DV6icjeTZUVOZYLkbf8wmONRSAgMovL\/F+zyI24OhTtWyOXd1Kbj2YUMBvLpmpAjv8zg==",
+        "shasum": "3a4e6bd5097b295e520b947c9be3259a9509a673",
+        "tarball": "http:\/\/localhost:4873\/npm_test\/-\/npm_test-2.0.0.tgz"
+      }
+    }
+  },
+  "readme": "ERROR: No README data found!",
+  "_attachments": {
+    "npm_test-1.0.0.tgz": {
+      "content_type": "application\/octet-stream",
+      "data": "H4sIAAAAAAAAE+2ST08CMRDFOe+nmPTAyawt7ELCVT149ihqmu4gI9I2bUGM4bvbbhGM4eYmxmR\/l6bvtW+mf6xUK\/mMlzaP5Ys3etAxnPNJVcE5PVHV0RPjkairsZiK0YALUU+mMOBdN3KOjQ\/SxVZ+m5PPAsfxn\/BRADAt18hmwDxpY0k+BfSBXSRni86T0ckUJS95Vhv0ypENByeLa0ntjHSDu\/iPvpZajIJWhD66qRwcC6Xlj6KsYm7U94cN2+sfe7KRS34LabuMCaiWBubsxjnjZqANJAO8RUULwmbOYDgE3FEAcSqzwvc345oUd\/\/QKnITlsadzvNKCrVv7+X27ooV++Kv36qnp6enSz4B8bhKUwAIAAA=",
+      "length": 281
+    },
+    "npm_test-2.0.0.tgz": {
+      "content_type": "application\/octet-stream",
+      "data": "H4sIAAAAAAAAE+2ST08CMRDFOe+nmPTAyawt7ELCVT149ihqmu4gI9I2bUGM4bvbbhGM4eYmxmR\/l6bvtW+mf6xUK\/mMlzaP5Ys3etAxnPNJVcE5PVHV0RPjkairsZiK0YALUU+mMOBdN3KOjQ\/SxVZ+m5PPAsfxn\/BRADAt18hmwDxpY0k+BfSBXSRni86T0ckUJS95Vhv0ypENByeLa0ntjHSDu\/iPvpZajIJWhD66qRwcC6Xlj6KsYm7U94cN2+sfe7KRS34LabuMCaiWBubsxjnjZqANJAO8RUULwmbOYDgE3FEAcSqzwvc345oUd\/\/QKnITlsadzvNKCrVv7+X27ooV++Kv36qnp6enSz4B8bhKUwAIAAA=",
+      "length": 281
+    }
+  }
+}

test/unit/partials/plugin/filter.js

@@ -0,0 +1,24 @@
+class FilterPlugin {
+  constructor(config) {
+    this._config = config;
+  }
+
+  filter_metadata(pkg) {
+    return new Promise((resolve) => {
+        // We use this to test what happens when a filter rejects
+        if(pkg.name === 'trigger-filter-failure') {
+          reject(new Error('Example filter failure'));
+          return;
+        }
+        // Example filter that removes a single blocked package
+        if (this._config.pkg === pkg.name) {
+          // In reality, we also want to remove references in attachments and dist-tags, etc. This is just a POC
+          delete pkg.versions[this._config.version];
+        }
+        resolve(pkg);
+      }
+    );
+  }
+}
+
+exports.default = FilterPlugin;