Merge pull request #306 from verdaccio/fix_possible_data_loss_upstream

Fix possible data loss upstream

README.md

@@ -74,6 +74,8 @@ $ npm set ca null
 
 Now you can navigate to [http://localhost:4873/](http://localhost:4873/) where your local packages will be listed and can be searched.
 
+> Warning: Verdaccio current is not support PM2's cluster mode, run it with cluster mode may cause unknown behavior
+
 #### Beta
 
 If you are an adventurous developer you can use and install the latest beta version, this is a non stable version, I'd recommend only use for testing purporses.

package.json

@@ -78,6 +78,7 @@
     "file-loader": "0.11.2",
     "flow-runtime": "0.13.0",
     "friendly-errors-webpack-plugin": "1.6.1",
+    "fs-extra": "4.0.1",
     "github-markdown-css": "2.8.0",
     "html-webpack-plugin": "2.29.0",
     "in-publish": "2.0.0",

src/lib/storage/local/local-data.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const Path = require('path');
+const logger = require('../../logger');
 
 /**
  * Handle local database.
@@ -15,34 +16,86 @@ const Path = require('path');
    */
    constructor(path) {
     this.path = path;
+    // Prevent any write action, wait admin to check what happened during startup
+    this.locked = false;
+    this.data = this._fetchLocalPackages();
+  }
+
+  /**
+   * Fetch local packages.
+   * @private
+   * @return {Object}
+   */
+  _fetchLocalPackages() {
+    const emptyDatabase = {list: []};
+
     try {
-      this.data = JSON.parse(fs.readFileSync(this.path, 'utf8'));
-    } catch(_) {
-      this.data = {list: []};
+      const dbFile = fs.readFileSync(this.path, 'utf8');
+
+      if (!dbFile) { // readFileSync is platform specific, FreeBSD might return null
+        return emptyDatabase;
+      }
+
+      const db = this._parseDatabase(dbFile);
+
+      if(!db) {
+        return emptyDatabase;
+      }
+
+      return db;
+    } catch (err) {
+      // readFileSync is platform specific, macOS, Linux and Windows thrown an error
+      // Only recreate if file not found to prevent data loss
+      if (err.code !== 'ENOENT') {
+        this.locked = true;
+        logger.logger.error(
+          'Failed to read package database file, please check the error printed below:\n',
+          `File Path: ${this.path}\n\n`,
+          err
+        );
+      }
+      return emptyDatabase;
+    }
+  }
+
+  /**
+   * Parse the local database.
+   * @param {Object} dbFile
+   * @private
+   * @return {Object}
+   */
+  _parseDatabase(dbFile) {
+    try {
+      return JSON.parse(dbFile);
+    } catch(err) {
+      logger.logger.error(`Package database file corrupted (invalid JSON), please check the error printed below.\nFile Path: ${this.path}`, err);
+      this.locked = true;
     }
   }
 
   /**
    * Add a new element.
    * @param {*} name
+   * @return {Error|*}
    */
   add(name) {
     if (this.data.list.indexOf(name) === -1) {
       this.data.list.push(name);
-      this.sync();
+      return this.sync();
     }
   }
 
   /**
    * Remove an element from the database.
    * @param {*} name
+   * @return {Error|*}
    */
   remove(name) {
     const i = this.data.list.indexOf(name);
     if (i !== -1) {
       this.data.list.splice(i, 1);
     }
-    this.sync();
+    return this.sync();
   }
 
   /**
@@ -55,8 +108,14 @@ const Path = require('path');
 
   /**
    * Syncronize {create} database whether does not exist.
+   * @return {Error|*}
    */
   sync() {
+    if (this.locked) {
+      logger.logger.error('Database is locked, please check error message printed during startup to prevent data loss.');
+      return new Error('Verdaccio database is locked, please contact your administrator to checkout logs during verdaccio startup.');
+    }
+
     // Uses sync to prevent ugly race condition
     try {
       require('mkdirp').sync(Path.dirname(this.path));
@@ -64,7 +123,11 @@ const Path = require('path');
       // perhaps a logger instance?
       /* eslint no-empty:off */
     }
+    try {
       fs.writeFileSync(this.path, JSON.stringify(this.data));
+    } catch (err) {
+      return err;
+    }
   }
 
 }

src/lib/storage/local/local-storage.js

@@ -119,6 +119,12 @@ class LocalStorage {
       }
       this._normalizePackage(data);
 
+      let removeFailed = this.localList.remove(name);
+      if (removeFailed) {
+        // This will happen when database is locked
+        return callback(this.utils.ErrorCode.get422(removeFailed.message));
+      }
+
       storage.unlink(pkgFileName, function(err) {
         if (err) {
           return callback(err);
@@ -145,7 +151,6 @@ class LocalStorage {
         });
       });
     });
-    this.localList.remove(name);
   }
 
   /**
@@ -296,7 +301,12 @@ class LocalStorage {
 
       data.versions[version] = metadata;
       this.utils.tag_version(data, version, tag);
-      this.localList.add(name);
+
+      let addFailed = this.localList.add(name);
+      if (addFailed) {
+        return cb(this.utils.ErrorCode.get422(addFailed.message));
+      }
+
       cb();
     }, callback);
   }

test/unit/local-data.spec.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const assert = require('assert');
+const LocalData = require('../../src/lib/storage/local/local-data');
+const path = require('path');
+const _ = require('lodash');
+const fs = require('fs-extra');
+
+
+describe('Local Database', function() {
+
+  const buildCorruptedPath = () => path.join(__dirname, './partials/storage/verdaccio-corrupted.db.json');
+  const buildValidDbPath = () => path.join(__dirname, './partials/storage/verdaccio.db.json');
+
+  describe('reading database', () => {
+
+    it('should return empty database on read corrupted database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+
+      assert(_.isEmpty(dataLocal.data.list));
+    });
+
+    it('should return a database on read valid database', () => {
+      const dataLocal = new LocalData(buildValidDbPath());
+
+      assert(_.isEmpty(dataLocal.data.list) === false);
+    });
+
+    it('should fails on sync a corrupted database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      const error = dataLocal.sync();
+
+      assert(_.isError(error));
+      assert(error.message.match(/locked/));
+      assert(dataLocal.locked);
+    });
+
+  });
+
+  describe('add/remove packages to database', () => {
+    it('should add a new package to local database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      assert(_.isEmpty(dataLocal.data.list));
+      dataLocal.add('package1');
+      assert(!_.isEmpty(dataLocal.data.list));
+    });
+
+    it('should remove a new package to local database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      const pkgName = 'package1';
+
+      assert(_.isEmpty(dataLocal.data.list));
+      dataLocal.add(pkgName);
+      dataLocal.remove(pkgName);
+      assert(_.isEmpty(dataLocal.data.list));
+    });
+  });
+
+  describe('sync packages to database', () => {
+    beforeEach(function() {
+      this.newDb = path.join(__dirname, './test-storage/verdaccio.temp.db.json');
+      fs.copySync(buildValidDbPath(), this.newDb);
+    });
+
+    it('should check sync packages', function() {
+      const localData1 = new LocalData(this.newDb);
+
+      localData1.add('package1');
+
+      const localData2 = new LocalData(this.newDb);
+
+      assert(_.isEmpty(localData2.data.list) === false);
+      assert(localData2.data.list.length === 2);
+
+    });
+
+
+  });
+
+});
+

test/unit/partials/storage/verdaccio-corrupted.db.json

@@ -0,0 +1 @@
+{"list"[],"secret":"14705eeaed167749990dafa07f908d2a9504bfdb0f6ca4102eed5acba0bc9076"}

test/unit/partials/storage/verdaccio.db.json

@@ -0,0 +1 @@
+{"list":["@verdaccio/test"],"secret":"48cd053de97d4ef34aea4f1efb902334442bea1e735df5fdc9424c986a281b3d"}

wiki/cli.md

@@ -18,6 +18,8 @@ or if you use `yarn`
 $ yarn add global verdaccio
 ```
 
+> Warning: Verdaccio current is not support PM2's cluster mode, run it with cluster mode may cause unknown behavior
+
 ## Commands
 
 ```bash

yarn.lock

@@ -2908,6 +2908,14 @@ fs-access@^1.0.0:
   dependencies:
     null-check "^1.0.0"
 
+fs-extra@4.0.1:
+  version "4.0.1"
+  resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.1.tgz#7fc0c6c8957f983f57f306a24e5b9ddd8d0dd880"
+  dependencies:
+    graceful-fs "^4.1.2"
+    jsonfile "^3.0.0"
+    universalify "^0.1.0"
+
 fs-readdir-recursive@^1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.0.0.tgz#8cd1745c8b4f8a29c8caec392476921ba195f560"
@@ -3136,7 +3144,7 @@ globule@^1.0.0:
     lodash "~4.17.4"
     minimatch "~3.0.2"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.4:
+graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.4, graceful-fs@^4.1.6:
   version "4.1.11"
   resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
 
@@ -3849,6 +3857,12 @@ json5@^0.5.0, json5@^0.5.1:
   version "0.5.1"
   resolved "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821"
 
+jsonfile@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-3.0.1.tgz#a5ecc6f65f53f662c4415c7675a0331d0992ec66"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 jsonfilter@^1.1.2:
   version "1.1.2"
   resolved "https://registry.npmjs.org/jsonfilter/-/jsonfilter-1.1.2.tgz#21ef7cedc75193813c75932e96a98be205ba5a11"
@@ -6654,6 +6668,10 @@ uniqs@^2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz#ffede4b36b25290696e6e165d4a59edb998e6b02"
 
+universalify@^0.1.0:
+  version "0.1.1"
+  resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7"
+
 unix-crypt-td-js@^1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/unix-crypt-td-js/-/unix-crypt-td-js-1.0.0.tgz#1c0824150481bc7a01d49e98f1ec668d82412f3b"