Merge pull request #306 from verdaccio/fix_possible_data_loss_upstream

Fix possible data loss upstream

README.md

@@ -74,6 +74,8 @@ $ npm set ca null
 
 Now you can navigate to [http://localhost:4873/](http://localhost:4873/) where your local packages will be listed and can be searched.
 
+> Warning: Verdaccio current is not support PM2's cluster mode, run it with cluster mode may cause unknown behavior
+
 #### Beta
 
 If you are an adventurous developer you can use and install the latest beta version, this is a non stable version, I'd recommend only use for testing purporses.

package.json

@@ -78,6 +78,7 @@
     "file-loader": "0.11.2",
     "flow-runtime": "0.13.0",
     "friendly-errors-webpack-plugin": "1.6.1",
+    "fs-extra": "4.0.1",
     "github-markdown-css": "2.8.0",
     "html-webpack-plugin": "2.29.0",
     "in-publish": "2.0.0",

src/lib/storage/local/local-data.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const Path = require('path');
+const logger = require('../../logger');
 
 /**
  * Handle local database.
@@ -15,34 +16,86 @@ const Path = require('path');
    */
    constructor(path) {
     this.path = path;
+    // Prevent any write action, wait admin to check what happened during startup
+    this.locked = false;
+    this.data = this._fetchLocalPackages();
+  }
+
+  /**
+   * Fetch local packages.
+   * @private
+   * @return {Object}
+   */
+  _fetchLocalPackages() {
+    const emptyDatabase = {list: []};
+
     try {
-      this.data = JSON.parse(fs.readFileSync(this.path, 'utf8'));
-    } catch(_) {
-      this.data = {list: []};
+      const dbFile = fs.readFileSync(this.path, 'utf8');
+
+      if (!dbFile) { // readFileSync is platform specific, FreeBSD might return null
+        return emptyDatabase;
+      }
+
+      const db = this._parseDatabase(dbFile);
+
+      if(!db) {
+        return emptyDatabase;
+      }
+
+      return db;
+    } catch (err) {
+      // readFileSync is platform specific, macOS, Linux and Windows thrown an error
+      // Only recreate if file not found to prevent data loss
+      if (err.code !== 'ENOENT') {
+        this.locked = true;
+        logger.logger.error(
+          'Failed to read package database file, please check the error printed below:\n',
+          `File Path: ${this.path}\n\n`,
+          err
+        );
+      }
+      return emptyDatabase;
+    }
+  }
+
+  /**
+   * Parse the local database.
+   * @param {Object} dbFile
+   * @private
+   * @return {Object}
+   */
+  _parseDatabase(dbFile) {
+    try {
+      return JSON.parse(dbFile);
+    } catch(err) {
+      logger.logger.error(`Package database file corrupted (invalid JSON), please check the error printed below.\nFile Path: ${this.path}`, err);
+      this.locked = true;
     }
   }
 
   /**
    * Add a new element.
    * @param {*} name
+   * @return {Error|*}
    */
   add(name) {
     if (this.data.list.indexOf(name) === -1) {
       this.data.list.push(name);
-      this.sync();
+      return this.sync();
     }
   }
 
   /**
    * Remove an element from the database.
    * @param {*} name
+   * @return {Error|*}
    */
   remove(name) {
     const i = this.data.list.indexOf(name);
     if (i !== -1) {
       this.data.list.splice(i, 1);
     }
-    this.sync();
+    return this.sync();
   }
 
   /**
@@ -55,8 +108,14 @@ const Path = require('path');
 
   /**
    * Syncronize {create} database whether does not exist.
+   * @return {Error|*}
    */
   sync() {
+    if (this.locked) {
+      logger.logger.error('Database is locked, please check error message printed during startup to prevent data loss.');
+      return new Error('Verdaccio database is locked, please contact your administrator to checkout logs during verdaccio startup.');
+    }
+
     // Uses sync to prevent ugly race condition
     try {
       require('mkdirp').sync(Path.dirname(this.path));
@@ -64,7 +123,11 @@ const Path = require('path');
       // perhaps a logger instance?
       /* eslint no-empty:off */
     }
-    fs.writeFileSync(this.path, JSON.stringify(this.data));
+    try {
+      fs.writeFileSync(this.path, JSON.stringify(this.data));
+    } catch (err) {
+      return err;
+    }
   }
 
 }

src/lib/storage/local/local-storage.js

@@ -119,6 +119,12 @@ class LocalStorage {
       }
       this._normalizePackage(data);
 
+      let removeFailed = this.localList.remove(name);
+      if (removeFailed) {
+        // This will happen when database is locked
+        return callback(this.utils.ErrorCode.get422(removeFailed.message));
+      }
+
       storage.unlink(pkgFileName, function(err) {
         if (err) {
           return callback(err);
@@ -145,7 +151,6 @@ class LocalStorage {
         });
       });
     });
-    this.localList.remove(name);
   }
 
   /**
@@ -296,7 +301,12 @@ class LocalStorage {
 
       data.versions[version] = metadata;
       this.utils.tag_version(data, version, tag);
-      this.localList.add(name);
+
+      let addFailed = this.localList.add(name);
+      if (addFailed) {
+        return cb(this.utils.ErrorCode.get422(addFailed.message));
+      }
+
       cb();
     }, callback);
   }

test/unit/local-data.spec.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const assert = require('assert');
+const LocalData = require('../../src/lib/storage/local/local-data');
+const path = require('path');
+const _ = require('lodash');
+const fs = require('fs-extra');
+
+
+describe('Local Database', function() {
+
+  const buildCorruptedPath = () => path.join(__dirname, './partials/storage/verdaccio-corrupted.db.json');
+  const buildValidDbPath = () => path.join(__dirname, './partials/storage/verdaccio.db.json');
+
+  describe('reading database', () => {
+
+    it('should return empty database on read corrupted database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+
+      assert(_.isEmpty(dataLocal.data.list));
+    });
+
+    it('should return a database on read valid database', () => {
+      const dataLocal = new LocalData(buildValidDbPath());
+
+      assert(_.isEmpty(dataLocal.data.list) === false);
+    });
+
+    it('should fails on sync a corrupted database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      const error = dataLocal.sync();
+
+      assert(_.isError(error));
+      assert(error.message.match(/locked/));
+      assert(dataLocal.locked);
+    });
+
+  });
+
+  describe('add/remove packages to database', () => {
+    it('should add a new package to local database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      assert(_.isEmpty(dataLocal.data.list));
+      dataLocal.add('package1');
+      assert(!_.isEmpty(dataLocal.data.list));
+    });
+
+    it('should remove a new package to local database', () => {
+      const dataLocal = new LocalData(buildCorruptedPath());
+      const pkgName = 'package1';
+
+      assert(_.isEmpty(dataLocal.data.list));
+      dataLocal.add(pkgName);
+      dataLocal.remove(pkgName);
+      assert(_.isEmpty(dataLocal.data.list));
+    });
+  });
+
+  describe('sync packages to database', () => {
+    beforeEach(function() {
+      this.newDb = path.join(__dirname, './test-storage/verdaccio.temp.db.json');
+      fs.copySync(buildValidDbPath(), this.newDb);
+    });
+
+    it('should check sync packages', function() {
+      const localData1 = new LocalData(this.newDb);
+
+      localData1.add('package1');
+
+      const localData2 = new LocalData(this.newDb);
+
+      assert(_.isEmpty(localData2.data.list) === false);
+      assert(localData2.data.list.length === 2);
+
+    });
+
+
+  });
+
+});
+

test/unit/partials/storage/verdaccio-corrupted.db.json

@@ -0,0 +1 @@
+{"list"[],"secret":"14705eeaed167749990dafa07f908d2a9504bfdb0f6ca4102eed5acba0bc9076"}

test/unit/partials/storage/verdaccio.db.json

@@ -0,0 +1 @@
+{"list":["@verdaccio/test"],"secret":"48cd053de97d4ef34aea4f1efb902334442bea1e735df5fdc9424c986a281b3d"}

wiki/cli.md

@@ -18,6 +18,8 @@ or if you use `yarn`
 $ yarn add global verdaccio
 ```
 
+> Warning: Verdaccio current is not support PM2's cluster mode, run it with cluster mode may cause unknown behavior
+
 ## Commands
 
 ```bash