Moved tokens, url safe and safe string utility to lib/security

refs #9178

- we could now also move any crypto usages to lib/security, but no priority
- the main goal is to tidy up our utils folder

core/server/api/authentication.js

@@ -9,6 +9,7 @@ var Promise = require('bluebird'),
     models = require('../models'),
     config = require('../config'),
     common = require('../lib/common'),
+    security = require('../lib/security'),
     spamPrevention = require('../web/middleware/api/spam-prevention'),
     mailAPI = require('./mail'),
     settingsAPI = require('./settings'),
@@ -167,7 +168,7 @@ authentication = {
                         throw new common.errors.NotFoundError({message: common.i18n.t('errors.api.users.userNotFound')});
                     }
 
-                    token = globalUtils.tokens.resetToken.generateHash({
+                    token = security.tokens.resetToken.generateHash({
                         expires: Date.now() + globalUtils.ONE_DAY_MS,
                         email: email,
                         dbHash: dbHash,
@@ -183,7 +184,7 @@ authentication = {
 
         function sendResetNotification(data) {
             var adminUrl = urlService.utils.urlFor('admin', true),
-                resetUrl = urlService.utils.urlJoin(adminUrl, 'reset', globalUtils.encodeBase64URLsafe(data.resetToken), '/');
+                resetUrl = urlService.utils.urlJoin(adminUrl, 'reset', security.url.encodeBase64(data.resetToken), '/');
 
             return mail.utils.generateContent({
                 data: {
@@ -251,9 +252,9 @@ authentication = {
         }
 
         function extractTokenParts(options) {
-            options.data.passwordreset[0].token = globalUtils.decodeBase64URLsafe(options.data.passwordreset[0].token);
+            options.data.passwordreset[0].token = security.url.decodeBase64(options.data.passwordreset[0].token);
 
-            tokenParts = globalUtils.tokens.resetToken.extract({
+            tokenParts = security.tokens.resetToken.extract({
                 token: options.data.passwordreset[0].token
             });
 
@@ -295,7 +296,7 @@ authentication = {
                         throw new common.errors.NotFoundError({message: common.i18n.t('errors.api.users.userNotFound')});
                     }
 
-                    tokenIsCorrect = globalUtils.tokens.resetToken.compare({
+                    tokenIsCorrect = security.tokens.resetToken.compare({
                         token: resetToken,
                         dbHash: dbHash,
                         password: user.get('password')
@@ -382,7 +383,7 @@ authentication = {
         }
 
         function processInvitation(invitation) {
-            var data = invitation.invitation[0], inviteToken = globalUtils.decodeBase64URLsafe(data.token);
+            var data = invitation.invitation[0], inviteToken = security.url.decodeBase64(data.token);
 
             return models.Invite.findOne({token: inviteToken, status: 'sent'}, options)
                 .then(function (_invite) {

core/server/api/invites.js

@@ -2,11 +2,11 @@ var Promise = require('bluebird'),
     _ = require('lodash'),
     pipeline = require('../lib/promise/pipeline'),
     mail = require('../services/mail'),
-    globalUtils = require('../utils'),
     urlService = require('../services/url'),
     localUtils = require('./utils'),
     models = require('../models'),
     common = require('../lib/common'),
+    security = require('../lib/security'),
     mailAPI = require('./mail'),
     settingsAPI = require('./settings'),
     docName = 'invites',
@@ -107,7 +107,7 @@ invites = {
                         invitedByName: loggedInUser.get('name'),
                         invitedByEmail: loggedInUser.get('email'),
                         // @TODO: resetLink sounds weird
-                        resetLink: urlService.utils.urlJoin(adminUrl, 'signup', globalUtils.encodeBase64URLsafe(invite.get('token')), '/')
+                        resetLink: urlService.utils.urlJoin(adminUrl, 'signup', security.url.encodeBase64(invite.get('token')), '/')
                     };
 
                     return mail.utils.generateContent({data: emailData, template: 'invite-user'});

core/server/controllers/channel.js

@@ -1,7 +1,7 @@
 var _ = require('lodash'),
     common = require('../lib/common'),
+    security = require('../lib/security'),
     filters = require('../filters'),
-    safeString = require('../utils').safeString,
     handleError = require('./frontend/error'),
     fetchData = require('./frontend/fetch-data'),
     setRequestIsSecure = require('./frontend/secure'),
@@ -13,7 +13,7 @@ var _ = require('lodash'),
 module.exports = function channelController(req, res, next) {
     // Parse the parameters we need from the URL
     var pageParam = req.params.page !== undefined ? req.params.page : 1,
-        slugParam = req.params.slug ? safeString(req.params.slug) : undefined;
+        slugParam = req.params.slug ? security.string.safe(req.params.slug) : undefined;
 
     // @TODO: fix this, we shouldn't change the channel object!
     // Set page on postOptions for the query made later

core/server/controllers/rss.js

@@ -1,7 +1,7 @@
 var _ = require('lodash'),
     url = require('url'),
     common = require('../lib/common'),
-    safeString = require('../utils').safeString,
+    security = require('../lib/security'),
     settingsCache = require('../services/settings/cache'),
 
     // Slightly less ugly temporary hack for location of things
@@ -47,7 +47,7 @@ function getData(channelOpts) {
 generate = function generate(req, res, next) {
     // Parse the parameters we need from the URL
     var pageParam = req.params.page !== undefined ? req.params.page : 1,
-        slugParam = req.params.slug ? safeString(req.params.slug) : undefined,
+        slugParam = req.params.slug ? security.string.safe(req.params.slug) : undefined,
         // Base URL needs to be the URL for the feed without pagination:
         baseUrl = getBaseUrlForRSSReq(req.originalUrl, pageParam);
 

core/server/data/export/index.js

@@ -2,9 +2,9 @@ var _ = require('lodash'),
     Promise = require('bluebird'),
     db = require('../../data/db'),
     commands = require('../schema').commands,
-    globalUtils = require('../../utils'),
     ghostVersion = require('../../utils/ghost-version'),
     common = require('../../lib/common'),
+    security = require('../../lib/security'),
     models = require('../../models'),
     excludedTables = ['accesstokens', 'refreshtokens', 'clients', 'client_trusted_domains'],
     modelOptions = {context: {internal: true}},
@@ -23,7 +23,7 @@ exportFileName = function exportFileName(options) {
 
     return models.Settings.findOne({key: 'title'}, _.merge({}, modelOptions, options)).then(function (result) {
         if (result) {
-            title = globalUtils.safeString(result.get('value')) + '.';
+            title = security.string.safe(result.get('value')) + '.';
         }
 
         return title + 'ghost.' + datetime + '.json';

core/server/lib/security/index.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = {
+    get url() {
+        return require('./url');
+    },
+
+    get tokens() {
+        return require('./tokens');
+    },
+
+    get string() {
+        return require('./string');
+    }
+};

core/server/lib/security/string.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const unidecode = require('unidecode'),
+    _ = require('lodash');
+
+module.exports.safe = function safe(string, options) {
+    options = options || {};
+
+    if (string === null) {
+        string = '';
+    }
+
+    // Handle the £ symbol separately, since it needs to be removed before the unicode conversion.
+    string = string.replace(/£/g, '-');
+
+    // Remove non ascii characters
+    string = unidecode(string);
+
+    // Replace URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}` and \`
+    string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|"|\{|\}|`|–|—)/g, '-')
+    // Remove apostrophes
+        .replace(/'/g, '')
+        // Make the whole thing lowercase
+        .toLowerCase();
+
+    // We do not need to make the following changes when importing data
+    if (!_.has(options, 'importing') || !options.importing) {
+        // Convert 2 or more dashes into a single dash
+        string = string.replace(/-+/g, '-')
+        // Remove trailing dash
+            .replace(/-$/, '')
+            // Remove any dashes at the beginning
+            .replace(/^-/, '');
+    }
+
+    // Handle whitespace at the beginning or end.
+    string = string.trim();
+
+    return string;
+};

core/server/lib/security/tokens.js

@@ -1,4 +1,6 @@
-var crypto = require('crypto');
+'use strict';
+
+const crypto = require('crypto');
 
 exports.resetToken = {
     generateHash: function generateHash(options) {

core/server/lib/security/url.js

@@ -0,0 +1,14 @@
+// The token is encoded URL safe by replacing '+' with '-', '\' with '_' and removing '='
+// NOTE: the token is not encoded using valid base64 anymore
+module.exports.encodeBase64 = function encodeBase64(base64String) {
+    return base64String.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+};
+
+// Decode url safe base64 encoding and add padding ('=')
+module.exports.decodeBase64 = function decodeBase64(base64String) {
+    base64String = base64String.replace(/-/g, '+').replace(/_/g, '/');
+    while (base64String.length % 4) {
+        base64String += '=';
+    }
+    return base64String;
+};

core/server/models/base/index.js

@@ -13,10 +13,10 @@ var _ = require('lodash'),
     config = require('../../config'),
     db = require('../../data/db'),
     common = require('../../lib/common'),
+    security = require('../../lib/security'),
     filters = require('../../filters'),
     schema = require('../../data/schema'),
     urlService = require('../../services/url'),
-    globalUtils = require('../../utils'),
     validation = require('../../data/validation'),
     plugins = require('../plugins'),
 
@@ -739,7 +739,7 @@ ghostBookshelf.Model = ghostBookshelf.Model.extend({
         // the slug may never be longer than the allowed limit of 191 chars, but should also
         // take the counter into count. We reduce a too long slug to 185 so we're always on the
         // safe side, also in terms of checking for existing slugs already.
-        slug = globalUtils.safeString(base, options);
+        slug = security.string.safe(base, options);
 
         if (slug.length > 185) {
             // CASE: don't cut the slug on import

core/server/services/auth/auth-strategies.js

@@ -2,6 +2,7 @@ var _ = require('lodash'),
     models = require('../../models'),
     globalUtils = require('../../utils'),
     common = require('../../lib/common'),
+    security = require('../../lib/security'),
     strategies;
 
 strategies = {
@@ -94,7 +95,7 @@ strategies = {
 
         handleInviteToken = function handleInviteToken() {
             var user, invite;
-            inviteToken = globalUtils.decodeBase64URLsafe(inviteToken);
+            inviteToken = security.url.decodeBase64(inviteToken);
 
             return models.Invite.findOne({token: inviteToken}, options)
                 .then(function addInviteUser(_invite) {

core/server/utils/index.js

@@ -1,6 +1,4 @@
-var unidecode = require('unidecode'),
-    _ = require('lodash'),
-    utils,
+var utils,
     getRandomInt;
 
 /**
@@ -57,60 +55,8 @@ utils = {
         return buf.join('');
     },
 
-    safeString: function (string, options) {
-        options = options || {};
-
-        if (string === null) {
-            string = '';
-        }
-
-        // Handle the £ symbol separately, since it needs to be removed before the unicode conversion.
-        string = string.replace(/£/g, '-');
-
-        // Remove non ascii characters
-        string = unidecode(string);
-
-        // Replace URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}` and \`
-        string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|"|\{|\}|`|–|—)/g, '-')
-        // Remove apostrophes
-            .replace(/'/g, '')
-            // Make the whole thing lowercase
-            .toLowerCase();
-
-        // We do not need to make the following changes when importing data
-        if (!_.has(options, 'importing') || !options.importing) {
-            // Convert 2 or more dashes into a single dash
-            string = string.replace(/-+/g, '-')
-            // Remove trailing dash
-                .replace(/-$/, '')
-                // Remove any dashes at the beginning
-                .replace(/^-/, '');
-        }
-
-        // Handle whitespace at the beginning or end.
-        string = string.trim();
-
-        return string;
-    },
-
-    // The token is encoded URL safe by replacing '+' with '-', '\' with '_' and removing '='
-    // NOTE: the token is not encoded using valid base64 anymore
-    encodeBase64URLsafe: function (base64String) {
-        return base64String.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
-    },
-
-    // Decode url safe base64 encoding and add padding ('=')
-    decodeBase64URLsafe: function (base64String) {
-        base64String = base64String.replace(/-/g, '+').replace(/_/g, '/');
-        while (base64String.length % 4) {
-            base64String += '=';
-        }
-        return base64String;
-    },
-
     readCSV: require('./read-csv'),
     zipFolder: require('./zip-folder'),
-    tokens: require('./tokens'),
     ghostVersion: require('./ghost-version')
 };
 

core/test/functional/routes/api/authentication_spec.js

@@ -6,7 +6,7 @@ var should = require('should'),
     userForKnex = testUtils.DataGenerator.forKnex.users[0],
     models = require('../../../../../core/server/models'),
     config = require('../../../../../core/server/config'),
-    utils = require('../../../../../core/server/utils'),
+    security = require('../../../../../core/server/lib/security'),
     ghost = testUtils.startGhost,
     request;
 
@@ -212,7 +212,7 @@ describe('Authentication API', function () {
         models.Settings
             .findOne({key: 'db_hash'})
             .then(function (response) {
-                var token = utils.tokens.resetToken.generateHash({
+                var token = security.tokens.resetToken.generateHash({
                     expires: Date.now() + (1000 * 60),
                     email: user.email,
                     dbHash: response.attributes.value,

core/test/unit/lib/security/string_spec.js

@@ -1,96 +1,93 @@
 var should = require('should'), // jshint ignore:line
-    nock = require('nock'),
-    configUtils = require('../../utils/configUtils'),
-    utils = require('../../../server/utils');
+    security = require('../../../../server/lib/security');
 
-describe('Server Utilities', function () {
+describe('Lib: Security - String', function () {
     describe('Safe String', function () {
-        var safeString = utils.safeString,
-            options = {};
+        var options = {};
 
         it('should remove beginning and ending whitespace', function () {
-            var result = safeString(' stringwithspace ', options);
+            var result = security.string.safe(' stringwithspace ', options);
             result.should.equal('stringwithspace');
         });
 
         it('can handle null strings', function () {
-            var result = safeString(null);
+            var result = security.string.safe(null);
             result.should.equal('');
         });
 
         it('should remove non ascii characters', function () {
-            var result = safeString('howtowin✓', options);
+            var result = security.string.safe('howtowin✓', options);
             result.should.equal('howtowin');
         });
 
         it('should replace spaces with dashes', function () {
-            var result = safeString('how to win', options);
+            var result = security.string.safe('how to win', options);
             result.should.equal('how-to-win');
         });
 
         it('should replace most special characters with dashes', function () {
-            var result = safeString('a:b/c?d#e[f]g!h$i&j(k)l*m+n,o;{p}=q\\r%s<t>u|v^w~x£y"z@1.2`3', options);
+            var result = security.string.safe('a:b/c?d#e[f]g!h$i&j(k)l*m+n,o;{p}=q\\r%s<t>u|v^w~x£y"z@1.2`3', options);
             result.should.equal('a-b-c-d-e-f-g-h-i-j-k-l-m-n-o-p-q-r-s-t-u-v-w-x-y-z-1-2-3');
         });
 
         it('should replace all of the html4 compat symbols in ascii except hyphen and underscore', function () {
             // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc,
             // it passed the test before it was removed
-            var result = safeString('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿');
+            var result = security.string.safe('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿');
             result.should.equal('_-c-y-ss-c-a-r-deg-23up-1o-1-41-23-4');
         });
 
         it('should replace all of the foreign chars in ascii', function () {
-            var result = safeString('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ');
+            var result = security.string.safe('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ');
             result.should.equal('aaaaaaaeceeeeiiiidnoooooxouuuuuthssaaaaaaaeceeeeiiiidnooooo-ouuuuythy');
         });
 
         it('should remove special characters at the beginning of a string', function () {
-            var result = safeString('.Not special', options);
+            var result = security.string.safe('.Not special', options);
             result.should.equal('not-special');
         });
 
         it('should remove apostrophes ', function () {
-            var result = safeString('how we shouldn\'t be', options);
+            var result = security.string.safe('how we shouldn\'t be', options);
             result.should.equal('how-we-shouldnt-be');
         });
 
         it('should convert to lowercase', function () {
-            var result = safeString('This has Upper Case', options);
+            var result = security.string.safe('This has Upper Case', options);
             result.should.equal('this-has-upper-case');
         });
 
         it('should convert multiple dashes into a single dash', function () {
-            var result = safeString('This :) means everything', options);
+            var result = security.string.safe('This :) means everything', options);
             result.should.equal('this-means-everything');
         });
 
         it('should remove trailing dashes from the result', function () {
-            var result = safeString('This.', options);
+            var result = security.string.safe('This.', options);
             result.should.equal('this');
         });
 
         it('should handle pound signs', function () {
-            var result = safeString('WHOOPS! I spent all my £ again!', options);
+            var result = security.string.safe('WHOOPS! I spent all my £ again!', options);
             result.should.equal('whoops-i-spent-all-my-again');
         });
 
         it('should properly handle unicode punctuation conversion', function () {
-            var result = safeString('に間違いがないか、再度確認してください。再読み込みしてください。', options);
+            var result = security.string.safe('に間違いがないか、再度確認してください。再読み込みしてください。', options);
             result.should.equal('nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai');
         });
 
         it('should not lose or convert dashes if options are passed with truthy importing flag', function () {
             var result,
                 options = {importing: true};
-            result = safeString('-slug-with-starting-ending-and---multiple-dashes-', options);
+            result = security.string.safe('-slug-with-starting-ending-and---multiple-dashes-', options);
             result.should.equal('-slug-with-starting-ending-and---multiple-dashes-');
         });
 
         it('should still remove/convert invalid characters when passed options with truthy importing flag', function () {
             var result,
                 options = {importing: true};
-            result = safeString('-slug-&with-✓-invalid-characters-に\'', options);
+            result = security.string.safe('-slug-&with-✓-invalid-characters-に\'', options);
             result.should.equal('-slug--with--invalid-characters-ni');
         });
     });

core/test/unit/lib/security/tokens_spec.js

@@ -1,13 +1,13 @@
 var should = require('should'), // jshint ignore:line
     uuid = require('uuid'),
-    utils = require('../../../server/utils');
+    security = require('../../../../server/lib/security');
 
 describe('Utils: tokens', function () {
     it('generate', function () {
         var expires = Date.now() + 60 * 1000,
             dbHash = uuid.v4(), token;
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: 'test1@ghost.org',
             expires: expires,
             password: 'password',
@@ -22,14 +22,14 @@ describe('Utils: tokens', function () {
         var expires = Date.now() + 60 * 1000,
             dbHash = uuid.v4(), token, tokenIsCorrect;
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: 'test1@ghost.org',
             expires: expires,
             password: '12345678',
             dbHash: dbHash
         });
 
-        tokenIsCorrect = utils.tokens.resetToken.compare({
+        tokenIsCorrect = security.tokens.resetToken.compare({
             token: token,
             dbHash: dbHash,
             password: '12345678'
@@ -42,14 +42,14 @@ describe('Utils: tokens', function () {
         var expires = Date.now() + 60 * 1000,
             dbHash = uuid.v4(), token, tokenIsCorrect;
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: 'test1@ghost.org',
             expires: expires,
             password: '12345678',
             dbHash: dbHash
         });
 
-        tokenIsCorrect = utils.tokens.resetToken.compare({
+        tokenIsCorrect = security.tokens.resetToken.compare({
             token: token,
             dbHash: dbHash,
             password: '123456'
@@ -62,14 +62,14 @@ describe('Utils: tokens', function () {
         var expires = Date.now() + 60 * 1000,
             dbHash = uuid.v4(), token, parts, email = 'test1@ghost.org';
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: email,
             expires: expires,
             password: '12345678',
             dbHash: dbHash
         });
 
-        parts = utils.tokens.resetToken.extract({
+        parts = security.tokens.resetToken.extract({
             token: token
         });
 
@@ -83,14 +83,14 @@ describe('Utils: tokens', function () {
         var expires = Date.now() + 60 * 1000,
             dbHash = uuid.v4(), token, parts, email = 'test3@ghost.org';
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: email,
             expires: expires,
             password: '$2a$10$t5dY1uRRdjvqfNlXhae3uuc0nuhi.Rd7/K/9JaHHwSkLm6UUa3NsW',
             dbHash: dbHash
         });
 
-        parts = utils.tokens.resetToken.extract({
+        parts = security.tokens.resetToken.extract({
             token: token
         });
 
@@ -105,26 +105,26 @@ describe('Utils: tokens', function () {
             email = 'test1@ghost.org',
             dbHash = uuid.v4(), token, tokenIsCorrect, parts;
 
-        token = utils.tokens.resetToken.generateHash({
+        token = security.tokens.resetToken.generateHash({
             email: email,
             expires: expires,
             password: '12345678',
             dbHash: dbHash
         });
 
-        token = utils.encodeBase64URLsafe(token);
+        token = security.url.encodeBase64(token);
         token = encodeURIComponent(token);
         token = decodeURIComponent(token);
-        token = utils.decodeBase64URLsafe(token);
+        token = security.url.decodeBase64(token);
 
-        parts = utils.tokens.resetToken.extract({
+        parts = security.tokens.resetToken.extract({
             token: token
         });
 
         parts.email.should.eql(email);
         parts.expires.should.eql(expires);
 
-        tokenIsCorrect = utils.tokens.resetToken.compare({
+        tokenIsCorrect = security.tokens.resetToken.compare({
             token: token,
             dbHash: dbHash,
             password: '12345678'