✨ Added Substack to Ghost CSV converter package (#121)

refs https://github.com/TryGhost/Ghost/pull/11539

- The script helps to migrate CSV exports from Substack to Ghost-compatible ones

ghost/substack-ghost-csv-converter/.eslintrc.js

@@ -0,0 +1,6 @@
+module.exports = {
+    plugins: ['ghost'],
+    extends: [
+        'plugin:ghost/node',
+    ]
+};

ghost/substack-ghost-csv-converter/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Ghost Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ghost/substack-ghost-csv-converter/README.md

@@ -0,0 +1,42 @@
+# Substack Ghost Csv Converter
+
+Allows converting CSV export from Substack to CSV compatible with Ghost.
+
+## Install
+
+`npm install @tryghost/substack-ghost-csv-converter --save`
+
+or
+
+`yarn add @tryghost/substack-ghost-csv-converter`
+
+
+## Usage
+
+To convert the CSV file provide a path to in as a first parameter and the output csv file path as second parameter: `subghost <csv_input> <csv_output>`
+
+## Develop
+
+This is a mono repository, managed with [lerna](https://lernajs.io/).
+
+Follow the instructions for the top-level repo.
+1. `git clone` this repo & `cd` into it as usual
+2. Run `yarn` to install top-level dependencies.
+
+
+## Run
+
+- `yarn dev`
+
+
+## Test
+
+- `yarn lint` run just eslint
+- `yarn test` run lint and tests
+
+
+
+
+# Copyright & License
+
+Copyright (c) 2020 Ghost Foundation - Released under the [MIT license](LICENSE).

ghost/substack-ghost-csv-converter/bin/index.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+const prettyCLI = require('@tryghost/pretty-cli');
+const ui = require('@tryghost/pretty-cli').ui;
+const chalk = require('chalk');
+const converter = require('../');
+
+prettyCLI
+    .configure({
+        name: 'subghost'
+    })
+    .positional('<source>', {
+        paramsDesc: 'Substack CSV file path',
+        mustExist: true
+    })
+    .positional('<dest>', {
+        paramsDesc: 'Ghost CSV destination file path. Will write to source if not present',
+        mustExist: false
+    })
+    .parseAndExit()
+    .then((argv) => {
+        const dest = argv.dest || argv.source;
+
+        ui.log(`Converting Substack CSV file...`);
+
+        return converter(argv.source, dest).then(() => {
+            ui.log(`Conversion finished. File written to: ${chalk.cyan(dest)}`);
+
+            process.exit(0);
+        }).catch((e) => {
+            ui.log(e);
+            process.exit(1);
+        });
+    });

ghost/substack-ghost-csv-converter/index.js

@@ -0,0 +1,37 @@
+const {converter} = require('./lib');
+
+const convertCSV = async (originFilePath, destinationFilePath) => {
+    await converter.normalizeMembersCSV({
+        path: originFilePath,
+        destination: destinationFilePath,
+        columnsToMap: [{
+            from: 'email_disabled',
+            to: 'subscribed_to_emails',
+            negate: true
+        }, {
+            from: 'stripe_connected_customer_id',
+            to: 'stripe_customer_id'
+        }],
+        columnsToExtract: [{
+            name: 'email',
+            lookup: /^email/i
+        }, {
+            name: 'name',
+            lookup: /name/i
+        }, {
+            name: 'note',
+            lookup: /note/i
+        }, {
+            name: 'subscribed_to_emails',
+            lookup: /subscribed_to_emails/i
+        }, {
+            name: 'stripe_customer_id',
+            lookup: /stripe_customer_id/i
+        }, {
+            name: 'complimentary_plan',
+            lookup: /complimentary_plan/i
+        }]
+    });
+};
+
+module.exports = convertCSV;

ghost/substack-ghost-csv-converter/lib/converter.js

@@ -0,0 +1,93 @@
+const Promise = require('bluebird');
+const csvParser = require('csv-parser');
+const _ = require('lodash');
+const fs = require('fs-extra');
+const {formatCSV} = require('.');
+
+const normalizeCSVFileToJSON = async (options) => {
+    const columnsToExtract = options.columnsToExtract || [];
+    const columnsToMap = options.columnsToMap || [];
+    let results = [];
+    const rows = [];
+
+    return new Promise(function (resolve, reject) {
+        let readFile = fs.createReadStream(options.path);
+
+        readFile.on('err', function (err) {
+            reject(err);
+        })
+            .pipe(csvParser({
+                mapHeaders: ({header}) => {
+                    let mapping = columnsToMap.find(column => (column.from === header));
+                    if (mapping) {
+                        return mapping.to;
+                    }
+
+                    return header;
+                }
+            }))
+            .on('data', function (row) {
+                rows.push(row);
+            })
+            .on('end', function () {
+                // If CSV is single column - return all values including header
+                var headers = _.keys(rows[0]), result = {}, columnMap = {};
+
+                if (columnsToExtract.length === 1 && headers.length === 1) {
+                    results = _.map(rows, function (value) {
+                        result = {};
+                        result[columnsToExtract[0].name] = value[headers[0]];
+                        return result;
+                    });
+                } else {
+                    // If there are multiple columns in csv file
+                    // try to match headers using lookup value
+
+                    _.map(columnsToExtract, function findMatches(column) {
+                        _.each(headers, function checkheader(header) {
+                            if (column.lookup.test(header)) {
+                                columnMap[column.name] = header;
+                            }
+                        });
+                    });
+
+                    results = _.map(rows, function evaluateRow(row) {
+                        var result = {};
+                        _.each(columnMap, function returnMatches(value, key) {
+                            const mapping = columnsToMap.find(column => (column.to === key));
+
+                            if (mapping && mapping.negate) {
+                                result[key] = !(String(row[value]).toLowerCase() === 'true');
+                            } else {
+                                result[key] = row[value];
+                            }
+                        });
+                        return result;
+                    });
+                }
+
+                resolve(results);
+            });
+    });
+};
+
+const normalizeMembersCSV = async (options) => {
+    const results = await normalizeCSVFileToJSON(options);
+
+    let fields = ['email', 'name', 'note', 'subscribed_to_emails', 'stripe_customer_id'];
+
+    if (results && results.length) {
+        fields = Object.keys(results[0]);
+    }
+
+    const normalizedCSV = formatCSV(results, fields);
+
+    const resultFilePath = options.destination || options.origin;
+
+    return fs.writeFile(resultFilePath, normalizedCSV);
+};
+
+module.exports = {
+    normalizeCSVFileToJSON: normalizeCSVFileToJSON,
+    normalizeMembersCSV: normalizeMembersCSV
+};

ghost/substack-ghost-csv-converter/lib/format-csv.js

@@ -0,0 +1,22 @@
+module.exports = function formatCSV(data, fields) {
+    let csv = `${fields.join(',')}\r\n`;
+    let entry;
+    let field;
+    let j;
+    let i;
+
+    for (j = 0; j < data.length; j = j + 1) {
+        entry = data[j];
+
+        for (i = 0; i < fields.length; i = i + 1) {
+            field = fields[i];
+            csv += entry[field] !== null ? entry[field] : '';
+            if (i !== fields.length - 1) {
+                csv += ',';
+            }
+        }
+        csv += '\r\n';
+    }
+
+    return csv;
+};

ghost/substack-ghost-csv-converter/lib/index.js

@@ -0,0 +1,9 @@
+module.exports = {
+    get formatCSV() {
+        return require('./format-csv');
+    },
+
+    get converter() {
+        return require('./converter');
+    }
+};

ghost/substack-ghost-csv-converter/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@tryghost/substack-ghost-csv-converter",
+  "version": "0.0.0",
+  "repository": "https://github.com/TryGhost/Members/tree/master/packages/substack-ghost-csv-converter",
+  "author": "Ghost Foundation",
+  "license": "MIT",
+  "main": "index.js",
+  "scripts": {
+    "dev": "echo \"Implement me!\"",
+    "test": "NODE_ENV=testing mocha './test/**/*.test.js'",
+    "lint": "eslint . --ext .js --cache",
+    "posttest": "yarn lint"
+  },
+  "bin": {
+    "subghost": "./bin/index.js"
+  },
+  "files": [
+    "index.js",
+    "lib",
+    "bin"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "mocha": "7.0.1",
+    "should": "13.2.3",
+    "sinon": "8.1.1"
+  },
+  "dependencies": {
+    "@tryghost/pretty-cli": "1.2.2",
+    "bluebird": "3.7.2",
+    "csv-parser": "2.3.2",
+    "ghost-ignition": "4.0.0",
+    "lodash": "4.17.15"
+  }
+}

ghost/substack-ghost-csv-converter/test/.eslintrc.js

@@ -0,0 +1,9 @@
+module.exports = {
+    plugins: ['ghost'],
+    extends: [
+        'plugin:ghost/test',
+    ],
+    "parserOptions": {
+        "ecmaVersion": 2017
+    },
+};

ghost/substack-ghost-csv-converter/test/fixtures/substack-csv-format.csv

@@ -0,0 +1,3 @@
+email,expiry,email_disabled,stripe_connected_customer_id
+member+substack_1@example.com,"Fri Dec 25 2020 14:40:02 GMT+0000 (Coordinated Universal Time)","false",cus_GbbIQRd8TnFqHq
+member+substack_2@example.com,"Fri Dec 26 2020 14:40:02 GMT+0000 (Coordinated Universal Time)","true"

ghost/substack-ghost-csv-converter/test/hello.test.js

@@ -0,0 +1,10 @@
+// Switch these lines once there are useful utils
+// const testUtils = require('./utils');
+require('./utils');
+
+describe('Hello world', function () {
+    it('Runs a test', function () {
+        // TODO: Write me!
+        'hello'.should.eql('hello');
+    });
+});

ghost/substack-ghost-csv-converter/test/lib/normalize-members-csv.test.js

@@ -0,0 +1,45 @@
+const path = require('path');
+const {converter} = require('../../lib');
+// Switch these lines once there are useful utils
+// const testUtils = require('./utils');
+require('../utils');
+
+describe('Converts Substack CSV to Ghost CSV formats', function () {
+    it('Reads CSV and converts it to normalized JSON', async function () {
+        const result = await converter.normalizeCSVFileToJSON({
+            path: path.resolve('./test/fixtures/substack-csv-format.csv'),
+            columnsToMap: [{
+                from: 'email_disabled',
+                to: 'subscribed_to_emails',
+                negate: true
+            }, {
+                from: 'stripe_connected_customer_id',
+                to: 'stripe_customer_id'
+            }],
+            columnsToExtract: [{
+                name: 'email',
+                lookup: /^email/i
+            }, {
+                name: 'name',
+                lookup: /name/i
+            }, {
+                name: 'note',
+                lookup: /note/i
+            }, {
+                name: 'subscribed_to_emails',
+                lookup: /subscribed_to_emails/i
+            }, {
+                name: 'stripe_customer_id',
+                lookup: /stripe_customer_id/i
+            }, {
+                name: 'complimentary_plan',
+                lookup: /complimentary_plan/i
+            }]
+        });
+
+        result.length.should.equal(2);
+        Object.keys(result[0]).should.deepEqual(['email', 'subscribed_to_emails', 'stripe_customer_id']);
+        result[0].should.deepEqual({email: 'member+substack_1@example.com', subscribed_to_emails: true, stripe_customer_id: 'cus_GbbIQRd8TnFqHq'});
+        result[1].should.deepEqual({email: 'member+substack_2@example.com', subscribed_to_emails: false, stripe_customer_id: undefined});
+    });
+});

ghost/substack-ghost-csv-converter/test/utils/assertions.js

@@ -0,0 +1,11 @@
+/**
+ * Custom Should Assertions
+ *
+ * Add any custom assertions to this file.
+ */
+
+// Example Assertion
+// should.Assertion.add('ExampleAssertion', function () {
+//     this.params = {operator: 'to be a valid Example Assertion'};
+//     this.obj.should.be.an.Object;
+// });

ghost/substack-ghost-csv-converter/test/utils/index.js

@@ -0,0 +1,11 @@
+/**
+ * Test Utilities
+ *
+ * Shared utils for writing tests
+ */
+
+// Require overrides - these add globals for tests
+require('./overrides');
+
+// Require assertions - adds custom should assertions
+require('./assertions');

ghost/substack-ghost-csv-converter/test/utils/overrides.js

@@ -0,0 +1,10 @@
+// This file is required before any test is run
+
+// Taken from the should wiki, this is how to make should global
+// Should is a global in our eslint test config
+global.should = require('should').noConflict();
+should.extend();
+
+// Sinon is a simple case
+// Sinon is a global in our eslint test config
+global.sinon = require('sinon');