Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-02-03 23:00:14 -05:00
Code Issues Activity

Added "base pack" support for data generator script

Browse source 
refs: https://github.com/TryGhost/Toolbox/issues/453

This makes it so that a JSON bundle can be imported as well as the data generation script
This commit is contained in:
Sam Lord 2022-11-03 14:54:17 +00:00
parent 9bdb25d184
commit 08250d44b4
4 changed files with 112 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ghost/core/core/cli/generate-data.js
View file

@ -4,7 +4,7 @@ const DataGenerator = require('@tryghost/data-generator');
module.exports = class REPL extends Command {
setup() {
this.help('Generates random data to populate the database for development & testing');
this.argument('--use-base-data', {type: 'boolean', defaultValue: false, desc: 'Only generate data outside of a defined base data set'});
this.argument('--base-data-pack', {type: 'string', defaultValue: '', desc: 'Base data pack file location, imported instead of random content'});
}
initializeContext(context) {
@ -27,7 +27,7 @@ module.exports = class REPL extends Command {
const knex = require('../server/data/db/connection');
const {tables: schema} = require('../server/data/schema/index');
const dataGenerator = new DataGenerator({
useBaseData: argv['use-base-data'],
baseDataPack: argv['base-data-pack'],
knex,
schema,
logger: {

83
ghost/data-generator/lib/data-generator.js
View file

@ -24,11 +24,13 @@ const {
MembersSubscriptionCreatedEventsImporter,
MembersSubscribeEventsImporter
} = require('./tables');
const fs = require('fs/promises');
const {faker} = require('@faker-js/faker');
const JsonImporter = require('./utils/json-importer');
/**
* @typedef {Object} DataGeneratorOptions
* @property {boolean} useBaseData
* @property {string} baseDataPack
* @property {import('knex/types').Knex} knex
* @property {Object} schema
* @property {Object} logger
@ -53,13 +55,14 @@ class DataGenerator {
* @param {DataGeneratorOptions} options
*/
constructor({
useBaseData = false,
baseDataPack = '',
knex,
schema,
logger,
modelQuantities = {}
}) {
this.useBaseData = useBaseData;
this.useBaseData = baseDataPack !== '';
this.baseDataPack = baseDataPack;
this.knex = knex;
this.schema = schema;
this.logger = logger;
@ -78,24 +81,76 @@ class DataGenerator {
let products;
let stripeProducts;
let stripePrices;
let benefits;
// Use an existant set of data for a more realisitic looking site
if (this.useBaseData) {
// Must have at least 2 in base data set
newsletters = await transaction.select('id').from('newsletters');
const baseData = JSON.parse(await (await fs.readFile(this.baseDataPack)).toString());
const jsonImporter = new JsonImporter(transaction);
// Must have at least 2 in base data set
await transaction('newsletters').delete();
newsletters = await jsonImporter.import({
name: 'newsletters',
data: baseData.newsletters
});
await transaction('posts_authors').delete();
await transaction('posts_tags').delete();
await transaction('posts').delete();
const postsImporter = new PostsImporter(transaction, {
newsletters
});
posts = await transaction.select('id').from('posts');
posts = await jsonImporter.import({
name: 'posts',
data: baseData.posts
});
postsImporter.addNewsletters({posts});
posts = await transaction.select('id', 'newsletter_id').from('posts');
tags = await transaction.select('id').from('tags');
await transaction('tags').delete();
tags = await jsonImporter.import({
name: 'tags',
data: baseData.tags
});
products = await transaction.select('id', 'name', 'monthly_price', 'yearly_price').from('products');
stripeProducts = await transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
stripePrices = await transaction.select('id', 'stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname');
await transaction('products').delete();
products = await jsonImporter.import({
name: 'products',
data: baseData.products,
rows: ['name', 'monthly_price', 'yearly_price']
});
benefits = await jsonImporter.import({
name: 'benefits',
data: baseData.benefits
});
await jsonImporter.import({
name: 'products_benefits',
data: baseData.products_benefits
});
stripeProducts = await jsonImporter.import({
name: 'stripe_products',
data: baseData.stripe_products,
rows: ['product_id', 'stripe_product_id']
});
stripePrices = await jsonImporter.import({
name: 'stripe_prices',
data: baseData.stripe_prices,
rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
});
// Import settings
await jsonImporter.import({
name: 'settings',
data: baseData.settings
});
await jsonImporter.import({
name: 'custom_theme_settings',
data: baseData.custom_theme_settings
});
} else {
const newslettersImporter = new NewslettersImporter(transaction);
// First newsletter is free, second is paid
@ -121,7 +176,7 @@ class DataGenerator {
products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
const stripeProductsImporter = new StripeProductsImporter(transaction);
stripeProducts = await stripeProductsImporter.importForEach(products, {
stripeProducts = await stripeProductsImporter.importForEach(products.slice(1), {
amount: 1,
rows: ['product_id', 'stripe_product_id']
});
@ -139,7 +194,7 @@ class DataGenerator {
});
const benefitsImporter = new BenefitsImporter(transaction);
const benefits = await benefitsImporter.import({amount: 5});
benefits = await benefitsImporter.import({amount: 5});
const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits});
// Up to 5 benefits for each product
@ -165,12 +220,12 @@ class DataGenerator {
await postsAuthorsImporter.importForEach(posts, {amount: 1});
// TODO: Use subscriptions to generate members_products table?
const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.slice(1)});
const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name !== 'Free')});
const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), {
amount: 1,
rows: ['product_id', 'member_id']
});
const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: [products[0]]});
const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name === 'Free')});
await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), {
amount: 1,
rows: ['product_id', 'member_id']

9
ghost/data-generator/lib/tables/subscriptions.js
View file

@ -26,12 +26,9 @@ class SubscriptionsImporter extends TableImporter {
return price.stripe_product_id === stripeProduct.stripe_product_id &&
(isMonthly ? price.interval === 'month' : price.interval === 'year');
});
// TODO: Understand why stripePrice can sometimes be undefined
if (stripePrice) {
billingInfo.cadence = isMonthly ? 'month' : 'year';
billingInfo.currency = stripePrice.currency;
billingInfo.amount = stripePrice.amount;
}
billingInfo.cadence = isMonthly ? 'month' : 'year';
billingInfo.currency = stripePrice.currency;
billingInfo.amount = stripePrice.amount;
}
const [startDate] = generateEvents({
total: 1,

38
ghost/data-generator/lib/utils/json-importer.js Normal file
View file

@ -0,0 +1,38 @@
const {faker} = require('@faker-js/faker');
class JsonImporter {
constructor(knex) {
this.knex = knex;
}
/**
* @typedef {Object} JsonImportOptions
* @property {string} name Name of the table to import
* @property {Object} data Models without ids to be imported
* @property {Array<string>} [rows] Set of rows to be returned
*/
/**
* Import a dataset to the database
* @param {JsonImportOptions} options
* @returns {Promise<Array<Object.<string, any>>>} Set of rows returned from database
*/
async import({
name,
data,
rows = []
}) {
for (const obj of data) {
if (!('id' in obj)) {
obj.id = faker.database.mongodbObjectId();
}
}
if (rows.findIndex(row => row === 'id') === -1) {
rows.unshift('id');
}
await this.knex.batchInsert(name, data, 500);
return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(name);
}
}
module.exports = JsonImporter;