mirror of
https://github.com/TryGhost/Ghost.git
synced 2025-02-10 23:36:14 -05:00
Refactored bulk insert/delete operations into separate module
no issue - Moved bulk db operations outside of importer module to create clearer separation of responsibilities
This commit is contained in:
parent
3a594ce22e
commit
f8adb4a672
2 changed files with 50 additions and 41 deletions
47
core/server/services/members/importer/bulk-operations.js
Normal file
47
core/server/services/members/importer/bulk-operations.js
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
const _ = require('lodash');
|
||||||
|
const db = require('../../../data/db');
|
||||||
|
|
||||||
|
const CHUNK_SIZE = 100;
|
||||||
|
|
||||||
|
async function insert(table, data) {
|
||||||
|
const result = {
|
||||||
|
successful: 0,
|
||||||
|
unsuccessful: 0,
|
||||||
|
errors: []
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const chunk of _.chunk(data, CHUNK_SIZE)) {
|
||||||
|
try {
|
||||||
|
await db.knex(table).insert(chunk);
|
||||||
|
result.successful += chunk.length;
|
||||||
|
} catch (error) {
|
||||||
|
result.unsuccessful += chunk.length;
|
||||||
|
result.errors.push(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function del(table, ids) {
|
||||||
|
const result = {
|
||||||
|
successful: 0,
|
||||||
|
unsuccessful: 0,
|
||||||
|
errors: []
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const chunk of _.chunk(ids, CHUNK_SIZE)) {
|
||||||
|
try {
|
||||||
|
await db.knex(table).whereIn('id', chunk).del();
|
||||||
|
result.successful += chunk.length;
|
||||||
|
} catch (error) {
|
||||||
|
result.unsuccessful += chunk.length;
|
||||||
|
result.errors.push(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports.insert = insert;
|
||||||
|
module.exports.del = del;
|
|
@ -4,14 +4,14 @@ const ObjectId = require('bson-objectid');
|
||||||
const moment = require('moment-timezone');
|
const moment = require('moment-timezone');
|
||||||
const errors = require('@tryghost/errors');
|
const errors = require('@tryghost/errors');
|
||||||
const membersService = require('../index');
|
const membersService = require('../index');
|
||||||
|
const bulkOperations = require('./bulk-operations');
|
||||||
const models = require('../../../models');
|
const models = require('../../../models');
|
||||||
const {i18n} = require('../../../lib/common');
|
const {i18n} = require('../../../lib/common');
|
||||||
const db = require('../../../data/db');
|
|
||||||
const logging = require('../../../../shared/logging');
|
const logging = require('../../../../shared/logging');
|
||||||
|
|
||||||
const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) => {
|
const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) => {
|
||||||
const createInserter = table => data => insert(db.knex, table, data);
|
const createInserter = table => data => bulkOperations.insert(table, data);
|
||||||
const createDeleter = table => data => del(db.knex, table, data);
|
const createDeleter = table => data => bulkOperations.del(table, data);
|
||||||
|
|
||||||
const deleteMembers = createDeleter('members');
|
const deleteMembers = createDeleter('members');
|
||||||
const insertLabelAssociations = createInserter('members_labels');
|
const insertLabelAssociations = createInserter('members_labels');
|
||||||
|
@ -107,44 +107,6 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
const CHUNK_SIZE = 100;
|
|
||||||
|
|
||||||
async function insert(knex, table, data) {
|
|
||||||
const result = {
|
|
||||||
successful: 0,
|
|
||||||
unsuccessful: 0,
|
|
||||||
errors: []
|
|
||||||
};
|
|
||||||
for (const chunk of _.chunk(data, CHUNK_SIZE)) {
|
|
||||||
try {
|
|
||||||
await knex(table).insert(chunk);
|
|
||||||
result.successful += chunk.length;
|
|
||||||
} catch (error) {
|
|
||||||
result.unsuccessful += chunk.length;
|
|
||||||
result.errors.push(error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function del(knex, table, ids) {
|
|
||||||
const result = {
|
|
||||||
successful: 0,
|
|
||||||
unsuccessful: 0,
|
|
||||||
errors: []
|
|
||||||
};
|
|
||||||
for (const chunk of _.chunk(ids, CHUNK_SIZE)) {
|
|
||||||
try {
|
|
||||||
await knex(table).whereIn('id', chunk).del();
|
|
||||||
result.successful += chunk.length;
|
|
||||||
} catch (error) {
|
|
||||||
result.unsuccessful += chunk.length;
|
|
||||||
result.errors.push(error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
function serializeMemberLabels(labels) {
|
function serializeMemberLabels(labels) {
|
||||||
return labels.reduce((labelsAcc, label) => {
|
return labels.reduce((labelsAcc, label) => {
|
||||||
if (!label) {
|
if (!label) {
|
||||||
|
|
Loading…
Add table
Reference in a new issue