Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-01-20 22:42:53 -05:00
Code Issues Activity

Imported nested tags by foreign key

Browse source 
no issue

- replace logic for preparing nested tags
- if you have nested tags in your file, we won't update or update the target tag
- we simply would like to add the relationship to the database
- use same approach as base class
  - add `posts_tags` to target post model
  - update identifiers
  - insert relation by foreign key `tag_id`
- bump bookshelf-relations to 0.1.10
This commit is contained in:
kirrg001 2018-01-28 18:58:37 +01:00 • committed by Katharina Irrgang
parent 5a4dd6b792
commit 68d8154d4f
8 changed files with 157 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
core/server/data/importer/importers/data/base.js
View file

@ -34,13 +34,22 @@ class Base {
this.importedDataToReturn = [];
this.importedData = [];
this.options.requiredImportedData = ['users'];
this.options.requiredExistingData = ['users'];
this.requiredFromFile = {};
this.requiredImportedData = {};
this.requiredExistingData = {};
if (!this.options.requiredImportedData) {
this.options.requiredImportedData = ['users'];
} else {
this.options.requiredImportedData.push('users');
}
if (!this.options.requiredExistingData) {
this.options.requiredExistingData = ['users'];
} else {
this.options.requiredExistingData.push('users');
}
if (!this.options.requiredFromFile) {
this.options.requiredFromFile = ['users'];
} else {

137
core/server/data/importer/importers/data/posts.js
View file

@ -11,7 +11,9 @@ class PostsImporter extends BaseImporter {
super(allDataFromFile, {
modelName: 'Post',
dataKeyToImport: 'posts',
requiredFromFile: ['tags', 'posts_tags']
requiredFromFile: ['posts', 'tags', 'posts_tags'],
requiredImportedData: ['tags'],
requiredExistingData: ['tags']
});
this.legacyKeys = {
@ -28,76 +30,93 @@ class PostsImporter extends BaseImporter {
}
/**
* We don't have to worry about existing tag id's.
* e.g. you import a tag, which exists (doesn't get imported)
* ...because we add tags by unique name.
* Naive function to attach related tags.
* Target tags should not be created. We add the relation by foreign key.
*/
addTagsToPosts() {
let postTags = this.requiredFromFile.posts_tags,
postsWithTags = new Map(),
duplicatedTagsPerPost = {},
tagsToAttach = [],
foundOriginalTag;
addNestedRelations() {
this.requiredFromFile.posts_tags = _.orderBy(this.requiredFromFile.posts_tags, ['post_id', 'sort_order'], ['asc', 'asc']);
postTags = _.orderBy(postTags, ['post_id', 'sort_order'], ['asc', 'asc']);
_.each(postTags, (postTag) => {
if (!postsWithTags.get(postTag.post_id)) {
postsWithTags.set(postTag.post_id, []);
/**
* {post_id: 1, tag_id: 2}
*/
_.each(this.requiredFromFile.posts_tags, (postTagRelation) => {
if (!postTagRelation.post_id) {
return;
}
if (postsWithTags.get(postTag.post_id).indexOf(postTag.tag_id) !== -1) {
if (!duplicatedTagsPerPost.hasOwnProperty(postTag.post_id)) {
duplicatedTagsPerPost[postTag.post_id] = [];
let postToImport = _.find(this.dataToImport, {id: postTagRelation.post_id});
// CASE: we won't import a relation when the target post does not exist
if (!postToImport) {
return;
}
if (!postToImport.tags || !_.isArray(postToImport.tags)) {
postToImport.tags = [];
}
// CASE: duplicate relation?
if (!_.find(postToImport.tags, {tag_id: postTagRelation.tag_id})) {
postToImport.tags.push({
tag_id: postTagRelation.tag_id
});
}
});
}
/**
* Replace all `tag_id` references.
*/
replaceIdentifiers() {
/**
* {post_id: 1, tag_id: 2}
*/
_.each(this.dataToImport, (postToImport, postIndex) => {
if (!postToImport.tags || !postToImport.tags.length) {
return;
}
let indexesToRemove = [];
_.each(postToImport.tags, (tag, tagIndex) => {
let tagInFile = _.find(this.requiredFromFile.tags, {id: tag.tag_id});
if (!tagInFile) {
let existingTag = _.find(this.requiredExistingData.tags, {id: tag.tag_id});
// CASE: tag is not in file, tag is not in db
if (!existingTag) {
indexesToRemove.push(tagIndex);
return;
} else {
this.dataToImport[postIndex].tags[tagIndex].tag_id = existingTag.id;
return;
}
}
duplicatedTagsPerPost[postTag.post_id].push(postTag.tag_id);
}
// CASE: search through imported data
let importedTag = _.find(this.requiredImportedData.tags, {slug: tagInFile.slug});
postsWithTags.get(postTag.post_id).push(postTag.tag_id);
});
postsWithTags.forEach((tagIds, postId) => {
tagsToAttach = [];
_.each(tagIds, (tagId) => {
foundOriginalTag = _.find(this.requiredFromFile.tags, {id: tagId});
if (!foundOriginalTag) {
if (importedTag) {
this.dataToImport[postIndex].tags[tagIndex].tag_id = importedTag.id;
return;
}
tagsToAttach.push(foundOriginalTag);
// CASE: search through existing data
let existingTag = _.find(this.requiredExistingData.tags, {slug: tagInFile.slug});
if (existingTag) {
this.dataToImport[postIndex].tags[tagIndex].tag_id = existingTag.id;
} else {
indexesToRemove.push(tagIndex);
}
});
_.each(tagsToAttach, (tag) => {
_.each(this.dataToImport, (obj) => {
if (obj.id === postId) {
if (!_.isArray(obj.tags)) {
obj.tags = [];
}
if (duplicatedTagsPerPost.hasOwnProperty(postId) && duplicatedTagsPerPost[postId].length) {
this.problems.push({
message: 'Detected duplicated tags for: ' + obj.title || obj.slug,
help: this.modelName,
context: JSON.stringify({
tags: _.map(_.filter(this.requiredFromFile.tags, (tag) => {
return _.indexOf(duplicatedTagsPerPost[postId], tag.id) !== -1;
}), (value) => {
return value.slug || value.name;
})
})
});
}
obj.tags.push({
name: tag.name
});
}
});
});
this.dataToImport[postIndex].tags = _.filter(this.dataToImport[postIndex].tags, ((tag, index) => {
return indexesToRemove.indexOf(index) === -1;
}));
});
return super.replaceIdentifiers();
}
beforeImport() {
@ -105,7 +124,7 @@ class PostsImporter extends BaseImporter {
let mobileDocContent;
this.sanitizeAttributes();
this.addTagsToPosts();
this.addNestedRelations();
// Remove legacy field language
this.dataToImport = _.filter(this.dataToImport, (data) => {

15
core/server/data/importer/importers/data/tags.js
View file

@ -19,6 +19,13 @@ class TagsImporter extends BaseImporter {
};
}
fetchExisting(modelOptions) {
return models.Tag.findAll(_.merge({columns: ['id', 'slug']}, modelOptions))
.then((existingData) => {
this.existingData = existingData.toJSON();
});
}
beforeImport() {
debug('beforeImport');
return super.beforeImport();
@ -28,7 +35,7 @@ class TagsImporter extends BaseImporter {
* Find tag before adding.
* Background:
* - the tag model is smart enough to regenerate unique fields
* - so if you import a tag name "test" and the same tag name exists, it would add "test-2"
* - so if you import a tag slug "test" and the same tag slug exists, it would add "test-2"
* - that's why we add a protection here to first find the tag
*
* @TODO: Add a flag to the base implementation e.g. `fetchBeforeAdd`
@ -57,6 +64,12 @@ class TagsImporter extends BaseImporter {
this.importedDataToReturn.push(importedModel.toJSON());
}
// for identifier lookup
this.importedData.push({
id: importedModel.id,
slug: importedModel.get('slug')
});
return importedModel;
})
.catch((err) => {

2
core/server/models/post.js
View file

@ -211,7 +211,7 @@ Post = ghostBookshelf.Model.extend({
// and deduplicate upper/lowercase tags
_.each(this.get('tags'), function each(item) {
for (i = 0; i < tagsToSave.length; i = i + 1) {
if (tagsToSave[i].name.toLocaleLowerCase() === item.name.toLocaleLowerCase()) {
if (tagsToSave[i].name && item.name && tagsToSave[i].name.toLocaleLowerCase() === item.name.toLocaleLowerCase()) {
return;
}
}

7
core/test/integration/data/importer/importers/data_spec.js
View file

@ -427,8 +427,10 @@ describe('Import', function () {
// NOTE: a duplicated tag.slug is a warning
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [tags.name] cannot be blank.');
response[1].errorType.should.equal('ValidationError');
response[1].message.should.eql('Value in [posts.title] cannot be blank.');
response[2].errorType.should.equal('ValidationError');
response[2].message.should.eql('Value in [settings.key] cannot be blank.');
@ -480,6 +482,7 @@ describe('Import', function () {
importedData.problems[2].message.should.eql('Entry was imported, but we were not able to ' +
'resolve the following user references: author_id, published_by. The user does not exist, fallback to owner user.');
importedData.problems[2].help.should.eql('Post');
// Grab the data from tables
return Promise.all([
@ -527,11 +530,9 @@ describe('Import', function () {
}).then(function () {
done(new Error('Allowed import of invalid tags data'));
}).catch(function (response) {
response.length.should.equal(2);
response.length.should.equal(1);
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [tags.name] cannot be blank.');
response[1].errorType.should.equal('ValidationError');
response[1].message.should.eql('Value in [tags.name] cannot be blank.');
done();
}).catch(done);
});

44
core/test/utils/fixtures/export/lts/export-003-broken-tags.json Normal file
View file

@ -0,0 +1,44 @@
{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"id": 2,
"uuid": "8492fbba-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost-2",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 1
}
],
"posts_tags": [
{
"id": 1,
"post_id": 2,
"tag_id": 100
},
{
"id": 2,
"post_id": 2,
"tag_id": 200
}
]
}
}

2
package.json
View file

@ -35,7 +35,7 @@
"bluebird": "3.5.1",
"body-parser": "1.18.2",
"bookshelf": "0.10.3",
"bookshelf-relations": "0.1.5",
"bookshelf-relations": "0.1.10",
"brute-knex": "https://github.com/cobbspur/brute-knex/tarball/8979834383c7d3ee868d9e18d9ee6f71976dfec4",
"bson-objectid": "1.2.2",
"chalk": "1.1.3",

6
yarn.lock
View file

@ -538,9 +538,9 @@ body-parser@~1.14.0:
raw-body "~2.1.5"
type-is "~1.6.10"
bookshelf-relations@0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/bookshelf-relations/-/bookshelf-relations-0.1.5.tgz#514b3b83874113540634430b15a8d8ac7e203fe1"
bookshelf-relations@0.1.10:
version "0.1.10"
resolved "https://registry.yarnpkg.com/bookshelf-relations/-/bookshelf-relations-0.1.10.tgz#91ad66e8d6800dc043241a0738a8515df5e394bc"
dependencies:
bluebird "^3.4.1"
ghost-ignition "^2.8.16"