diff --git a/ghost/core/core/server/services/bulk-email/bulk-email-processor.js b/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
index 37a8f3d753..0d6bba2eb1 100644
--- a/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
+++ b/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
@@ -7,7 +7,6 @@ const logging = require('@tryghost/logging');
const models = require('../../models');
const MailgunClient = require('@tryghost/mailgun-client');
const sentry = require('../../../shared/sentry');
-const labs = require('../../../shared/labs');
const debug = require('@tryghost/debug')('mega');
const postEmailSerializer = require('../mega/post-email-serializer');
const configService = require('../../../shared/config');
@@ -173,10 +172,8 @@ module.exports = {
// Load newsletter data on email
await emailBatchModel.relations.email.getLazyRelation('newsletter', {require: false, ...knexOptions});
- if (labs.isSet('newsletterPaywall')) {
- // Load post data on email - for content gating on paywall
- await emailBatchModel.relations.email.getLazyRelation('post', {require: false, ...knexOptions});
- }
+ // Load post data on email - for content gating on paywall
+ await emailBatchModel.relations.email.getLazyRelation('post', {require: false, ...knexOptions});
// send the email
const sendResponse = await this.send(emailBatchModel.relations.email.toJSON(), recipientRows, memberSegment);
diff --git a/ghost/core/core/server/services/mega/post-email-serializer.js b/ghost/core/core/server/services/mega/post-email-serializer.js
index 84de1051b1..7a413988ae 100644
--- a/ghost/core/core/server/services/mega/post-email-serializer.js
+++ b/ghost/core/core/server/services/mega/post-email-serializer.js
@@ -329,28 +329,48 @@ const PostEmailSerializer = {
let htmlTemplate = render({post, site: this.getSite(), templateSettings, newsletter: newsletter.toJSON()});
// The plaintext version that is returned here is actually never really used for sending because we'll use htmlToPlaintext again later
- let content = {
+ let result = {
html: this.formatHtmlForEmail(htmlTemplate),
plaintext: post.plaintext
};
- // Also replace the links in the HTML version
+ /**
+ * If a part of the email is members-only and the post is paid-only, add a paywall:
+ * - Just before sending the email, we'll hide the paywall or paid content depending on the member segment it is sent to.
+ * - We already need to do URL-replacement on the HTML here
+ * - Link replacement cannot happen later because renderEmailForSegment is called multiple times for a single email (which would result in duplicate redirects)
+ */
+ const isPaidPost = post.visibility === 'paid' || post.visibility === 'tiers';
+
+ const paywallIndex = (result.html || '').indexOf('');
+ if (paywallIndex !== -1 && isPaidPost) {
+ const postContentEndIdx = result.html.indexOf('');
+
+ if (postContentEndIdx !== -1) {
+ const paywallHTML = '' + this.renderPaywallCTA(post);
+
+ // Append it just before the end of the post content
+ result.html = result.html.slice(0, postContentEndIdx) + paywallHTML + result.html.slice(postContentEndIdx);
+ }
+ }
+
+ // Now replace the links in the HTML version
if (labs.isSet('emailClicks')) {
if ((!options.isBrowserPreview && !options.isTestEmail) || process.env.NODE_ENV === 'development') {
- content.html = await linkReplacement.service.replaceLinks(content.html, newsletter, postModel);
+ result.html = await linkReplacement.service.replaceLinks(result.html, newsletter, postModel);
}
}
// Clean up any unknown replacements strings to get our final content
- const {html, plaintext} = this.normalizeReplacementStrings(content);
+ const {html, plaintext} = this.normalizeReplacementStrings(result);
const data = {
subject: post.email_subject || post.title,
html,
plaintext
};
- if (labs.isSet('newsletterPaywall')) {
- data.post = post;
- }
+
+ // Add post for checking access in renderEmailForSegment (only for previews)
+ data.post = post;
return data;
},
@@ -400,25 +420,45 @@ const PostEmailSerializer = {
const result = {...email};
- /** Checks and hides content for newsletter behind paywall card
- * based on member's status and post access
- * Adds CTA in case content is hidden.
- */
- if (labs.isSet('newsletterPaywall')) {
- const paywallIndex = (result.html || '').indexOf('');
- if (paywallIndex !== -1 && memberSegment && result.post) {
+ // Note about link tracking:
+ // Don't add new HTML in here, but add it in the serialize method and surround it with the required HTML comments or attributes
+ // This is because we can't replace links at this point (this is executed multiple times, once per batch and we don't want to generate duplicate links for the same email)
+
+ // Remove the paywall or members-only content based on the current member segment
+ const startMembersOnlyContent = (result.html || '').indexOf('');
+ const startPaywall = result.html.indexOf('');
+ let endPost = result.html.indexOf('');
+
+ if (endPost === -1) {
+ // Default to the end of the HTML (shouldn't happen, but just in case if we have members-only content that should get removed)
+ endPost = result.html.length;
+ }
+
+ // We support the cases where there is no but there is a paywall (in case of bugs)
+ // We also support the case where there is no but there is a (in case of bugs)
+ if (startMembersOnlyContent !== -1 || startPaywall !== -1) {
+ // By default remove the paywall if no memberSegment is passed
+ let memberHasAccess = true;
+
+ if (memberSegment && result.post) {
let statusFilter = memberSegment === 'status:free' ? {status: 'free'} : {status: 'paid'};
const postVisiblity = result.post.visibility;
// For newsletter paywall, specific tiers visibility is considered on par to paid tiers
result.post.visibility = postVisiblity === 'tiers' ? 'paid' : postVisiblity;
- const memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+ memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+ }
- if (!memberHasAccess) {
- const postContentEndIdx = result.html.search(/[\s\n\r]+?/);
- result.html = result.html.slice(0, paywallIndex) + this.renderPaywallCTA(result.post) + result.html.slice(postContentEndIdx);
- result.plaintext = htmlToPlaintext.excerpt(result.html);
+ if (!memberHasAccess) {
+ if (startMembersOnlyContent !== -1) {
+ // Remove the members-only content, but keep the paywall (if there is a paywall)
+ result.html = result.html.slice(0, startMembersOnlyContent) + result.html.slice(startPaywall === -1 ? endPost : startPaywall);
+ }
+ } else {
+ if (startPaywall !== -1) {
+ // Remove the paywall
+ result.html = result.html.slice(0, startPaywall) + result.html.slice(endPost);
}
}
}
@@ -435,7 +475,7 @@ const PostEmailSerializer = {
});
result.html = this.formatHtmlForEmail($.html());
- result.plaintext = htmlToPlaintext.email(result.html);
+ result.plaintext = htmlToPlaintext.email(result.html);
delete result.post;
return result;
diff --git a/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js b/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
index fa0865a7bd..24816b9d6a 100644
--- a/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
+++ b/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
@@ -6,12 +6,17 @@ const urlUtils = require('../../../../../core/shared/url-utils');
const urlService = require('../../../../../core/server/services/url');
const labs = require('../../../../../core/shared/labs');
const {parseReplacements, renderEmailForSegment, serialize, _getTemplateSettings, createUnsubscribeUrl, createPostSignupUrl, _PostEmailSerializer} = require('../../../../../core/server/services/mega/post-email-serializer');
-
+const {HtmlValidate} = require('html-validate');
+
function assertKeys(object, keys) {
assert.deepStrictEqual(Object.keys(object).sort(), keys.sort());
}
describe('Post Email Serializer', function () {
+ afterEach(function () {
+ sinon.restore();
+ });
+
it('creates replacement pattern for valid format and value', function () {
const html = 'Hey %%{first_name}%%, what is up?';
const plaintext = 'Hey %%{first_name}%%, what is up?';
@@ -42,6 +47,10 @@ describe('Post Email Serializer', function () {
});
describe('serialize', function () {
+ afterEach(function () {
+ sinon.restore();
+ });
+
it('should output valid HTML and escape HTML characters in mobiledoc', async function () {
sinon.stub(_PostEmailSerializer, 'serializePostModel').callsFake(async () => {
return {
@@ -66,7 +75,7 @@ describe('Post Email Serializer', function () {
};
const settingsMock = sinon.stub(settingsCache, 'get');
- settingsMock.callsFake(function (key, options) {
+ settingsMock.callsFake((key, options) => {
if (customSettings[key]) {
return customSettings[key];
}
@@ -98,9 +107,6 @@ describe('Post Email Serializer', function () {
const output = await serialize({}, newsletterMock, {isBrowserPreview: false});
- // Test if the email HTML is valid standard HTML5
- const {HtmlValidate} = require('html-validate');
-
const htmlvalidate = new HtmlValidate({
extends: [
'html-validate:document',
@@ -161,6 +167,183 @@ describe('Post Email Serializer', function () {
// Check if the template is rendered fully to the end (to make sure we acutally test all these mobiledocs)
assert.equal(output.html.includes('Heading test <3'), true);
});
+
+ it('output should already contain paywall when there is members-only content', async function () {
+ sinon.stub(_PostEmailSerializer, 'serializePostModel').callsFake(async () => {
+ return {
+ // This is not realistic, but just to test escaping
+ url: 'https://testpost.com/',
+ title: 'This is a test',
+ excerpt: 'This is a test',
+ authors: 'This is a test',
+ feature_image_alt: 'This is a test',
+ feature_image_caption: 'This is a test',
+ visibility: 'tiers',
+
+ // eslint-disable-next-line
+ mobiledoc: JSON.stringify({"version":"0.3.1","atoms":[],"cards":[["paywall",{}]],"markups":[],"sections":[[1,"p",[[0,[],0,"Free content"]]],[10,0],[1,"p",[[0,[],0,"Members only content"]]]],"ghostVersion":"4.0"})
+ };
+ });
+ const customSettings = {
+ accent_color: '#000099',
+ timezone: 'UTC'
+ };
+
+ const settingsMock = sinon.stub(settingsCache, 'get');
+ settingsMock.callsFake((key, options) => {
+ if (customSettings[key]) {
+ return customSettings[key];
+ }
+
+ return settingsMock.wrappedMethod.call(settingsCache, key, options);
+ });
+ const template = {
+ name: 'My newsletter',
+ header_image: '',
+ show_header_icon: true,
+ show_header_title: true,
+ show_feature_image: true,
+ title_font_category: 'sans-serif',
+ title_alignment: 'center',
+ body_font_category: 'serif',
+ show_badge: true,
+ show_header_name: true,
+ // Note: we don't need to check the footer content because this should contain valid HTML (not text)
+ footer_content: 'Footer content with valid HTML'
+ };
+ const newsletterMock = {
+ get: function (key) {
+ return template[key];
+ },
+ toJSON: function () {
+ return template;
+ }
+ };
+
+ const output = await serialize({}, newsletterMock, {isBrowserPreview: false});
+ assert(output.html.includes(''));
+ assert(output.html.includes(''));
+ assert(output.html.includes(''));
+
+ // Paywall content
+ assert(output.html.includes('Subscribe to'));
+ });
+
+ it('output should not contain paywall when there is members-only content but it is a free post', async function () {
+ sinon.stub(_PostEmailSerializer, 'serializePostModel').callsFake(async () => {
+ return {
+ // This is not realistic, but just to test escaping
+ url: 'https://testpost.com/',
+ title: 'This is a test',
+ excerpt: 'This is a test',
+ authors: 'This is a test',
+ feature_image_alt: 'This is a test',
+ feature_image_caption: 'This is a test',
+ visibility: 'members',
+
+ // eslint-disable-next-line
+ mobiledoc: JSON.stringify({"version":"0.3.1","atoms":[],"cards":[["paywall",{}]],"markups":[],"sections":[[1,"p",[[0,[],0,"Free content"]]],[10,0],[1,"p",[[0,[],0,"Members only content"]]]],"ghostVersion":"4.0"})
+ };
+ });
+ const customSettings = {
+ accent_color: '#000099',
+ timezone: 'UTC'
+ };
+
+ const settingsMock = sinon.stub(settingsCache, 'get');
+ settingsMock.callsFake((key, options) => {
+ if (customSettings[key]) {
+ return customSettings[key];
+ }
+
+ return settingsMock.wrappedMethod.call(settingsCache, key, options);
+ });
+ const template = {
+ name: 'My newsletter',
+ header_image: '',
+ show_header_icon: true,
+ show_header_title: true,
+ show_feature_image: true,
+ title_font_category: 'sans-serif',
+ title_alignment: 'center',
+ body_font_category: 'serif',
+ show_badge: true,
+ show_header_name: true,
+ // Note: we don't need to check the footer content because this should contain valid HTML (not text)
+ footer_content: 'Footer content with valid HTML'
+ };
+ const newsletterMock = {
+ get: function (key) {
+ return template[key];
+ },
+ toJSON: function () {
+ return template;
+ }
+ };
+
+ const output = await serialize({}, newsletterMock, {isBrowserPreview: false});
+ assert(output.html.includes(''));
+ assert(!output.html.includes(''));
+ assert(output.html.includes(''));
+ assert(!output.html.includes('Subscribe to'));
+ });
+
+ it('output should not contain paywall if there is no members-only-content', async function () {
+ sinon.stub(_PostEmailSerializer, 'serializePostModel').callsFake(async () => {
+ return {
+ // This is not realistic, but just to test escaping
+ url: 'https://testpost.com/',
+ title: 'This is a test',
+ excerpt: 'This is a test',
+ authors: 'This is a test',
+ feature_image_alt: 'This is a test',
+ feature_image_caption: 'This is a test',
+
+ // eslint-disable-next-line
+ mobiledoc: JSON.stringify({"version":"0.3.1","atoms":[],"cards":[],"markups":[],"sections":[[1,"p",[[0,[],0,"Free content only"]]]],"ghostVersion":"4.0"})
+ };
+ });
+ const customSettings = {
+ accent_color: '#000099',
+ timezone: 'UTC'
+ };
+
+ const settingsMock = sinon.stub(settingsCache, 'get');
+ settingsMock.callsFake(function (key, options) {
+ if (customSettings[key]) {
+ return customSettings[key];
+ }
+
+ return settingsMock.wrappedMethod.call(settingsCache, key, options);
+ });
+ const template = {
+ name: 'My newsletter',
+ header_image: '',
+ show_header_icon: true,
+ show_header_title: true,
+ show_feature_image: true,
+ title_font_category: 'sans-serif',
+ title_alignment: 'center',
+ body_font_category: 'serif',
+ show_badge: true,
+ show_header_name: true,
+ // Note: we don't need to check the footer content because this should contain valid HTML (not text)
+ footer_content: 'Footer content with valid HTML'
+ };
+ const newsletterMock = {
+ get: function (key) {
+ return template[key];
+ },
+ toJSON: function () {
+ return template;
+ }
+ };
+
+ const output = await serialize({}, newsletterMock, {isBrowserPreview: false});
+ assert(output.html.includes(''));
+ assert(!output.html.includes(''));
+ assert(!output.html.includes(''));
+ });
});
describe('renderEmailForSegment', function () {
@@ -216,7 +399,58 @@ describe('Post Email Serializer', function () {
assert.equal(output.plaintext, 'hello');
});
- it('should show paywall content for free members on paid posts', function () {
+ it('should show paywall and hide members-only content for free members on paid posts', function () {
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Members content
Paywall
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, 'status:free');
+ assert.equal(output.html, `Free content
Paywall
`);
+ assert.equal(output.plaintext, `Free content\n\n\nPaywall`);
+ });
+
+ it('should show paywall and hide members-only content for free members on paid posts (without )', function () {
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Members content
Paywall
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, 'status:free');
+ assert.equal(output.html, `Free content
Paywall
`);
+ assert.equal(output.plaintext, `Free content\n\n\nPaywall`);
+ });
+
+ it('should hide members-only content for free members on paid posts (without )', function () {
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Members content
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, 'status:free');
+ assert.equal(output.html, `Free content
`);
+ assert.equal(output.plaintext, `Free content`);
+ });
+
+ it('should hide members-only content for free members on paid posts (without and )', function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
sinon.stub(labs, 'isSet').returns(true);
const email = {
@@ -229,19 +463,11 @@ describe('Post Email Serializer', function () {
};
let output = renderEmailForSegment(email, 'status:free');
- assert(output.html.includes());
- assert(output.html.includes(`Free content
`));
- assert(output.html.includes(`Subscribe to`));
- assert(output.html.includes(`https://site.com/blah/#/portal/signup`));
- assert(!output.html.includes(`Members content
`));
-
- assert(output.plaintext.includes(`Free content`));
- assert(output.plaintext.includes(`Subscribe to`));
- assert(output.plaintext.includes(`https://site.com/blah/#/portal/signup`));
- assert(!output.plaintext.includes(`Members content`));
+ assert.equal(output.html, `Free content
`);
+ assert.equal(output.plaintext, `Free content`);
});
- it('should show full cta for paid members on paid posts', function () {
+ it('should not modify HTML when there are no HTML comments', function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
sinon.stub(labs, 'isSet').returns(true);
const email = {
@@ -249,12 +475,64 @@ describe('Post Email Serializer', function () {
status: 'published',
visibility: 'paid'
},
- html: 'Free content
Members content
',
+ html: 'Free content
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, 'status:free');
+ assert.equal(output.html, `Free content
`);
+ assert.equal(output.plaintext, `Free content`);
+ });
+
+ it('should hide paywall when is missing (paid members)', function () {
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Paywall
',
plaintext: 'Free content. Members content'
};
let output = renderEmailForSegment(email, 'status:-free');
- assert.equal(output.html, `Free content
Members content
`);
+ assert.equal(output.html, `Free content
`);
+ assert.equal(output.plaintext, `Free content`);
+ });
+
+ it('should show members-only content for paid members on paid posts', function () {
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Members content
Paywall
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, 'status:-free');
+ assert.equal(output.html, `Free content
Members content
`);
+ assert.equal(output.plaintext, `Free content\n\nMembers content`);
+ });
+
+ it('should show members-only content for unknown members on paid posts', function () {
+ // Test if the default behaviour is to hide any paywalls and show the members-only content
+ sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+ sinon.stub(labs, 'isSet').returns(true);
+ const email = {
+ post: {
+ status: 'published',
+ visibility: 'paid'
+ },
+ html: 'Free content
Members content
Paywall
',
+ plaintext: 'Free content. Members content'
+ };
+
+ let output = renderEmailForSegment(email, null);
+ assert.equal(output.html, `Free content
Members content
`);
assert.equal(output.plaintext, `Free content\n\nMembers content`);
});
@@ -266,23 +544,16 @@ describe('Post Email Serializer', function () {
status: 'published',
visibility: 'tiers'
},
- html: 'Free content
Members content
',
+ html: 'Free content
Members content
Paywall
',
plaintext: 'Free content. Members content'
};
let output = renderEmailForSegment(email, 'status:free');
- assert(output.html.includes(`Free content
`));
- assert(output.html.includes(`Subscribe to`));
- assert(output.html.includes(`https://site.com/blah/#/portal/signup`));
- assert(!output.html.includes(`Members content
`));
-
- assert(output.plaintext.includes(`Free content`));
- assert(output.plaintext.includes(`Subscribe to`));
- assert(output.plaintext.includes(`https://site.com/blah/#/portal/signup`));
- assert(!output.plaintext.includes(`Members content`));
+ assert.equal(output.html, `Free content
Paywall
`);
+ assert.equal(output.plaintext, `Free content\n\n\nPaywall`);
});
- it('should show full cta for paid members on specific tier posts', function () {
+ it('should show members-only content for paid members on specific tier posts', function () {
sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
sinon.stub(labs, 'isSet').returns(true);
const email = {