From 9c616fe0674279f053a09644f045bbb9e8de3d9d Mon Sep 17 00:00:00 2001
From: Rishabh Garg <zrishabhgarg@gmail.com>
Date: Tue, 26 Jul 2022 19:16:08 +0530
Subject: [PATCH] Added content paywall for newsletters (#15048)

refs TryGhost/Team#1680

- extends the public preview card so that the paywall is shown in newsletters for paid-only posts based on member's access
- adds CTA for paywalled content in newsletters
- the segmentation for paywall only considers free and non-free members, so post with specific `tiers` and `paid-only` access settings are sent to all paid members

Co-authored-by: Djordje Vlaisavljevic <dzvlais@gmail.com>
---
 .../bulk-email/bulk-email-processor.js        |   6 +
 .../server/services/mega/email-preview.js     |   6 +-
 .../services/mega/post-email-serializer.js    |  99 +++++++++++-
 .../server/services/mega/segment-parser.js    |  11 +-
 ghost/core/core/shared/labs.js                |   3 +-
 .../admin/__snapshots__/settings.test.js.snap |   4 +-
 .../mega/post-email-serializer.test.js        | 153 +++++++++++++++++-
 .../services/mega/segment-parser.test.js      |  34 ++++
 8 files changed, 308 insertions(+), 8 deletions(-)

diff --git a/ghost/core/core/server/services/bulk-email/bulk-email-processor.js b/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
index 41edd456a7..183c3c4711 100644
--- a/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
+++ b/ghost/core/core/server/services/bulk-email/bulk-email-processor.js
@@ -7,6 +7,7 @@ const logging = require('@tryghost/logging');
 const models = require('../../models');
 const mailgunProvider = require('./mailgun');
 const sentry = require('../../../shared/sentry');
+const labs = require('../../../shared/labs');
 const debug = require('@tryghost/debug')('mega');
 const postEmailSerializer = require('../mega/post-email-serializer');
 
@@ -170,6 +171,11 @@ module.exports = {
             // Load newsletter data on email
             await emailBatchModel.relations.email.getLazyRelation('newsletter', {require: false, ...knexOptions});
 
+            if (labs.isSet('newsletterPaywall')) {
+                // Load post data on email - for content gating on paywall
+                await emailBatchModel.relations.email.getLazyRelation('post', {require: false, ...knexOptions});
+            }
+
             // send the email
             const sendResponse = await this.send(emailBatchModel.relations.email.toJSON(), recipientRows, memberSegment);
 
diff --git a/ghost/core/core/server/services/mega/email-preview.js b/ghost/core/core/server/services/mega/email-preview.js
index 17e6327f17..b0d69e2599 100644
--- a/ghost/core/core/server/services/mega/email-preview.js
+++ b/ghost/core/core/server/services/mega/email-preview.js
@@ -36,7 +36,11 @@ class EmailPreview {
             );
         });
 
-        return emailContent;
+        return {
+            subject: emailContent.subject,
+            html: emailContent.html,
+            plaintext: emailContent.plaintext
+        };
     }
 }
 
diff --git a/ghost/core/core/server/services/mega/post-email-serializer.js b/ghost/core/core/server/services/mega/post-email-serializer.js
index db5cf8a748..296a810fbe 100644
--- a/ghost/core/core/server/services/mega/post-email-serializer.js
+++ b/ghost/core/core/server/services/mega/post-email-serializer.js
@@ -2,15 +2,18 @@ const _ = require('lodash');
 const template = require('./template');
 const settingsCache = require('../../../shared/settings-cache');
 const urlUtils = require('../../../shared/url-utils');
+const labs = require('../../../shared/labs');
 const moment = require('moment-timezone');
 const api = require('../../api').endpoints;
 const apiShared = require('../../api').shared;
 const {URL} = require('url');
 const mobiledocLib = require('../../lib/mobiledoc');
 const htmlToPlaintext = require('../../../shared/html-to-plaintext');
+const membersService = require('../members');
 const {isUnsplashImage, isLocalContentImage} = require('@tryghost/kg-default-cards/lib/utils');
 const {textColorForBackgroundColor, darkenToContrastThreshold} = require('@tryghost/color-utils');
 const logging = require('@tryghost/logging');
+const urlService = require('../../services/url');
 
 const ALLOWED_REPLACEMENTS = ['first_name'];
 
@@ -74,6 +77,27 @@ const createUnsubscribeUrl = (uuid, newsletterUuid) => {
     return unsubscribeUrl.href;
 };
 
+/**
+ * createPostSignupUrl
+ *
+ * Takes a post object. Returns the url that should be used to signup from newsletter
+ *
+ * @param {Object} post post object
+ */
+const createPostSignupUrl = (post) => {
+    let url = urlService.getUrlByResourceId(post.id, {absolute: true});
+
+    // For email-only posts, use site url as base
+    if (post.status !== 'published' && url.match(/\/404\//)) {
+        url = urlUtils.getSiteUrl();
+    }
+
+    const signupUrl = new URL(url);
+    signupUrl.hash = `/portal/signup`;
+
+    return signupUrl.href;
+};
+
 // NOTE: serialization is needed to make sure we do post transformations such as image URL transformation from relative to absolute
 const serializePostModel = async (model) => {
     // fetch mobiledoc rather than html and plaintext so we can render email-specific contents
@@ -297,18 +321,87 @@ const serialize = async (postModel, newsletter, options = {isBrowserPreview: fal
         html: formatHtmlForEmail(htmlTemplate),
         plaintext: post.plaintext
     });
-
-    return {
+    const data = {
         subject: post.email_subject || post.title,
         html,
         plaintext
     };
+    if (labs.isSet('newsletterPaywall')) {
+        data.post = post;
+    }
+    return data;
 };
 
+/**
+ * renderPaywallCTA
+ *
+ * outputs html for rendering paywall CTA in newsletter
+ *
+ * @param {Object} post Post Object
+ */
+
+function renderPaywallCTA(post) {
+    const accentColor = settingsCache.get('accent_color');
+    const siteTitle = settingsCache.get('title') || 'Ghost';
+    const signupUrl = createPostSignupUrl(post);
+
+    return `<div class="align-center" style="text-align: center;">
+    <hr
+        style="position: relative; display: block; width: 100%; margin: 3em 0; padding: 0; height: 1px; border: 0; border-top: 1px solid #e5eff5;">
+    <h2
+        style="margin-top: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; line-height: 1.11em; font-weight: 700; text-rendering: optimizeLegibility; margin: 1.5em 0 0.5em 0; font-size: 26px;">
+        Subscribe to continue reading.</h2>
+    <p style="margin: 0 auto 1.5em auto; line-height: 1.6em; max-width: 480px;">Become a paid member of ${siteTitle} to get access to all
+        subscriber-only content.</p>
+    <div class="btn btn-accent" style="box-sizing: border-box; width: 100%; display: table;">
+        <table border="0" cellspacing="0" cellpadding="0" align="center"
+            style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: auto;">
+            <tbody>
+                <tr>
+                    <td align="center"
+                        style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; vertical-align: top; text-align: center; border-radius: 5px;"
+                        valign="top" bgcolor="${accentColor}">
+                        <a href="${signupUrl}"
+                            style="overflow-wrap: anywhere; border: solid 1px #3498db; border-radius: 5px; box-sizing: border-box; cursor: pointer; display: inline-block; font-size: 14px; font-weight: bold; margin: 0; padding: 12px 25px; text-decoration: none; background-color: ${accentColor}; border-color: ${accentColor}; color: #FFFFFF;"
+                            target="_blank">Subscribe
+                        </a>
+                    </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    <p style="margin: 0 0 1.5em 0; line-height: 1.6em;"></p>
+</div>`;
+}
+
 function renderEmailForSegment(email, memberSegment) {
     const cheerio = require('cheerio');
 
     const result = {...email};
+
+    /** Checks and hides content for newsletter behind paywall card
+     *  based on member's status and post access
+     *  Adds CTA in case content is hidden.
+    */
+    if (labs.isSet('newsletterPaywall')) {
+        const paywallIndex = (result.html || '').indexOf('<!--members-only-->');
+        if (paywallIndex !== -1 && memberSegment) {
+            let statusFilter = memberSegment === 'status:free' ? {status: 'free'} : {status: 'paid'};
+            const postVisiblity = result.post.visibility;
+
+            // For newsletter paywall, specific tiers visibility is considered on par to paid tiers
+            result.post.visibility = postVisiblity === 'tiers' ? 'paid' : postVisiblity;
+
+            const memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+
+            if (!memberHasAccess) {
+                const postContentEndIdx = result.html.search(/[\s\n\r]+?<!-- POST CONTENT END -->/);
+                result.html = result.html.slice(0, paywallIndex) + renderPaywallCTA(result.post) + result.html.slice(postContentEndIdx);
+                result.plaintext = htmlToPlaintext.excerpt(result.html);
+            }
+        }
+    }
+
     const $ = cheerio.load(result.html);
 
     $('[data-gh-segment]').get().forEach((node) => {
@@ -322,6 +415,7 @@ function renderEmailForSegment(email, memberSegment) {
 
     result.html = formatHtmlForEmail($.html());
     result.plaintext = htmlToPlaintext.email(result.html);
+    delete result.post;
 
     return result;
 }
@@ -329,6 +423,7 @@ function renderEmailForSegment(email, memberSegment) {
 module.exports = {
     serialize,
     createUnsubscribeUrl,
+    createPostSignupUrl,
     renderEmailForSegment,
     parseReplacements,
     // Export for tests
diff --git a/ghost/core/core/server/services/mega/segment-parser.js b/ghost/core/core/server/services/mega/segment-parser.js
index d89b416a04..603526fbcd 100644
--- a/ghost/core/core/server/services/mega/segment-parser.js
+++ b/ghost/core/core/server/services/mega/segment-parser.js
@@ -1,11 +1,20 @@
+const labs = require('../../../shared/labs');
+
 const getSegmentsFromHtml = (html) => {
     const cheerio = require('cheerio');
     const $ = cheerio.load(html);
 
-    const allSegments = $('[data-gh-segment]')
+    let allSegments = $('[data-gh-segment]')
         .get()
         .map(el => el.attribs['data-gh-segment']);
 
+    /**
+     * Always add free and paid segments if email has paywall card
+     */
+    if (labs.isSet('newsletterPaywall') && html.indexOf('<!--members-only-->') !== -1) {
+        allSegments = allSegments.concat(['status:free', 'status:-free']);
+    }
+
     // only return unique elements
     return [...new Set(allSegments)];
 };
diff --git a/ghost/core/core/shared/labs.js b/ghost/core/core/shared/labs.js
index f260fbd795..a7e9188357 100644
--- a/ghost/core/core/shared/labs.js
+++ b/ghost/core/core/shared/labs.js
@@ -25,7 +25,8 @@ const BETA_FEATURES = [
 const ALPHA_FEATURES = [
     'urlCache',
     'beforeAfterCard',
-    'comments'
+    'comments',
+    'newsletterPaywall'
 ];
 
 module.exports.GA_KEYS = [...GA_FEATURES];
diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/settings.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/settings.test.js.snap
index 7727c5620c..82c45d6240 100644
--- a/ghost/core/test/e2e-api/admin/__snapshots__/settings.test.js.snap
+++ b/ghost/core/test/e2e-api/admin/__snapshots__/settings.test.js.snap
@@ -230,7 +230,7 @@ Object {
     },
     Object {
       "key": "labs",
-      "value": "{\\"activitypub\\":true,\\"urlCache\\":true,\\"beforeAfterCard\\":true,\\"comments\\":true,\\"members\\":true}",
+      "value": "{\\"activitypub\\":true,\\"urlCache\\":true,\\"beforeAfterCard\\":true,\\"comments\\":true,\\"newsletterPaywall\\":true,\\"members\\":true}",
     },
     Object {
       "key": "slack_url",
@@ -280,7 +280,7 @@ exports[`Settings API Browse Can request all settings 2: [headers] 1`] = `
 Object {
   "access-control-allow-origin": "http://127.0.0.1:2369",
   "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
-  "content-length": "3379",
+  "content-length": "3406",
   "content-type": "application/json; charset=utf-8",
   "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
   "vary": "Origin, Accept-Encoding",
diff --git a/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js b/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
index efb4fe449f..6d3425569b 100644
--- a/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
+++ b/ghost/core/test/unit/server/services/mega/post-email-serializer.test.js
@@ -3,8 +3,10 @@ const sinon = require('sinon');
 const settingsCache = require('../../../../../core/shared/settings-cache');
 const models = require('../../../../../core/server/models');
 const urlUtils = require('../../../../../core/shared/url-utils');
+const urlService = require('../../../../../core/server/services/url');
+const labs = require('../../../../../core/shared/labs');
 
-const {parseReplacements, renderEmailForSegment, _getTemplateSettings, createUnsubscribeUrl} = require('../../../../../core/server/services/mega/post-email-serializer');
+const {parseReplacements, renderEmailForSegment, _getTemplateSettings, createUnsubscribeUrl, createPostSignupUrl} = require('../../../../../core/server/services/mega/post-email-serializer');
 
 describe('Post Email Serializer', function () {
     it('creates replacement pattern for valid format and value', function () {
@@ -37,6 +39,10 @@ describe('Post Email Serializer', function () {
     });
 
     describe('renderEmailForSegment', function () {
+        afterEach(function () {
+            sinon.restore();
+        });
+
         it('shouldn\'t change an email that has no member segment', function () {
             const email = {
                 otherProperty: true,
@@ -84,6 +90,122 @@ describe('Post Email Serializer', function () {
             output.html.should.equal('hello');
             output.plaintext.should.equal('hello');
         });
+
+        it('should show paywall content for free members on paid posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'paid'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:free');
+            output.html.should.containEql(`<p>Free content</p>`);
+            output.html.should.containEql(`Subscribe to continue reading`);
+            output.html.should.containEql(`https://site.com/blah/#/portal/signup`);
+            output.html.should.not.containEql(`<p>Members content</p>`);
+
+            output.plaintext.should.containEql(`Free content`);
+            output.plaintext.should.containEql(`Subscribe to continue reading`);
+            output.plaintext.should.containEql(`https://site.com/blah/#/portal/signup`);
+            output.plaintext.should.not.containEql(`Members content`);
+        });
+
+        it('should show full cta for paid members on paid posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'paid'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:-free');
+            output.html.should.equal(`<p>Free content</p><!--members-only--><p>Members content</p>`);
+            output.plaintext.should.equal(`Free content\n\nMembers content`);
+        });
+
+        it('should show paywall content for free members on specific tier posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'tiers'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:free');
+            output.html.should.containEql(`<p>Free content</p>`);
+            output.html.should.containEql(`Subscribe to continue reading`);
+            output.html.should.containEql(`https://site.com/blah/#/portal/signup`);
+            output.html.should.not.containEql(`<p>Members content</p>`);
+
+            output.plaintext.should.containEql(`Free content`);
+            output.plaintext.should.containEql(`Subscribe to continue reading`);
+            output.plaintext.should.containEql(`https://site.com/blah/#/portal/signup`);
+            output.plaintext.should.not.containEql(`Members content`);
+        });
+
+        it('should show full cta for paid members on specific tier posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'paid'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:-free');
+            output.html.should.equal(`<p>Free content</p><!--members-only--><p>Members content</p>`);
+            output.plaintext.should.equal(`Free content\n\nMembers content`);
+        });
+
+        it('should show full content for free members on free posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'public'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:free');
+            output.html.should.equal(`<p>Free content</p><!--members-only--><p>Members content</p>`);
+            output.plaintext.should.equal(`Free content\n\nMembers content`);
+        });
+
+        it('should show full content for paid members on free posts', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            sinon.stub(labs, 'isSet').returns(true);
+            const email = {
+                post: {
+                    status: 'published',
+                    visibility: 'public'
+                },
+                html: '<p>Free content</p><!--members-only--><p>Members content</p>',
+                plaintext: 'Free content. Members content'
+            };
+
+            let output = renderEmailForSegment(email, 'status:-free');
+            output.html.should.equal(`<p>Free content</p><!--members-only--><p>Members content</p>`);
+            output.plaintext.should.equal(`Free content\n\nMembers content`);
+        });
     });
 
     describe('createUnsubscribeUrl', function () {
@@ -114,6 +236,35 @@ describe('Post Email Serializer', function () {
         });
     });
 
+    describe('createPostSignupUrl', function () {
+        before(function () {
+            models.init();
+        });
+
+        afterEach(function () {
+            sinon.restore();
+        });
+
+        it('generates signup url on post for published post', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/blah/');
+            const unsubscribeUrl = createPostSignupUrl({
+                status: 'published',
+                id: 'abc123'
+            });
+            unsubscribeUrl.should.eql('https://site.com/blah/#/portal/signup');
+        });
+
+        it('generates signup url on homepage for email only post', function () {
+            sinon.stub(urlService, 'getUrlByResourceId').returns('https://site.com/test/404/');
+            sinon.stub(urlUtils, 'getSiteUrl').returns('https://site.com/test/');
+            const unsubscribeUrl = createPostSignupUrl({
+                status: 'sent',
+                id: 'abc123'
+            });
+            unsubscribeUrl.should.eql('https://site.com/test/#/portal/signup');
+        });
+    });
+
     describe('getTemplateSettings', function () {
         before(function () {
             models.init();
diff --git a/ghost/core/test/unit/server/services/mega/segment-parser.test.js b/ghost/core/test/unit/server/services/mega/segment-parser.test.js
index 2c64da2217..5a4a42c089 100644
--- a/ghost/core/test/unit/server/services/mega/segment-parser.test.js
+++ b/ghost/core/test/unit/server/services/mega/segment-parser.test.js
@@ -1,8 +1,14 @@
 const should = require('should');
+const sinon = require('sinon');
+const labs = require('../../../../../core/shared/labs');
 
 const {getSegmentsFromHtml} = require('../../../../../core/server/services/mega/segment-parser');
 
 describe('MEGA: Segment Parser', function () {
+    afterEach(function () {
+        sinon.restore();
+    });
+
     it('extracts a single segments used in HTML', function () {
         const html = '<div data-gh-segment="status:-free"><p>Plain html with no replacements</p></div>';
 
@@ -41,6 +47,34 @@ describe('MEGA: Segment Parser', function () {
         segments[1].should.equal('status:free');
     });
 
+    it('extracts all segments for paywalled content', function () {
+        sinon.stub(labs, 'isSet').returns(true);
+
+        const html = '<p>Free content</p><!--members-only--><p>Members content</p>';
+
+        const segments = getSegmentsFromHtml(html);
+
+        segments.length.should.equal(2);
+        segments[0].should.equal('status:free');
+        segments[1].should.equal('status:-free');
+    });
+
+    it('extracts all unique segments including paywalled content', function () {
+        sinon.stub(labs, 'isSet').returns(true);
+        const html = `
+            <div data-gh-segment="status:-free"><p>Text for paid</p></div>
+            <div data-gh-segment="status:free"><p>Text for free</p></div>
+            <div data-gh-segment="status:-free"><p>Another message for paid member</p></div>
+            <p>Free content</p><!--members-only--><p>Members content</p>
+        `;
+
+        const segments = getSegmentsFromHtml(html);
+
+        segments.length.should.equal(2);
+        segments[0].should.equal('status:-free');
+        segments[1].should.equal('status:free');
+    });
+
     it('extracts no segments from HTML', function () {
         const html = '<div data-gh-somethingelse="status:-free"><p>Plain html with no replacements</p></div>';