Added new hidden comments API implementation (#21444)

ref PLG-227

- Behind flags
- Changed Comments API for members and guests to not return hidden or
removed comments - with the only exception being if a hidden or removed
comment have published replies, in which case it will be greyed out as
per the previous version on the UI.
- Wired up a new admin API endpoint for comment to receive all comments.
It's on par with the members / guests endpoint, with the difference
being that it it shows hidden comment's content, where previously the
html property was nullified.

apps/comments-ui/src/components/content/Comment.tsx

@@ -107,23 +107,25 @@ type UnpublishedCommentProps = {
     openEditMode: () => void;
 }
 const UnpublishedComment: React.FC<UnpublishedCommentProps> = ({comment, openEditMode}) => {
-    const {admin, t} = useAppContext();
+    const {t} = useAppContext();
-
+    let notPublishedMessage:string = '';
-    let notPublishedMessage;
-    if (admin && comment.status === 'hidden') {
-        notPublishedMessage = t('This comment has been hidden.');
-    } else {
-        notPublishedMessage = t('This comment has been removed.');
-    }
 
     const avatar = (<BlankAvatar />);
     const hasReplies = comment.replies && comment.replies.length > 0;
 
+    if (comment.status === 'hidden') {
+        notPublishedMessage = t('This comment has been hidden.');
+    } else if (comment.status === 'deleted') {
+        notPublishedMessage = t('This comment has been removed.');
+    }
+
     return (
         <CommentLayout avatar={avatar} hasReplies={hasReplies}>
             <div className="mt-[-3px] flex items-start">
                 <div className="flex h-10 flex-row items-center gap-4 pb-[8px] pr-4">
-                    <p className="text-md mt-[4px] font-sans italic leading-normal text-black/20 sm:text-lg dark:text-white/35">{notPublishedMessage}</p>
+                    <p className="text-md mt-[4px] font-sans italic leading-normal text-black/20 sm:text-lg dark:text-white/35">
+                        {notPublishedMessage}
+                    </p>
                     <div className="mt-[4px]">
                         <MoreButton comment={comment} toggleEdit={openEditMode} />
                     </div>

apps/comments-ui/src/components/content/Content.tsx

@@ -14,10 +14,12 @@ const Content = () => {
 
     const {pagination, member, comments, commentCount, commentsEnabled, title, showCount, secundaryFormCount} = useAppContext();
     let commentsElements;
+    const commentsDataset = comments;
+
     if (labs && labs.commentImprovements) {
-        commentsElements = comments.slice().map(comment => <Comment key={comment.id} comment={comment} />);
+        commentsElements = commentsDataset.slice().map(comment => <Comment key={comment.id} comment={comment} />);
     } else {
-        commentsElements = comments.slice().reverse().map(comment => <Comment key={comment.id} comment={comment} />);
+        commentsElements = commentsDataset.slice().reverse().map(comment => <Comment key={comment.id} comment={comment} />);
     }
 
     useEffect(() => {

apps/comments-ui/test/e2e/actions.test.ts

@@ -107,7 +107,9 @@ test.describe('Actions', async () => {
         await expect(frame.getByText('This is a reply 123')).toHaveCount(1);
     });
 
-    test('Reply-to-reply action not shown without labs flag', async ({page}) => {
+    test('Reply-to-reply action not shown without labs flag', async ({
+        page
+    }) => {
         mockedApi.addComment({
             html: '<p>This is comment 1</p>',
             replies: [
@@ -193,11 +195,15 @@ test.describe('Actions', async () => {
         const profileModal = detailsFrame.getByTestId('profile-modal');
         await expect(profileModal).toBeVisible();
 
-        await expect(detailsFrame.getByTestId('name-input')).toHaveValue('John Doe');
+        await expect(detailsFrame.getByTestId('name-input')).toHaveValue(
+            'John Doe'
+        );
         await expect(detailsFrame.getByTestId('expertise-input')).toHaveValue('');
 
         await detailsFrame.getByTestId('name-input').fill('Testy McTest');
-        await detailsFrame.getByTestId('expertise-input').fill('Software development');
+        await detailsFrame
+            .getByTestId('expertise-input')
+            .fill('Software development');
 
         await detailsFrame.getByTestId('save-button').click();
 
@@ -209,7 +215,9 @@ test.describe('Actions', async () => {
         await waitEditorFocused(editor);
 
         await expect(frame.getByTestId('member-name')).toHaveText('Testy McTest');
-        await expect(frame.getByTestId('expertise-button')).toHaveText('·Software development');
+        await expect(frame.getByTestId('expertise-button')).toHaveText(
+            '·Software development'
+        );
     });
 
     test.describe('Sorting - flag needs to be enabled', () => {
@@ -292,7 +300,9 @@ test.describe('Actions', async () => {
 
             await expect(comments.nth(0)).toContainText('This is comment 3');
         });
-        test('Renders Sorting Form dropdown, with Best, Newest Oldest', async ({page}) => {
+        test('Renders Sorting Form dropdown, with Best, Newest Oldest', async ({
+            page
+        }) => {
             mockedApi.addComment({
                 html: '<p>This is comment 1</p>'
             });
@@ -330,7 +340,9 @@ test.describe('Actions', async () => {
 
             await sortingForm.click();
 
-            const sortingDropdown = frame.getByTestId('comments-sorting-form-dropdown');
+            const sortingDropdown = frame.getByTestId(
+                'comments-sorting-form-dropdown'
+            );
             await expect(sortingDropdown).toBeVisible();
 
             // check if inner options are visible
@@ -370,7 +382,9 @@ test.describe('Actions', async () => {
 
             await sortingForm.click();
 
-            const sortingDropdown = await frame.getByTestId('comments-sorting-form-dropdown');
+            const sortingDropdown = await frame.getByTestId(
+                'comments-sorting-form-dropdown'
+            );
 
             const newestOption = await sortingDropdown.getByText('Newest');
             await newestOption.click();
@@ -407,7 +421,9 @@ test.describe('Actions', async () => {
 
             await sortingForm.click();
 
-            const sortingDropdown = await frame.getByTestId('comments-sorting-form-dropdown');
+            const sortingDropdown = await frame.getByTestId(
+                'comments-sorting-form-dropdown'
+            );
 
             const newestOption = await sortingDropdown.getByText('Oldest');
             await newestOption.click();

apps/comments-ui/test/utils/MockedApi.ts

@@ -6,14 +6,16 @@ export class MockedApi {
     postId: string;
     member: any;
     settings: any;
+    members: any[];
 
     #lastCommentDate = new Date('2021-01-01T00:00:00.000Z');
 
-    constructor({postId = 'ABC', comments = [], member = undefined, settings = {}}: {postId?: string, comments?: any[], member?: any, settings?: any}) {
+    constructor({postId = 'ABC', comments = [], member = undefined, settings = {}, members = []}: {postId?: string, comments?: any[], member?: any, settings?: any, members?: any[]}) {
         this.postId = postId;
         this.comments = comments;
         this.member = member;
         this.settings = settings;
+        this.members = [];
     }
 
     addComment(overrides: any = {}) {
@@ -47,10 +49,20 @@ export class MockedApi {
         }
     }
 
+    createMember(overrides) {
+        const newMember = buildMember(overrides);
+        this.members.push(newMember);
+        return newMember;
+    }
+
     setMember(overrides) {
         this.member = buildMember(overrides);
     }
 
+    logoutMember() {
+        this.member = null;
+    }
+
     setSettings(overrides) {
         this.settings = buildSettings(overrides);
     }

ghost/core/core/frontend/src/admin-auth/message-handler.js

@@ -23,6 +23,18 @@ window.addEventListener('message', async function (event) {
         }), siteOrigin);
     }
 
+    if (data.action === 'browseComments') {
+        try {
+            const res = await fetch(
+                adminUrl + '/comments/?limit=50&order=created_at%20desc'
+            );
+            const json = await res.json();
+            respond(null, json);
+        } catch (err) {
+            respond(err, null);
+        }
+    }
+
     if (data.action === 'getUser') {
         try {
             const res = await fetch(

ghost/core/core/server/api/endpoints/comments.js

@@ -1,4 +1,5 @@
 const models = require('../../models');
+const commentsService = require('../../services/comments');
 
 function handleCacheHeaders(model, frame) {
     if (model) {
@@ -39,6 +40,33 @@ const controller = {
 
             handleCacheHeaders(result, frame);
 
+            return result;
+        }
+    },
+    browse: {
+        headers: {
+            cacheInvalidate: false
+        },
+        options: [
+            'post_id',
+            'include',
+            'page',
+            'limit',
+            'fields',
+            'filter',
+            'order',
+            'debug'
+        ],
+        validation: {
+            options: {
+                post_id: {
+                    required: true
+                }
+            }
+        },
+        permissions: true,
+        async query(frame) {
+            const result = await commentsService.controller.adminBrowse(frame);
             return result;
         }
     }

ghost/core/core/server/models/comment.js

@@ -3,6 +3,7 @@ const _ = require('lodash');
 const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 const {ValidationError} = require('@tryghost/errors');
+const labs = require('../../shared/labs');
 
 const messages = {
     emptyComment: 'The body of a comment cannot be empty',
@@ -60,6 +61,44 @@ const Comment = ghostBookshelf.Model.extend({
             // Note: this limit is not working
             .query('limit', 3);
     },
+    customQuery(qb) {
+        qb.where(function () {
+            this.whereNotIn('comments.status', ['hidden', 'deleted'])
+                .orWhereExists(function () {
+                    this.select(1)
+                        .from('comments as replies')
+                        .whereRaw('replies.parent_id = comments.id')
+                        .whereNotIn('replies.status', ['hidden', 'deleted']);
+                });
+        });
+    },
+
+    adminCustomQuery(qb) {
+        qb.where(function () {
+            this.whereNotIn('comments.status', ['deleted'])
+                .orWhereExists(function () {
+                    this.select(1)
+                        .from('comments as replies')
+                        .whereRaw('replies.parent_id = comments.id')
+                        .whereNotIn('replies.status', ['deleted']);
+                });
+        });
+    },
+
+    applyCustomQuery(options) {
+        if (labs.isSet('commentImprovements')) {
+            if (!options.isAdmin) { // if it's an admin request, we don't need to apply the custom query
+                this.query((qb) => {
+                    this.customQuery(qb, options);
+                });
+            }
+            if (options.isAdmin) {
+                this.query((qb) => {
+                    this.adminCustomQuery(qb, options);
+                });
+            }
+        }
+    },
 
     emitChange: function emitChange(event, options) {
         const eventToTrigger = 'comment' + '.' + event;
@@ -122,6 +161,7 @@ const Comment = ghostBookshelf.Model.extend({
         return null;
     }
 }, {
+
     destroy: function destroy(unfilteredOptions) {
         let options = this.filterOptions(unfilteredOptions, 'destroy', {extraAllowedProperties: ['id']});
 
@@ -222,11 +262,9 @@ const Comment = ghostBookshelf.Model.extend({
             'replies.count.liked'
         ].filter(relation => withRelated.includes(relation));
         const result = await ghostBookshelf.Model.findPage.call(this, options);
-
         for (const model of result.data) {
             await model.load(relationsToLoadIndividually, _.omit(options, 'withRelated'));
         }
-
         return result;
     },
 
@@ -276,6 +314,7 @@ const Comment = ghostBookshelf.Model.extend({
 
         // The comment model additionally supports having a parentId option
         options.push('parentId');
+        options.push('isAdmin');
 
         return options;
     }

ghost/core/core/server/services/comments/CommentsController.js

@@ -51,10 +51,30 @@ module.exports = class CommentsController {
                 frame.options.filter = `post_id:${frame.options.post_id}`;
             }
         }
-
         return await this.service.getComments(frame.options);
     }
 
+    async adminBrowse(frame) {
+        if (frame.options.post_id) {
+            if (frame.options.filter) {
+                frame.options.mongoTransformer = function (query) {
+                    return {
+                        $and: [
+                            {
+                                post_id: frame.options.post_id
+                            },
+                            query
+                        ]
+                    };
+                };
+            } else {
+                frame.options.filter = `post_id:${frame.options.post_id}`;
+            }
+        }
+        frame.options.isAdmin = true;
+        return await this.service.getAdminComments(frame.options);
+    }
+
     /**
      * @param {Frame} frame
      */

ghost/core/core/server/services/comments/CommentsService.js

@@ -176,6 +176,13 @@ class CommentsService {
         return page;
     }
 
+    async getAdminComments(options) {
+        this.checkEnabled();
+        const page = await this.models.Comment.findPage({...options, parentId: null});
+
+        return page;
+    }
+
     /**
      * @param {string} id - The ID of the Comment to get replies from
      * @param {any} options

ghost/core/core/server/web/api/endpoints/admin/routes.js

@@ -51,6 +51,7 @@ module.exports = function apiRoutes() {
 
     router.get('/mentions', mw.authAdminApi, http(api.mentions.browse));
 
+    router.get('/comments/post/:post_id', mw.authAdminApi, http(api.comments.browse));
     router.put('/comments/:id', mw.authAdminApi, http(api.comments.edit));
 
     // ## Pages

ghost/core/test/e2e-api/admin/comments.test.js

@@ -4,8 +4,67 @@ const {
     fixtureManager,
     mockManager
 } = require('../../utils/e2e-framework');
+const models = require('../../../core/server/models');
+let postId;
+const dbFns = {
+    /**
+     * @typedef {Object} AddCommentData
+     * @property {string} [post_id=post_id]
+     * @property {string} member_id
+     * @property {string} [parent_id]
+     * @property {string} [html='This is a comment']
+     * @property {string} [status='published']
+     */
+    /**
+     * @typedef {Object} AddCommentReplyData
+     * @property {string} member_id
+     * @property {string} [html='This is a reply']
+     * @property {date} [created_at]
+     */
+    /**
+     * @typedef {AddCommentData & {replies: AddCommentReplyData[]}} AddCommentWithRepliesData
+     */
 
-describe('Comments API', function () {
+    /**
+     * @param {AddCommentData} data
+     * @returns {Promise<any>}
+     */
+    addComment: async (data) => {
+        return await models.Comment.add({
+            post_id: data.post_id || postId,
+            member_id: data.member_id,
+            parent_id: data.parent_id,
+            html: data.html || '<p>This is a comment</p>',
+            created_at: data.created_at,
+            status: data.status || 'published'
+        });
+    },
+    /**
+     * @param {AddCommentWithRepliesData}  data
+     * @returns {Promise<any>}
+     */
+    addCommentWithReplies: async (data) => {
+        const {replies, ...commentData} = data;
+
+        const parent = await dbFns.addComment(commentData);
+        const createdReplies = [];
+
+        for (const reply of replies) {
+            const createdReply = await dbFns.addComment({
+                post_id: parent.get('post_id'),
+                member_id: reply.member_id,
+                parent_id: parent.get('id'),
+                html: reply.html || '<p>This is a reply</p>',
+                status: reply.status || 'published'
+            });
+            createdReplies.push(createdReply);
+        }
+
+        return {parent, replies: createdReplies};
+    }
+};
+
+describe('Admin Comments API', function () {
     let adminApi;
     let membersApi;
 
@@ -84,4 +143,103 @@ describe('Comments API', function () {
             });
         });
     });
+
+    describe('browse', function () {
+        it('Can browse comments as an admin', async function () {
+            const post = fixtureManager.get('posts', 1);
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 1',
+                status: 'published'
+            });
+
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 2',
+                status: 'published'
+            });
+            const res = await adminApi.get('/comments/post/' + post.id + '/');
+            assert.equal(res.body.comments.length, 2);
+        });
+
+        it('Does not return deleted comments, but returns hidden comments', async function () {
+            const post = fixtureManager.get('posts', 1);
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 1',
+                status: 'deleted'
+            });
+
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 2',
+                status: 'hidden'
+            });
+            const res = await adminApi.get('/comments/post/' + post.id + '/');
+            // check that there is no deleted comments by looping through the returned comments
+            for (const comment of res.body.comments) {
+                assert.notEqual(comment.status, 'deleted');
+            }
+        });
+
+        it('Returns deleted comments if is has hidden or published replies', async function () {
+            const post = fixtureManager.get('posts', 1);
+            await dbFns.addCommentWithReplies({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 1',
+                status: 'deleted',
+                replies: [
+                    {
+                        member_id: fixtureManager.get('members', 0).id,
+                        html: 'Reply 1',
+                        status: 'published'
+                    }
+                ]
+            });
+
+            const res = await adminApi.get('/comments/post/' + post.id + '/');
+
+            // find deleted comment
+            const deletedComment = res.body.comments.find(comment => comment.status === 'deleted');
+
+            assert.equal(deletedComment.html, 'Comment 1');
+
+            const publishedReply = res.body.comments.find(comment => comment.id === deletedComment.id).replies?.find(reply => reply.status === 'published');
+
+            assert.equal(publishedReply.html, 'Reply 1');
+        });
+
+        it('Does show HTML of deleted and hidden comments since we are admin', async function () {
+            const post = fixtureManager.get('posts', 1);
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 1',
+                status: 'deleted'
+            });
+
+            await dbFns.addComment({
+                member_id: fixtureManager.get('members', 0).id,
+                post_id: post.id,
+                html: 'Comment 2',
+                status: 'hidden'
+            });
+            const res = await adminApi.get('/comments/post/' + post.id + '/');
+
+            const deletedComment = res.body.comments.find(comment => comment.status === 'deleted');
+            assert.equal(deletedComment.html, 'Comment 1');
+
+            const hiddenComment = res.body.comments.find(comment => comment.status === 'hidden');
+            assert.equal(hiddenComment.html, 'Comment 2');
+            // console.log(res.body);
+            // assert.equal(res.body.comments[0].html, 'Comment 2');
+
+            // assert.equal(res.body.comments[1].html, 'Comment 1');
+        });
+    });
 });

ghost/core/test/e2e-api/members-comments/comments.test.js

@@ -398,6 +398,125 @@ describe('Comments API', function () {
                 ]);
             });
 
+            it('excludes hidden comments', async function () {
+                const hiddenComment = await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    html: 'This is a hidden comment',
+                    status: 'hidden'
+                });
+
+                const data2 = await membersAgent
+                    .get(`/api/comments/post/${postId}/`)
+                    .expectStatus(200);
+
+                // check that hiddenComment.id is not in the response
+                should(data2.body.comments.map(c => c.id)).not.containEql(hiddenComment.id);
+                should(data2.body.comments.length).eql(0);
+            });
+
+            it('excludes deleted comments', async function () {
+                // await mockManager.mockLabsEnabled('commentImprovements');
+                await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    html: 'This is a deleted comment',
+                    status: 'deleted'
+                });
+
+                const data2 = await membersAgent
+                    .get(`/api/comments/post/${postId}/`)
+                    .expectStatus(200);
+
+                // go through all comments and check if the deleted comment is not there
+                data2.body.comments.forEach((comment) => {
+                    should(comment.html).not.eql('This is a deleted comment');
+                });
+
+                data2.body.comments.length.should.eql(0);
+            });
+
+            it('shows hidden and deleted comment where there is a reply', async function () {
+                await mockManager.mockLabsEnabled('commentImprovements');
+                await setupBrowseCommentsData();
+                const hiddenComment = await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    html: 'This is a hidden comment',
+                    status: 'hidden'
+                });
+
+                const deletedComment = await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    html: 'This is a deleted comment',
+                    status: 'deleted'
+                });
+
+                await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    parent_id: hiddenComment.get('id'),
+                    html: 'This is a reply to a hidden comment'
+                });
+
+                await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 2).id,
+                    parent_id: deletedComment.get('id'),
+                    html: 'This is a reply to a deleted comment'
+                });
+
+                const data2 = await membersAgent
+                    .get(`/api/comments/post/${postId}`)
+                    .expectStatus(200);
+
+                // check if hidden and deleted comments have their html removed
+                data2.body.comments.forEach((comment) => {
+                    should.notEqual(comment.html, 'This is a hidden comment');
+                    should.notEqual(comment.html, 'This is a deleted comment');
+                });
+
+                // check if hiddenComment.id and deletedComment.id are in the response
+                should(data2.body.comments.map(c => c.id)).containEql(hiddenComment.id);
+                should(data2.body.comments.map(c => c.id)).containEql(deletedComment.id);
+
+                // check if the replies to hidden and deleted comments are in the response
+                data2.body.comments.forEach((comment) => {
+                    if (comment.id === hiddenComment.id) {
+                        should(comment.replies.length).eql(1);
+                        should(comment.replies[0].html).eql('This is a reply to a hidden comment');
+                    } else if (comment.id === deletedComment.id) {
+                        should(comment.replies.length).eql(1);
+                        should(comment.replies[0].html).eql('This is a reply to a deleted comment');
+                    }
+                });
+            });
+
+            it('Returns nothing if both parent and reply are hidden', async function () {
+                await mockManager.mockLabsEnabled('commentImprovements');
+                const hiddenComment = await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 0).id,
+                    html: 'This is a hidden comment',
+                    status: 'hidden'
+                });
+
+                await dbFns.addComment({
+                    post_id: postId,
+                    member_id: fixtureManager.get('members', 1).id,
+                    parent_id: hiddenComment.get('id'),
+                    html: 'This is a reply to a hidden comment',
+                    status: 'hidden'
+                });
+
+                const data2 = await membersAgent
+                    .get(`/api/comments/post/${postId}`)
+                    .expectStatus(200);
+
+                should(data2.body.comments.length).eql(0);
+            });
+
             it('cannot comment on a post', async function () {
                 await testCannotCommentOnPost(401);
             });