Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-01 02:41:39 -05:00
Code Issues Activity

Added caching to LastSeenAtUpdater (#20964)

Browse source 
ref
https://linear.app/tryghost/issue/ENG-1543/debounce-the-members-lastseenatupdater

- The `LastSeenAtUpdater.updateLastSeenAt` function is called in
response to a `MemberClickEvent` — when a member clicks a link in an
email with tracking enabled. This function can be called many times for
the same member in a short period of time if e.g. a link checker is
clicking all the links in an email they received.
- This function should only update a member's `last_seen_at` timestamp
once per day. To accomplish this, `updateLastSeenAt` runs a
`select...for update` query to find the member's current `last_seen_at`
timestamp, and only updates the timestamp if the current `last_seen_at`
is before the start of the current day. The `for update` is required to
avoid a race condition, which previously caused this function to update
the `last_seen_at` timestamp more frequently than needed, which results
in many unnecessary database queries. However, we still run the initial
`select...for update` query for each event, which seems to be resulting
in contention for locks on the member's row in the `members` table.
- This commit introduces a simple in-memory cache so that we avoid
calling `updateLastSeenAt` if the member's `last_seen_at` timestamp has
already been updated in the current day, which should avoid running so
many `select...for update` queries and locking the `members` table up.
This commit is contained in:
Chris Raible 2024-09-13 00:54:43 -07:00 • committed by GitHub
parent c98ff3856e
commit 971d497c1e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 793 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
ghost/core/core/server/services/members-events/index.js
View file

@ -12,7 +12,7 @@ class MembersEventsServiceWrapper {
}
// Wire up all the dependencies
const {EventStorage, LastSeenAtUpdater} = require('@tryghost/members-events-service');
const {EventStorage, LastSeenAtUpdater, LastSeenAtCache} = require('@tryghost/members-events-service');
const models = require('../../models');
// Listen for events and store them in the database
@ -26,6 +26,14 @@ class MembersEventsServiceWrapper {
const db = require('../../data/db');
// Create the last seen at cache and inject it into the last seen at updater
this.lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache
}
});
// Create the last seen at updater
this.lastSeenAtUpdater = new LastSeenAtUpdater({
services: {
settingsCache
@ -34,12 +42,22 @@ class MembersEventsServiceWrapper {
return members.api;
},
db,
events
events,
lastSeenAtCache: this.lastSeenAtCache
});
// Subscribe to domain events
this.eventStorage.subscribe(DomainEvents);
this.lastSeenAtUpdater.subscribe(DomainEvents);
}
// Clear the last seen at cache
// Utility used for testing purposes
clearLastSeenAtCache() {
if (this.lastSeenAtCache) {
this.lastSeenAtCache.clear();
}
}
}
module.exports = new MembersEventsServiceWrapper();

6
ghost/core/test/e2e-frontend/members.test.js
View file

@ -13,6 +13,7 @@ const models = require('../../core/server/models');
const {fixtureManager} = require('../utils/e2e-framework');
const DataGenerator = require('../utils/fixtures/data-generator');
const members = require('../../core/server/services/members');
const membersEventsService = require('../../core/server/services/members-events');
const crypto = require('crypto');
function assertContentIsPresent(res) {
@ -89,6 +90,11 @@ describe('Front-end members behavior', function () {
sinon.restore();
});
beforeEach(function () {
// Clear the lastSeenAtCache to avoid side effects from other tests
membersEventsService.clearLastSeenAtCache();
});
describe('Member routes', function () {
it('should error serving webhook endpoint without any parameters', async function () {
await request.post('/members/webhooks/stripe')

2
ghost/core/test/e2e-server/click-tracking.test.js
View file

@ -4,6 +4,7 @@ const {agentProvider, mockManager, fixtureManager, matchers} = require('../utils
const urlUtils = require('../../core/shared/url-utils');
const jobService = require('../../core/server/services/jobs/job-service');
const {anyGhostAgent, anyContentVersion, anyNumber, anyISODateTime, anyObjectId} = matchers;
const membersEventsService = require('../../core/server/services/members-events');
describe('Click Tracking', function () {
let agent;
@ -20,6 +21,7 @@ describe('Click Tracking', function () {
mockManager.mockMail();
mockManager.mockMailgun();
webhookMockReceiver = mockManager.mockWebhookRequests();
membersEventsService.clearLastSeenAtCache();
});
afterEach(function () {

129
ghost/core/test/integration/services/last-seen-at-updater.test.js
View file

@ -2,6 +2,7 @@ require('should');
const {agentProvider, fixtureManager, mockManager} = require('../../utils/e2e-framework');
const models = require('../../../core/server/models');
const assert = require('assert/strict');
const sinon = require('sinon');
let agent;
describe('Last Seen At Updater', function () {
@ -11,36 +12,116 @@ describe('Last Seen At Updater', function () {
await agent.loginAsOwner();
});
it('updateLastSeenAtWithoutKnownLastSeen', async function () {
const membersEvents = require('../../../core/server/services/members-events');
describe('updateLastSeenAtWithoutKnownLastSeen', function () {
it('works', async function () {
const membersEvents = require('../../../core/server/services/members-events');
// Fire lots of EmailOpenedEvent for the same
const memberId = fixtureManager.get('members', 0).id;
const firstDate = new Date(Date.UTC(2099, 11, 31, 21, 0, 0, 0));
// In UTC this is 2099-12-31 21:00:00
// In CET this is 2099-12-31 22:00:00
const secondDate = new Date(Date.UTC(2099, 11, 31, 22, 0, 0, 0));
// In UTC this is 2099-12-31 22:00:00
// In CET this is 2099-12-31 23:00:00
const newDay = new Date(Date.UTC(2099, 11, 31, 23, 0, 0, 0));
// In UTC this is 2099-12-31 23:00:00
// In CET this is 2100-01-01 00:00:00
async function assertLastSeen(date) {
const member = await models.Member.findOne({id: memberId}, {require: true});
assert.equal(member.get('last_seen_at').getTime(), date.getTime());
}
mockManager.mockSetting('timezone', 'CET');
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, firstDate);
await assertLastSeen(firstDate);
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, secondDate);
await assertLastSeen(firstDate); // not changed
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, newDay);
await assertLastSeen(newDay); // changed
});
});
// Fire lots of EmailOpenedEvent for the same
const memberId = fixtureManager.get('members', 0).id;
describe('cachedUpdateLastSeenAt', function () {
it('works', async function () {
const membersEvents = require('../../../core/server/services/members-events');
// Fire lots of MemberClickEvents for the same member
const memberId = fixtureManager.get('members', 0).id;
await models.Member.edit({last_seen_at: null}, {id: memberId});
const firstDate = new Date(Date.UTC(2099, 11, 31, 21, 0, 0, 0));
// In UTC this is 2099-12-31 21:00:00
// In CET this is 2099-12-31 22:00:00
const previousLastSeen = new Date(Date.UTC(2099, 11, 29, 20, 0, 0, 0));
const firstDate = new Date(Date.UTC(2099, 11, 31, 21, 0, 0, 0));
// In UTC this is 2099-12-31 21:00:00
// In CET this is 2099-12-31 22:00:00
const secondDate = new Date(Date.UTC(2099, 11, 31, 22, 0, 0, 0));
// In UTC this is 2099-12-31 22:00:00
// In CET this is 2099-12-31 23:00:00
const secondDate = new Date(Date.UTC(2099, 11, 31, 22, 0, 0, 0));
// In UTC this is 2099-12-31 22:00:00
// In CET this is 2099-12-31 23:00:00
const newDay = new Date(Date.UTC(2099, 11, 31, 23, 0, 0, 0));
// In UTC this is 2099-12-31 23:00:00
// In CET this is 2100-01-01 00:00:00
async function assertLastSeen(date) {
const member = await models.Member.findOne({id: memberId}, {require: true});
assert.equal(member.get('last_seen_at').getTime(), date.getTime());
}
const newDay = new Date(Date.UTC(2099, 11, 31, 23, 0, 0, 0));
// In UTC this is 2099-12-31 23:00:00
// In CET this is 2100-01-01 00:00:00
mockManager.mockSetting('timezone', 'CET');
const clock = sinon.useFakeTimers(firstDate);
await membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, previousLastSeen, firstDate);
async function assertLastSeen(date) {
const member = await models.Member.findOne({id: memberId}, {require: true});
assert.equal(member.get('last_seen_at').getTime(), date.getTime());
}
await assertLastSeen(firstDate);
await membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, firstDate, secondDate);
await assertLastSeen(firstDate); // not changed
mockManager.mockSetting('timezone', 'CET');
// Advance the clock two hours to newDay
await clock.tickAsync(1000 * 60 * 60 * 2); // 2 hours
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, firstDate);
await assertLastSeen(firstDate);
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, secondDate);
await assertLastSeen(firstDate); // not changed
await membersEvents.lastSeenAtUpdater.updateLastSeenAtWithoutKnownLastSeen(memberId, newDay);
await assertLastSeen(newDay); // changed
await membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, secondDate, newDay);
await assertLastSeen(newDay); // changed
clock.restore();
});
it('does not call updateLastSeenAt multiple times for the same member on the same day', async function () {
const membersEvents = require('../../../core/server/services/members-events');
// Clear the cache to ensure it's empty
membersEvents.lastSeenAtUpdater._lastSeenAtCache.clear();
// Fire lots of MemberClickEvents for the same member
const memberId = fixtureManager.get('members', 0).id;
await models.Member.edit({last_seen_at: null}, {id: memberId});
const previousLastSeen = new Date(Date.UTC(2099, 11, 29, 20, 0, 0, 0));
const firstDate = new Date(Date.UTC(2099, 11, 31, 21, 0, 0, 0));
mockManager.mockSetting('timezone', 'CET');
const clock = sinon.useFakeTimers(firstDate);
const spy = sinon.spy(membersEvents.lastSeenAtUpdater, 'updateLastSeenAt');
await Promise.all([
membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, previousLastSeen, firstDate),
membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, previousLastSeen, firstDate),
membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, previousLastSeen, firstDate),
membersEvents.lastSeenAtUpdater.cachedUpdateLastSeenAt(memberId, previousLastSeen, firstDate)
]);
assert.equal(spy.callCount, 1);
clock.restore();
});
});
});

96
ghost/members-events-service/lib/LastSeenAtCache.js Normal file
View file

@ -0,0 +1,96 @@
const moment = require('moment-timezone');
/**
* A cache that stores the member ids that have been seen today. This cache is used to avoid having to query the database for the last_seen_at timestamp of a member multiple times in the same day.
*
* @constructor
* @param {Object} settingsCache - An instance of the settings cache
* @property {Set} _cache - A set that stores all the member ids that have been seen today
* @property {Object} _settingsCache - An instance of the settings cache
* @property {string} _startOfDay - The start of the current day in the site timezone, formatted in ISO 8601
*/
class LastSeenAtCache {
/**
*
* @param {Object} deps - Dependencies
* @param {Object} deps.services - The list of service dependencies
* @param {Object} deps.services.settingsCache - The settings service
*/
constructor({services: {settingsCache}}) {
this._cache = new Set();
this._settingsCache = settingsCache;
this._startOfDay = this._getStartOfCurrentDay();
}
/**
* @method add - Adds a member id to the cache
* @param {string} memberId
*/
add(memberId) {
this._cache.add(memberId);
}
/**
* @method remove - Removes a member id from the cache
* @param {string} memberId
*/
remove(memberId) {
this._cache.delete(memberId);
}
/**
* @method shouldUpdateMember - Checks if a member should be updated
* @param {string} memberId
* @returns {boolean} - Returns true if the member should be updated
*/
shouldUpdateMember(memberId) {
return !this._has(memberId);
}
/**
* @method clear - Clears the cache
*/
clear() {
this._cache.clear();
}
/**
* @method _has - Refreshes the cache and checks if a member id is in the cache
* @param {string} memberId
* @returns {boolean}
*/
_has(memberId) {
this._refresh();
return this._cache.has(memberId);
}
/**
* @method _shouldClear - Checks if the cache should be cleared, based on the current day
* @returns {boolean} - Returns true if the cache should be cleared
*/
_shouldClear() {
return this._startOfDay !== this._getStartOfCurrentDay();
}
/**
* @method _refresh - Clears the cache if the day has changed
*/
_refresh() {
if (this._shouldClear()) {
this.clear();
this._startOfDay = this._getStartOfCurrentDay();
}
}
/**
* Returns the start of the current day in the site timezone
* @returns {string} The start of the current day in the site timezone, formatted as a ISO 8601 string
*/
_getStartOfCurrentDay() {
const timezone = this._settingsCache.get('timezone') || 'Etc/UTC';
const startOfDay = moment().tz(timezone).startOf('day').utc().toISOString();
return startOfDay;
}
}
module.exports = LastSeenAtCache;

57
ghost/members-events-service/lib/LastSeenAtUpdater.js
View file

@ -3,6 +3,7 @@ const moment = require('moment-timezone');
const {IncorrectUsageError} = require('@tryghost/errors');
const {EmailOpenedEvent} = require('@tryghost/email-events');
const logging = require('@tryghost/logging');
const LastSeenAtCache = require('./LastSeenAtCache');
/**
* Listen for `MemberViewEvent` to update the `member.last_seen_at` timestamp
@ -16,6 +17,7 @@ class LastSeenAtUpdater {
* @param {() => object} deps.getMembersApi - A function which returns an instance of members-api
* @param {any} deps.db Database connection
* @param {any} deps.events The event emitter
* @param {any} deps.lastSeenAtCache An instance of the last seen at cache
*/
constructor({
services: {
@ -23,7 +25,8 @@ class LastSeenAtUpdater {
},
getMembersApi,
db,
events
events,
lastSeenAtCache
}) {
if (!getMembersApi) {
throw new IncorrectUsageError({message: 'Missing option getMembersApi'});
@ -33,6 +36,7 @@ class LastSeenAtUpdater {
this._settingsCacheService = settingsCache;
this._db = db;
this._events = events;
this._lastSeenAtCache = lastSeenAtCache || new LastSeenAtCache({services: {settingsCache}});
}
/**
* Subscribe to events of this domainEvents service
@ -41,7 +45,7 @@ class LastSeenAtUpdater {
subscribe(domainEvents) {
domainEvents.subscribe(MemberPageViewEvent, async (event) => {
try {
await this.updateLastSeenAt(event.data.memberId, event.data.memberLastSeenAt, event.timestamp);
await this.cachedUpdateLastSeenAt(event.data.memberId, event.data.memberLastSeenAt, event.timestamp);
} catch (err) {
logging.error(`Error in LastSeenAtUpdater.MemberPageViewEvent listener for member ${event.data.memberId}`);
logging.error(err);
@ -50,7 +54,7 @@ class LastSeenAtUpdater {
domainEvents.subscribe(MemberLinkClickEvent, async (event) => {
try {
await this.updateLastSeenAt(event.data.memberId, event.data.memberLastSeenAt, event.timestamp);
await this.cachedUpdateLastSeenAt(event.data.memberId, event.data.memberLastSeenAt, event.timestamp);
} catch (err) {
logging.error(`Error in LastSeenAtUpdater.MemberLinkClickEvent listener for member ${event.data.memberId}`);
logging.error(err);
@ -101,6 +105,15 @@ class LastSeenAtUpdater {
});
}
/**
* Updates the member.last_seen_at field if it wasn't updated in the current day yet (in the publication timezone)
*/
async cachedUpdateLastSeenAt(memberId, memberLastSeenAt, timestamp) {
if (this._lastSeenAtCache.shouldUpdateMember(memberId)) {
await this.updateLastSeenAt(memberId, memberLastSeenAt, timestamp);
}
}
/**
* Updates the member.last_seen_at field if it wasn't updated in the current day yet (in the publication timezone)
* Example: current time is 2022-02-28 18:00:00
@ -115,20 +128,30 @@ class LastSeenAtUpdater {
// This isn't strictly necessary since we will fetch the member row for update and double check this
// This is an optimization to avoid unnecessary database queries if last_seen_at is already after the beginning of the current day
if (memberLastSeenAt === null || moment(moment.utc(timestamp).tz(timezone).startOf('day')).isAfter(memberLastSeenAt)) {
const membersApi = this._getMembersApi();
await this._db.knex.transaction(async (trx) => {
// To avoid a race condition, we lock the member row for update, then the last_seen_at field again to prevent simultaneous updates
const currentMember = await membersApi.members.get({id: memberId}, {require: true, transacting: trx, forUpdate: true});
const currentMemberLastSeenAt = currentMember.get('last_seen_at');
if (currentMemberLastSeenAt === null || moment(moment.utc(timestamp).tz(timezone).startOf('day')).isAfter(currentMemberLastSeenAt)) {
const memberToUpdate = await currentMember.refresh({transacting: trx, forUpdate: false, withRelated: ['labels', 'newsletters']});
const updatedMember = await memberToUpdate.save({last_seen_at: moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss')}, {transacting: trx, patch: true, method: 'update'});
// The standard event doesn't get emitted inside the transaction, so we do it manually
this._events.emit('member.edited', updatedMember);
return Promise.resolve(updatedMember);
}
return Promise.resolve(undefined);
});
try {
// Pre-emptively update local cache so we don't update the same member again in the same day
this._lastSeenAtCache.add(memberId);
const membersApi = this._getMembersApi();
await this._db.knex.transaction(async (trx) => {
// To avoid a race condition, we lock the member row for update, then the last_seen_at field again to prevent simultaneous updates
const currentMember = await membersApi.members.get({id: memberId}, {require: true, transacting: trx, forUpdate: true});
const currentMemberLastSeenAt = currentMember.get('last_seen_at');
if (currentMemberLastSeenAt === null || moment(moment.utc(timestamp).tz(timezone).startOf('day')).isAfter(currentMemberLastSeenAt)) {
const memberToUpdate = await currentMember.refresh({transacting: trx, forUpdate: false, withRelated: ['labels', 'newsletters']});
const updatedMember = await memberToUpdate.save({last_seen_at: moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss')}, {transacting: trx, patch: true, method: 'update'});
// The standard event doesn't get emitted inside the transaction, so we do it manually
this._events.emit('member.edited', updatedMember);
return Promise.resolve(updatedMember);
}
return Promise.resolve(undefined);
});
} catch (err) {
// Remove the member from the cache if an error occurs
// This is to ensure that the member is updated on the next event if this one fails
this._lastSeenAtCache.remove(memberId);
// Bubble up the error to the event listener
throw err;
}
}
}

1
ghost/members-events-service/lib/index.js
View file

@ -1,4 +1,5 @@
module.exports = {
LastSeenAtUpdater: require('./LastSeenAtUpdater'),
LastSeenAtCache: require('./LastSeenAtCache'),
EventStorage: require('./EventStorage')
};

277
ghost/members-events-service/test/last-seen-at-cache.test.js Normal file
View file

@ -0,0 +1,277 @@
// Switch these lines once there are useful utils
// const testUtils = require('./utils');
require('./utils');
const LastSeenAtCache = require('../lib/LastSeenAtCache');
const assert = require('assert/strict');
const sinon = require('sinon');
const moment = require('moment-timezone');
describe('LastSeenAtCache', function () {
let clock;
before(function () {
clock = sinon.useFakeTimers();
});
afterEach(function () {
clock.restore();
});
describe('constructor', function () {
it('intitializes an instance of the LastSeenAtCache successfully', function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
assert.ok(lastSeenAtCache._cache instanceof Set, 'lastSeenAtCache._cache should be a Set');
assert.equal(lastSeenAtCache._settingsCache.get('timezone'), 'Etc/UTC', 'lastSeenAtCache._timezone should be set');
assert.ok(lastSeenAtCache._startOfDay, 'lastSeenAtCache._startOfDay should be set');
});
});
describe('_getStartOfCurrentDay', function () {
it('returns the start of the current day in the site timezone', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
const result = lastSeenAtCache._getStartOfCurrentDay();
assert.equal(result, '2022-02-28T00:00:00.000Z', 'start of the current day in UTC should be returned');
});
it('works correctly on another timezone', async function () {
const now = moment('2022-02-28T18:00:00Z').utc(); // 2022-02-28 01:00:00 in Bangkok
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Asia/Bangkok';
}
}
}
});
const result = lastSeenAtCache._getStartOfCurrentDay();
assert.equal(result, '2022-02-28T17:00:00.000Z', 'start of the current day in Bangkok should be returned'); // 2022-02-28 00:00:00 in Bankok
});
it('defaults to Etc/UTC if the timezone is not set', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return null;
}
}
}
});
const result = lastSeenAtCache._getStartOfCurrentDay();
assert.equal(result, '2022-02-28T00:00:00.000Z', 'start of the current day in UTC should be returned');
});
});
describe('add', function () {
it('adds a member id to the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
assert.ok(lastSeenAtCache._cache.has('member-id'));
});
});
describe('remove', function () {
it('removes a member id from the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
assert.equal(lastSeenAtCache._cache.size, 1, 'the member id should be in the cache');
lastSeenAtCache.remove('member-id');
assert.equal(lastSeenAtCache._cache.size, 0, 'the member id should be removed from the cache');
});
});
describe('_has', function () {
it('returns true if the member id is in the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
assert.equal(lastSeenAtCache._has('member-id'), true, 'the member id should be in the cache');
});
it('returns false if the member id is not in the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
assert.equal(lastSeenAtCache._has('member-id'), false, 'the member id should not be in the cache');
});
});
describe('clear', function () {
it('clears the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
lastSeenAtCache.clear();
assert.equal(lastSeenAtCache._cache.size, 0, 'the cache should be empty');
});
});
describe('_shouldClear', function () {
it('returns true only if the day has changed', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
const result = lastSeenAtCache._shouldClear();
assert.equal(result, false, 'the day has not changed');
clock.tick(24 * 60 * 60 * 1000);
const result2 = lastSeenAtCache._shouldClear();
assert.equal(result2, true, 'the day has changed');
});
it('returns true only if the day has changed in another timezone', async function () {
const now = moment('2022-02-28T18:00:00Z').utc(); // 2022-02-28 01:00:00 in Bangkok
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Asia/Bangkok';
}
}
}
});
const result = lastSeenAtCache._shouldClear();
assert.equal(result, false, 'the day has not changed');
clock.tick(24 * 60 * 60 * 1000);
const result2 = lastSeenAtCache._shouldClear();
assert.equal(result2, true, 'the day has changed');
});
});
describe('_refresh', function () {
it('clears the cache if the day has changed', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
const day0 = lastSeenAtCache._getStartOfCurrentDay();
lastSeenAtCache.add('member-id');
lastSeenAtCache._refresh();
assert.equal(lastSeenAtCache._cache.size, 1, 'the cache should not be cleared');
clock.tick(24 * 60 * 60 * 1000);
const day1 = lastSeenAtCache._getStartOfCurrentDay();
assert.notEqual(day0, day1, 'the day has changed');
lastSeenAtCache._refresh();
assert.equal(lastSeenAtCache._cache.size, 0, 'the cache should be cleared');
assert.equal(lastSeenAtCache._startOfDay, day1, 'the start of the day should be updated');
});
});
describe('shouldUpdateMember', function () {
it('returns true if the member id is not in the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
assert.equal(lastSeenAtCache.shouldUpdateMember('member-id'), true, 'the member id should not be in the cache');
});
it('returns false if the member id is in the cache', async function () {
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
assert.equal(lastSeenAtCache.shouldUpdateMember('member-id'), false, 'the member id should be in the cache');
});
it('returns true if the day has changed', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
clock = sinon.useFakeTimers(now.toDate());
const lastSeenAtCache = new LastSeenAtCache({
services: {
settingsCache: {
get() {
return 'Etc/UTC';
}
}
}
});
lastSeenAtCache.add('member-id');
clock.tick(24 * 60 * 60 * 1000);
assert.equal(lastSeenAtCache.shouldUpdateMember('member-id'), true, 'the day has changed');
});
});
});

247
ghost/members-events-service/test/last-seen-at-updater.test.js
View file

@ -10,6 +10,9 @@ const {MemberPageViewEvent, MemberCommentEvent, MemberSubscribeEvent, MemberLink
const moment = require('moment');
const {EmailOpenedEvent} = require('@tryghost/email-events');
const EventEmitter = require('events');
const logging = require('@tryghost/logging');
sinon.stub(logging, 'error');
describe('LastSeenAtUpdater', function () {
let events;
@ -61,6 +64,32 @@ describe('LastSeenAtUpdater', function () {
DomainEvents.dispatch(MemberPageViewEvent.create({memberId: '1', memberLastSeenAt: previousLastSeen, url: '/'}, now.toDate()));
assert(updater.updateLastSeenAt.calledOnceWithExactly('1', previousLastSeen, now.toDate()));
});
it('Catches errors in updateLastSeenAt on MemberPageViewEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
const previousLastSeen = moment('2022-02-27T23:00:00Z').toISOString();
const stub = sinon.stub().resolves();
const settingsCache = sinon.stub().returns('Etc/UTC');
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
update: stub
}
};
},
events
});
updater.subscribe(DomainEvents);
sinon.stub(updater, 'updateLastSeenAt').throws('Error');
DomainEvents.dispatch(MemberPageViewEvent.create({memberId: '1', memberLastSeenAt: previousLastSeen, url: '/'}, now.toDate()));
assert(true, 'The LastSeenAtUpdater should catch errors in updateLastSeenAt');
});
it('Calls updateLastSeenAt on MemberLinkClickEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
@ -87,8 +116,34 @@ describe('LastSeenAtUpdater', function () {
DomainEvents.dispatch(MemberLinkClickEvent.create({memberId: '1', memberLastSeenAt: previousLastSeen, url: '/'}, now.toDate()));
assert(updater.updateLastSeenAt.calledOnceWithExactly('1', previousLastSeen, now.toDate()));
});
it('Catches errors in updateLastSeenAt on MemberLinkClickEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
const previousLastSeen = moment('2022-02-27T23:00:00Z').toISOString();
const stub = sinon.stub().resolves();
const settingsCache = sinon.stub().returns('Etc/UTC');
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
update: stub
}
};
},
events
});
updater.subscribe(DomainEvents);
sinon.stub(updater, 'updateLastSeenAt').throws('Error');
DomainEvents.dispatch(MemberLinkClickEvent.create({memberId: '1', memberLastSeenAt: previousLastSeen, url: '/'}, now.toDate()));
assert(true, 'The LastSeenAtUpdater should catch errors in updateLastSeenAt');
});
it('Calls updateLastSeenAt on EmailOpenedEvents', async function () {
it('Calls updateLastSeenAtWithoutKnownLastSeen on EmailOpenedEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
const settingsCache = sinon.stub().returns('Etc/UTC');
const db = {
@ -123,6 +178,40 @@ describe('LastSeenAtUpdater', function () {
assert(updater.updateLastSeenAtWithoutKnownLastSeen.calledOnceWithExactly('1', now.toDate()));
assert(db.update.calledOnce);
});
it('Catches errors in updateLastSeenAtWithoutKnownLastSeen on EmailOpenedEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
const settingsCache = sinon.stub().returns('Etc/UTC');
const db = {
knex() {
return this;
},
where() {
return this;
},
andWhere() {
return this;
},
update: sinon.stub()
};
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {};
},
db,
events
});
updater.subscribe(DomainEvents);
sinon.stub(updater, 'updateLastSeenAtWithoutKnownLastSeen').throws('Error');
DomainEvents.dispatch(EmailOpenedEvent.create({memberId: '1', emailRecipientId: '1', emailId: '1', timestamp: now.toDate()}));
await DomainEvents.allSettled();
assert(true, 'The LastSeenAtUpdater should catch errors in updateLastSeenAtWithoutKnownLastSeen');
});
it('Calls updateLastCommentedAt on MemberCommentEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
@ -149,6 +238,31 @@ describe('LastSeenAtUpdater', function () {
assert(updater.updateLastCommentedAt.calledOnceWithExactly('1', now.toDate()));
});
it('Catches errors in updateLastCommentedAt on MemberCommentEvents', async function () {
const now = moment('2022-02-28T18:00:00Z').utc();
const stub = sinon.stub().resolves();
const settingsCache = sinon.stub().returns('Etc/UTC');
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
update: stub
}
};
},
events
});
updater.subscribe(DomainEvents);
sinon.stub(updater, 'updateLastCommentedAt').throws('Error');
DomainEvents.dispatch(MemberCommentEvent.create({memberId: '1'}, now.toDate()));
assert(true, 'The LastSeenAtUpdater should catch errors in updateLastCommentedAt');
});
it('Doesn\'t fire on other events', async function () {
const spy = sinon.spy();
const updater = new LastSeenAtUpdater({
@ -329,6 +443,137 @@ describe('LastSeenAtUpdater', function () {
});
});
describe('cachedUpdateLastSeenAt', function () {
it('calls updateLastSeenAt if the member has not been seen today', async function () {
const now = moment.utc('2022-02-28T18:00:00Z');
const clock = sinon.useFakeTimers(now.toDate());
const saveStub = sinon.stub().resolves();
const refreshStub = sinon.stub().resolves({save: saveStub});
const settingsCache = sinon.stub().returns('Europe/Brussels');
const transactionStub = sinon.stub().callsFake((callback) => {
return callback();
});
const getStub = sinon.stub();
getStub.onFirstCall().resolves({get: () => null, save: saveStub, refresh: refreshStub});
getStub.onSecondCall().resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
getStub.resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
get: getStub
}
};
},
db: {
knex: {
transaction: transactionStub
}
},
events
});
sinon.stub(events, 'emit');
sinon.stub(updater, 'updateLastSeenAt');
await updater.cachedUpdateLastSeenAt('1', now.toDate(), now.toDate());
assert(updater.updateLastSeenAt.calledOnceWithExactly('1', now.toDate(), now.toDate()));
clock.restore();
});
it('only calls updateLastSeenAt once per day per member', async function () {
const now = moment.utc('2022-02-28T18:00:00Z');
const previousLastSeen = moment.utc('2022-02-26T00:00:00Z').toDate();
const clock = sinon.useFakeTimers(now.toDate());
const saveStub = sinon.stub().resolves();
const refreshStub = sinon.stub().resolves({save: saveStub});
const settingsCache = sinon.stub().returns('Europe/Brussels');
const transactionStub = sinon.stub().callsFake((callback) => {
return callback();
});
const getStub = sinon.stub();
getStub.onFirstCall().resolves({get: () => null, save: saveStub, refresh: refreshStub});
getStub.onSecondCall().resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
getStub.resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
get: getStub
}
};
},
db: {
knex: {
transaction: transactionStub
}
},
events
});
sinon.stub(events, 'emit');
sinon.spy(updater, 'updateLastSeenAt');
await Promise.all([
updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate()),
updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate()),
updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate()),
updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate())
]);
assert(updater.updateLastSeenAt.calledOnce, 'The LastSeenAtUpdater should only update a member once per day.');
clock.restore();
});
it('calls updateLastSeenAt again if it fails the first time', async function () {
const now = moment.utc('2022-02-28T18:00:00Z');
const previousLastSeen = moment.utc('2022-02-26T00:00:00Z').toDate();
const clock = sinon.useFakeTimers(now.toDate());
const saveStub = sinon.stub().resolves();
const refreshStub = sinon.stub().resolves({save: saveStub});
const settingsCache = sinon.stub().returns('Europe/Brussels');
// Throw an error on the first transaction, succeed on the second
const transactionStub = sinon.stub().onCall(0).throws('Error').onCall(1).callsFake((callback) => {
return callback();
});
const getStub = sinon.stub();
getStub.onFirstCall().resolves({get: () => null, save: saveStub, refresh: refreshStub});
getStub.onSecondCall().resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
getStub.resolves({get: () => now.toDate(), save: saveStub, refresh: refreshStub});
const updater = new LastSeenAtUpdater({
services: {
settingsCache: {
get: settingsCache
}
},
getMembersApi() {
return {
members: {
get: getStub
}
};
},
db: {
knex: {
transaction: transactionStub
}
},
events
});
sinon.stub(events, 'emit');
sinon.spy(updater, 'updateLastSeenAt');
assert.rejects(updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate()));
await updater.cachedUpdateLastSeenAt('1', previousLastSeen, now.toDate());
assert(updater.updateLastSeenAt.calledTwice, 'The LastSeenAtUpdater should attempt to update a member again if the first update fails.');
clock.restore();
});
});
describe('updateLastCommentedAt', function () {
it('works correctly on another timezone (not updating last_commented_at)', async function () {
const now = moment('2022-02-28T04:00:00Z').utc();