diff --git a/.changeset/nice-walls-admire.md b/.changeset/nice-walls-admire.md new file mode 100644 index 000000000..55de8a6fd --- /dev/null +++ b/.changeset/nice-walls-admire.md @@ -0,0 +1,10 @@ +--- +"@logto/core": patch +--- + +migrate `/dashboard` API data source + +Updates: +- the responded data from `/dashboard/users/new` API is now based on UTC time +- the responded data from `/dashboard/users/active` API is now based on UTC time +- the query parameter `date` for `/dashboard/users/active` API is now based on UTC time \ No newline at end of file diff --git a/packages/core/src/event-listeners/record-active-users.ts b/packages/core/src/event-listeners/record-active-users.ts index 937a0177f..fca7d4017 100644 --- a/packages/core/src/event-listeners/record-active-users.ts +++ b/packages/core/src/event-listeners/record-active-users.ts @@ -1,7 +1,7 @@ import { generateStandardId } from '@logto/shared'; -import { getUtcStartOfTheDay } from '#src/oidc/utils.js'; import type Queries from '#src/tenants/Queries.js'; +import { getUtcStartOfTheDay } from '#src/utils/utc.js'; export const recordActiveUsers = async (accessToken: { accountId?: string }, queries: Queries) => { const { accountId } = accessToken; @@ -16,6 +16,6 @@ export const recordActiveUsers = async (accessToken: { accountId?: string }, que await insertActiveUser({ id: generateStandardId(), userId: accountId, - date: getUtcStartOfTheDay(new Date()).getTime(), + date: getUtcStartOfTheDay(new Date()), }); }; diff --git a/packages/core/src/oidc/utils.ts b/packages/core/src/oidc/utils.ts index 3f630425e..e0f4e356e 100644 --- a/packages/core/src/oidc/utils.ts +++ b/packages/core/src/oidc/utils.ts @@ -63,9 +63,3 @@ export const isOriginAllowed = ( return [...corsAllowedOrigins, ...redirectUriOrigins].includes(origin); }; - -export const getUtcStartOfTheDay = (date: Date) => { - return new Date( - Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), 0, 0, 0, 0) - ); -}; diff --git a/packages/core/src/queries/daily-active-user.ts b/packages/core/src/queries/daily-active-user.ts index 547f6aa91..821cf3129 100644 --- a/packages/core/src/queries/daily-active-user.ts +++ b/packages/core/src/queries/daily-active-user.ts @@ -1,14 +1,42 @@ import { DailyActiveUsers } from '@logto/schemas'; -import type { CommonQueryMethods } from 'slonik'; +import { convertToIdentifiers } from '@logto/shared'; +import { sql, type CommonQueryMethods } from 'slonik'; import { buildInsertIntoWithPool } from '#src/database/insert-into.js'; +const { table, fields } = convertToIdentifiers(DailyActiveUsers); + export const createDailyActiveUsersQueries = (pool: CommonQueryMethods) => { const insertActiveUser = buildInsertIntoWithPool(pool)(DailyActiveUsers, { onConflict: { ignore: true }, }); + const getDailyActiveUserCountsByTimeInterval = async ( + startTimeExclusive: number, + endTimeInclusive: number + ) => + pool.any<{ date: string; count: number }>(sql` + select date(${fields.date}), count(distinct(${fields.userId})) + from ${table} + where ${fields.date} > to_timestamp(${startTimeExclusive}::double precision / 1000) + and ${fields.date} <= to_timestamp(${endTimeInclusive}::double precision / 1000) + group by date(${fields.date}) + `); + + const countActiveUsersByTimeInterval = async ( + startTimeExclusive: number, + endTimeInclusive: number + ) => + pool.one<{ count: number }>(sql` + select count(distinct(${fields.userId})) + from ${table} + where ${fields.date} > to_timestamp(${startTimeExclusive}::double precision / 1000) + and ${fields.date} <= to_timestamp(${endTimeInclusive}::double precision / 1000) + `); + return { insertActiveUser, + getDailyActiveUserCountsByTimeInterval, + countActiveUsersByTimeInterval, }; }; diff --git a/packages/core/src/queries/daily-token-usage.ts b/packages/core/src/queries/daily-token-usage.ts index 92f82b9c2..fae7df6a5 100644 --- a/packages/core/src/queries/daily-token-usage.ts +++ b/packages/core/src/queries/daily-token-usage.ts @@ -3,7 +3,7 @@ import { convertToIdentifiers, generateStandardId } from '@logto/shared'; import type { CommonQueryMethods } from 'slonik'; import { sql } from 'slonik'; -import { getUtcStartOfTheDay } from '#src/oidc/utils.js'; +import { getUtcStartOfTheDay } from '#src/utils/utc.js'; const { table, fields } = convertToIdentifiers(DailyTokenUsage); const { fields: fieldsWithPrefix } = convertToIdentifiers(DailyTokenUsage, true); @@ -33,7 +33,7 @@ export const createDailyTokenUsageQueries = (pool: CommonQueryMethods) => { insert into ${table} (${fields.id}, ${fields.date}, ${fields.usage}) values (${generateStandardId()}, to_timestamp(${getUtcStartOfTheDay( date - ).getTime()}::double precision / 1000), 1) + )}::double precision / 1000), 1) on conflict (${fields.date}, ${fields.tenantId}) do update set ${fields.usage} = ${ fieldsWithPrefix.usage } + 1 diff --git a/packages/core/src/queries/log.ts b/packages/core/src/queries/log.ts index c6ddaf923..d8e632bee 100644 --- a/packages/core/src/queries/log.ts +++ b/packages/core/src/queries/log.ts @@ -1,5 +1,5 @@ import { - token, + type token, type hook, Logs, type HookExecutionStats, @@ -77,33 +77,6 @@ export const createLogQueries = (pool: CommonQueryMethods) => { const findLogById = buildFindEntityByIdWithPool(pool)(Logs); - const getDailyActiveUserCountsByTimeInterval = async ( - startTimeExclusive: number, - endTimeInclusive: number - ) => - pool.any<{ date: string; count: number }>(sql` - select date(${fields.createdAt}), count(distinct(${fields.payload}->>'userId')) - from ${table} - where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000) - and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000) - and ${fields.key} like ${`${token.Type.ExchangeTokenBy}.%`} - and ${fields.payload}->>'result' = 'Success' - group by date(${fields.createdAt}) - `); - - const countActiveUsersByTimeInterval = async ( - startTimeExclusive: number, - endTimeInclusive: number - ) => - pool.one<{ count: number }>(sql` - select count(distinct(${fields.payload}->>'userId')) - from ${table} - where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000) - and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000) - and ${fields.key} like ${`${token.Type.ExchangeTokenBy}.%`} - and ${fields.payload}->>'result' = 'Success' - `); - const getHookExecutionStatsByHookId = async (hookId: string) => { const startTimeExclusive = subDays(new Date(), 1).getTime(); return pool.one(sql` @@ -120,8 +93,6 @@ export const createLogQueries = (pool: CommonQueryMethods) => { countLogs, findLogs, findLogById, - getDailyActiveUserCountsByTimeInterval, - countActiveUsersByTimeInterval, getHookExecutionStatsByHookId, }; }; diff --git a/packages/core/src/queries/user.ts b/packages/core/src/queries/user.ts index 620095b36..03772d499 100644 --- a/packages/core/src/queries/user.ts +++ b/packages/core/src/queries/user.ts @@ -253,12 +253,17 @@ export const createUserQueries = (pool: CommonQueryMethods) => { startTimeExclusive: number, endTimeInclusive: number ) => + /** + * Since the DAU data stored in the `daily_active_users` table is in UTC, we need to convert + * the `created_at` time to UTC in the result. This keeps the result consistent with the daily + * active user counts. + */ pool.any<{ date: string; count: number }>(sql` - select date(${fields.createdAt}), count(*) + select date(${fields.createdAt} at time zone 'UTC'), count(*) from ${table} where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000) and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000) - group by date(${fields.createdAt}) + group by date(${fields.createdAt} at time zone 'UTC') `); return { diff --git a/packages/core/src/routes/dashboard.openapi.json b/packages/core/src/routes/dashboard.openapi.json index dd690f006..72fe9f237 100644 --- a/packages/core/src/routes/dashboard.openapi.json +++ b/packages/core/src/routes/dashboard.openapi.json @@ -21,7 +21,7 @@ "/api/dashboard/users/new": { "get": { "summary": "Get new user count", - "description": "Get new user count in the past 7 days.", + "description": "Get new user count in the past 7 days. Based on UTC time.", "parameters": [], "responses": { "200": { @@ -33,12 +33,12 @@ "/api/dashboard/users/active": { "get": { "summary": "Get active user data", - "description": "Get active user data, including daily active user (DAU), weekly active user (WAU) and monthly active user (MAU). It also includes an array of DAU in the past 30 days.", + "description": "Get active user data, including daily active user (DAU), weekly active user (WAU) and monthly active user (MAU). It also includes an array of DAU in the past 30 days. Based on UTC time.", "parameters": [ { "in": "query", "name": "date", - "description": "The date to get active user data." + "description": "The date (based on UTC) to get active user data." } ], "responses": { diff --git a/packages/core/src/routes/dashboard.test.ts b/packages/core/src/routes/dashboard.test.ts index ae7ef9ae1..351fff858 100644 --- a/packages/core/src/routes/dashboard.test.ts +++ b/packages/core/src/routes/dashboard.test.ts @@ -1,11 +1,12 @@ // The FP version works better for `format()` /* eslint-disable import/no-duplicates */ import { pickDefault } from '@logto/shared/esm'; -import { endOfDay, subDays } from 'date-fns'; +import { subDays } from 'date-fns'; import { format } from 'date-fns/fp'; import { MockTenant } from '#src/test-utils/tenant.js'; import { createRequester } from '#src/utils/test-utils.js'; +import { getUtcEndOfTheDay } from '#src/utils/utc.js'; /* eslint-enable import/no-duplicates */ const { jest } = import.meta; @@ -41,13 +42,13 @@ const users = { }; const { countUsers, getDailyNewUserCountsByTimeInterval } = users; -const logs = { +const dailyActiveUsers = { getDailyActiveUserCountsByTimeInterval: jest.fn().mockResolvedValue(mockDailyActiveUserCounts), countActiveUsersByTimeInterval: jest.fn().mockResolvedValue({ count: mockActiveUserCount }), }; -const { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval } = logs; +const { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval } = dailyActiveUsers; -const tenantContext = new MockTenant(undefined, { logs, users }); +const tenantContext = new MockTenant(undefined, { dailyActiveUsers, users }); const dashboardRoutes = await pickDefault(import('./dashboard.js')); describe('dashboardRoutes', () => { @@ -72,14 +73,14 @@ describe('dashboardRoutes', () => { describe('GET /dashboard/users/new', () => { beforeEach(() => { - jest.useFakeTimers().setSystemTime(new Date('2022-05-14')); + jest.useFakeTimers().setSystemTime(Date.UTC(2022, 4, 14)); }); it('should call getDailyNewUserCountsByTimeInterval with the time interval (14 days ago 23:59:59.999, today 23:59:59.999]', async () => { await logRequest.get('/dashboard/users/new'); expect(getDailyNewUserCountsByTimeInterval).toHaveBeenCalledWith( - subDays(endOfDay(Date.now()), 14).valueOf(), - endOfDay(Date.now()).valueOf() + getUtcEndOfTheDay(subDays(Date.now(), 14)), + getUtcEndOfTheDay(Date.now()) ); }); @@ -100,7 +101,7 @@ describe('dashboardRoutes', () => { }); describe('GET /dashboard/users/active', () => { - const mockToday = new Date(2022, 4, 30); + const mockToday = Date.UTC(2022, 4, 30); beforeEach(() => { jest.useFakeTimers().setSystemTime(mockToday); @@ -112,44 +113,44 @@ describe('dashboardRoutes', () => { }); it('should call getDailyActiveUserCountsByTimeInterval with the time interval (2022-05-31, 2022-06-30] when the parameter `date` is 2022-06-30', async () => { - const targetDate = new Date(2022, 5, 30); + const targetDate = Date.UTC(2022, 5, 30); await logRequest.get(`/dashboard/users/active?date=${formatToQueryDate(targetDate)}`); expect(getDailyActiveUserCountsByTimeInterval).toHaveBeenCalledWith( - endOfDay(new Date(2022, 4, 31)).valueOf(), - endOfDay(targetDate).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 4, 31)), + getUtcEndOfTheDay(targetDate) ); }); it('should call getDailyActiveUserCountsByTimeInterval with the time interval (30 days ago, tomorrow] when there is no parameter `date`', async () => { await logRequest.get('/dashboard/users/active'); expect(getDailyActiveUserCountsByTimeInterval).toHaveBeenCalledWith( - endOfDay(new Date(2022, 3, 30)).valueOf(), - endOfDay(mockToday).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 3, 30)), + getUtcEndOfTheDay(mockToday) ); }); it('should call countActiveUsersByTimeInterval with correct parameters when the parameter `date` is 2022-06-30', async () => { - const targetDate = new Date(2022, 5, 30); + const targetDate = Date.UTC(2022, 5, 30); await logRequest.get(`/dashboard/users/active?date=${formatToQueryDate(targetDate)}`); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 1, - endOfDay(new Date(2022, 5, 16)).valueOf(), - endOfDay(new Date(2022, 5, 23)).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 5, 16)), + getUtcEndOfTheDay(Date.UTC(2022, 5, 23)) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 2, - endOfDay(new Date(2022, 5, 23)).valueOf(), - endOfDay(targetDate).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 5, 23)), + getUtcEndOfTheDay(targetDate) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 3, - endOfDay(new Date(2022, 4, 1)).valueOf(), - endOfDay(new Date(2022, 4, 31)).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 4, 1)), + getUtcEndOfTheDay(Date.UTC(2022, 4, 31)) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 4, - endOfDay(new Date(2022, 4, 31)).valueOf(), - endOfDay(targetDate).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 4, 31)), + getUtcEndOfTheDay(targetDate) ); }); @@ -157,23 +158,23 @@ describe('dashboardRoutes', () => { await logRequest.get('/dashboard/users/active'); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 1, - endOfDay(new Date(2022, 4, 16)).valueOf(), - endOfDay(new Date(2022, 4, 23)).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 4, 16)), + getUtcEndOfTheDay(Date.UTC(2022, 4, 23)) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 2, - endOfDay(new Date(2022, 4, 23)).valueOf(), - endOfDay(mockToday).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 4, 23)), + getUtcEndOfTheDay(mockToday) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 3, - endOfDay(new Date(2022, 2, 31)).valueOf(), - endOfDay(new Date(2022, 3, 30)).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 2, 31)), + getUtcEndOfTheDay(Date.UTC(2022, 3, 30)) ); expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith( 4, - endOfDay(new Date(2022, 3, 30)).valueOf(), - endOfDay(mockToday).valueOf() + getUtcEndOfTheDay(Date.UTC(2022, 3, 30)), + getUtcEndOfTheDay(mockToday) ); }); diff --git a/packages/core/src/routes/dashboard.ts b/packages/core/src/routes/dashboard.ts index b5c29d5ff..712819653 100644 --- a/packages/core/src/routes/dashboard.ts +++ b/packages/core/src/routes/dashboard.ts @@ -1,24 +1,21 @@ import { dateRegex } from '@logto/core-kit'; import { getNewUsersResponseGuard, getActiveUsersResponseGuard } from '@logto/schemas'; -import { endOfDay, format, subDays } from 'date-fns'; +import { subDays } from 'date-fns'; import { number, object, string } from 'zod'; import koaGuard from '#src/middleware/koa-guard.js'; +import { getUtcDateString, getUtcEndOfTheDay } from '#src/utils/utc.js'; import type { AuthedRouter, RouterInitArgs } from './types.js'; -const getDateString = (date: Date | number) => format(date, 'yyyy-MM-dd'); - const indices = (length: number) => [...Array.from({ length }).keys()]; -const getEndOfDayTimestamp = (date: Date | number) => endOfDay(date).valueOf(); - export default function dashboardRoutes( ...[router, { queries }]: RouterInitArgs ) { const { - logs: { countActiveUsersByTimeInterval, getDailyActiveUserCountsByTimeInterval }, users: { countUsers, getDailyNewUserCountsByTimeInterval }, + dailyActiveUsers: { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval }, } = queries; router.get( @@ -47,24 +44,24 @@ export default function dashboardRoutes( const today = Date.now(); const dailyNewUserCounts = await getDailyNewUserCountsByTimeInterval( // (14 days ago 23:59:59.999, today 23:59:59.999] - getEndOfDayTimestamp(subDays(today, 14)), - getEndOfDayTimestamp(today) + getUtcEndOfTheDay(subDays(today, 14)), + getUtcEndOfTheDay(today) ); const last14DaysNewUserCounts = new Map( dailyNewUserCounts.map(({ date, count }) => [date, count]) ); - const todayNewUserCount = last14DaysNewUserCounts.get(getDateString(today)) ?? 0; + const todayNewUserCount = last14DaysNewUserCounts.get(getUtcDateString(today)) ?? 0; const yesterday = subDays(today, 1); - const yesterdayNewUserCount = last14DaysNewUserCounts.get(getDateString(yesterday)) ?? 0; + const yesterdayNewUserCount = last14DaysNewUserCounts.get(getUtcDateString(yesterday)) ?? 0; const todayDelta = todayNewUserCount - yesterdayNewUserCount; const last7DaysNewUserCount = indices(7) - .map((index) => getDateString(subDays(today, index))) + .map((index) => getUtcDateString(subDays(today, index))) .reduce((sum, date) => sum + (last14DaysNewUserCounts.get(date) ?? 0), 0); const newUserCountFrom13DaysAgoTo7DaysAgo = indices(7) - .map((index) => getDateString(subDays(today, index + 7))) + .map((index) => getUtcDateString(subDays(today, index + 7))) .reduce((sum, date) => sum + (last14DaysNewUserCounts.get(date) ?? 0), 0); const last7DaysDelta = last7DaysNewUserCount - newUserCountFrom13DaysAgoTo7DaysAgo; @@ -108,39 +105,39 @@ export default function dashboardRoutes( ] = await Promise.all([ getDailyActiveUserCountsByTimeInterval( // (30 days ago 23:59:59.999, target day 23:59:59.999] - getEndOfDayTimestamp(subDays(targetDay, 30)), - getEndOfDayTimestamp(targetDay) + getUtcEndOfTheDay(subDays(targetDay, 30)), + getUtcEndOfTheDay(targetDay) ), countActiveUsersByTimeInterval( // (14 days ago 23:59:59.999, 7 days ago 23:59:59.999] - getEndOfDayTimestamp(subDays(targetDay, 14)), - getEndOfDayTimestamp(subDays(targetDay, 7)) + getUtcEndOfTheDay(subDays(targetDay, 14)), + getUtcEndOfTheDay(subDays(targetDay, 7)) ), countActiveUsersByTimeInterval( // (7 days ago 23:59:59.999, target day 23:59:59.999] - getEndOfDayTimestamp(subDays(targetDay, 7)), - getEndOfDayTimestamp(targetDay) + getUtcEndOfTheDay(subDays(targetDay, 7)), + getUtcEndOfTheDay(targetDay) ), countActiveUsersByTimeInterval( // (60 days ago 23:59:59.999, 30 days ago 23:59:59.999] - getEndOfDayTimestamp(subDays(targetDay, 60)), - getEndOfDayTimestamp(subDays(targetDay, 30)) + getUtcEndOfTheDay(subDays(targetDay, 60)), + getUtcEndOfTheDay(subDays(targetDay, 30)) ), countActiveUsersByTimeInterval( // (30 days ago 23:59:59.999, target day 23:59:59.999] - getEndOfDayTimestamp(subDays(targetDay, 30)), - getEndOfDayTimestamp(targetDay) + getUtcEndOfTheDay(subDays(targetDay, 30)), + getUtcEndOfTheDay(targetDay) ), ]); - const previousDate = getDateString(subDays(targetDay, 1)); - const targetDate = getDateString(targetDay); + const previousDate = getUtcDateString(subDays(targetDay, 1)); + const targetDate = getUtcDateString(targetDay); const previousDAU = last30DauCounts.find(({ date }) => date === previousDate)?.count ?? 0; const dau = last30DauCounts.find(({ date }) => date === targetDate)?.count ?? 0; const dauCurve = indices(30).map((index) => { - const dateString = getDateString(subDays(targetDay, 29 - index)); + const dateString = getUtcDateString(subDays(targetDay, 29 - index)); const count = last30DauCounts.find(({ date }) => date === dateString)?.count ?? 0; return { date: dateString, count }; diff --git a/packages/core/src/utils/utc.ts b/packages/core/src/utils/utc.ts new file mode 100644 index 000000000..1e348ef78 --- /dev/null +++ b/packages/core/src/utils/utc.ts @@ -0,0 +1,28 @@ +const getUtcDate = (date: Date | number) => { + const _date = new Date(date); + return new Date(Date.UTC(_date.getUTCFullYear(), _date.getUTCMonth(), _date.getUTCDate())); +}; + +/** + * Get the start timestamp of the day in UTC, with time set to 00:00:00.000 + * @param date date + * @returns timestamp in milliseconds + */ +export const getUtcStartOfTheDay = (date: Date | number) => + new Date(getUtcDate(date).setUTCHours(0, 0, 0, 0)).getTime(); + +/** + * Get the end timestamp of the day in UTC, with time set to 23:59:59.999 + * @param date date + * @returns timestamp in milliseconds + */ +export const getUtcEndOfTheDay = (date: Date | number) => + new Date(getUtcDate(date).setUTCHours(23, 59, 59, 999)).getTime(); + +/** + * Get date string in yyyy-MM-dd format in UTC + * @param date + * @returns date string in yyyy-MM-dd format + */ +export const getUtcDateString = (date: Date | number) => + getUtcDate(date).toISOString().slice(0, 'yyyy-MM-dd'.length);