refactor(core): migrate dashboard api data deps

.changeset/nice-walls-admire.md

@@ -0,0 +1,10 @@
+---
+"@logto/core": patch
+---
+
+migrate `/dashboard` API data source
+
+Updates:
+- the responded data from `/dashboard/users/new` API is now based on UTC time
+- the responded data from `/dashboard/users/active` API is now based on UTC time
+- the query parameter `date` for `/dashboard/users/active` API is now based on UTC time

packages/core/src/event-listeners/record-active-users.ts

@@ -1,7 +1,7 @@
 import { generateStandardId } from '@logto/shared';
 
-import { getUtcStartOfTheDay } from '#src/oidc/utils.js';
 import type Queries from '#src/tenants/Queries.js';
+import { getUtcStartOfTheDay } from '#src/utils/utc.js';
 
 export const recordActiveUsers = async (accessToken: { accountId?: string }, queries: Queries) => {
   const { accountId } = accessToken;
@@ -16,6 +16,6 @@ export const recordActiveUsers = async (accessToken: { accountId?: string }, que
   await insertActiveUser({
     id: generateStandardId(),
     userId: accountId,
-    date: getUtcStartOfTheDay(new Date()).getTime(),
+    date: getUtcStartOfTheDay(new Date()),
   });
 };

packages/core/src/oidc/utils.ts

@@ -63,9 +63,3 @@ export const isOriginAllowed = (
 
   return [...corsAllowedOrigins, ...redirectUriOrigins].includes(origin);
 };
-
-export const getUtcStartOfTheDay = (date: Date) => {
-  return new Date(
-    Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), 0, 0, 0, 0)
-  );
-};

packages/core/src/queries/daily-active-user.ts

@@ -1,14 +1,42 @@
 import { DailyActiveUsers } from '@logto/schemas';
-import type { CommonQueryMethods } from 'slonik';
+import { convertToIdentifiers } from '@logto/shared';
+import { sql, type CommonQueryMethods } from 'slonik';
 
 import { buildInsertIntoWithPool } from '#src/database/insert-into.js';
 
+const { table, fields } = convertToIdentifiers(DailyActiveUsers);
+
 export const createDailyActiveUsersQueries = (pool: CommonQueryMethods) => {
   const insertActiveUser = buildInsertIntoWithPool(pool)(DailyActiveUsers, {
     onConflict: { ignore: true },
   });
 
+  const getDailyActiveUserCountsByTimeInterval = async (
+    startTimeExclusive: number,
+    endTimeInclusive: number
+  ) =>
+    pool.any<{ date: string; count: number }>(sql`
+      select date(${fields.date}), count(distinct(${fields.userId}))
+      from ${table}
+      where ${fields.date} > to_timestamp(${startTimeExclusive}::double precision / 1000)
+      and ${fields.date} <= to_timestamp(${endTimeInclusive}::double precision / 1000)
+      group by date(${fields.date})
+    `);
+
+  const countActiveUsersByTimeInterval = async (
+    startTimeExclusive: number,
+    endTimeInclusive: number
+  ) =>
+    pool.one<{ count: number }>(sql`
+      select count(distinct(${fields.userId}))
+      from ${table}
+      where ${fields.date} > to_timestamp(${startTimeExclusive}::double precision / 1000)
+      and ${fields.date} <= to_timestamp(${endTimeInclusive}::double precision / 1000)
+    `);
+
   return {
     insertActiveUser,
+    getDailyActiveUserCountsByTimeInterval,
+    countActiveUsersByTimeInterval,
   };
 };

packages/core/src/queries/daily-token-usage.ts

@@ -3,7 +3,7 @@ import { convertToIdentifiers, generateStandardId } from '@logto/shared';
 import type { CommonQueryMethods } from 'slonik';
 import { sql } from 'slonik';
 
-import { getUtcStartOfTheDay } from '#src/oidc/utils.js';
+import { getUtcStartOfTheDay } from '#src/utils/utc.js';
 
 const { table, fields } = convertToIdentifiers(DailyTokenUsage);
 const { fields: fieldsWithPrefix } = convertToIdentifiers(DailyTokenUsage, true);
@@ -33,7 +33,7 @@ export const createDailyTokenUsageQueries = (pool: CommonQueryMethods) => {
       insert into ${table} (${fields.id}, ${fields.date}, ${fields.usage})
       values (${generateStandardId()}, to_timestamp(${getUtcStartOfTheDay(
         date
-      ).getTime()}::double precision / 1000), 1)
+      )}::double precision / 1000), 1)
       on conflict (${fields.date}, ${fields.tenantId}) do update set ${fields.usage} = ${
         fieldsWithPrefix.usage
       } + 1

packages/core/src/queries/log.ts

@@ -1,5 +1,5 @@
 import {
-  token,
+  type token,
   type hook,
   Logs,
   type HookExecutionStats,
@@ -77,33 +77,6 @@ export const createLogQueries = (pool: CommonQueryMethods) => {
 
   const findLogById = buildFindEntityByIdWithPool(pool)(Logs);
 
-  const getDailyActiveUserCountsByTimeInterval = async (
-    startTimeExclusive: number,
-    endTimeInclusive: number
-  ) =>
-    pool.any<{ date: string; count: number }>(sql`
-      select date(${fields.createdAt}), count(distinct(${fields.payload}->>'userId'))
-      from ${table}
-      where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000)
-      and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000)
-      and ${fields.key} like ${`${token.Type.ExchangeTokenBy}.%`}
-      and ${fields.payload}->>'result' = 'Success'
-      group by date(${fields.createdAt})
-    `);
-
-  const countActiveUsersByTimeInterval = async (
-    startTimeExclusive: number,
-    endTimeInclusive: number
-  ) =>
-    pool.one<{ count: number }>(sql`
-      select count(distinct(${fields.payload}->>'userId'))
-      from ${table}
-      where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000)
-      and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000)
-      and ${fields.key} like ${`${token.Type.ExchangeTokenBy}.%`}
-      and ${fields.payload}->>'result' = 'Success'
-    `);
-
   const getHookExecutionStatsByHookId = async (hookId: string) => {
     const startTimeExclusive = subDays(new Date(), 1).getTime();
     return pool.one<HookExecutionStats>(sql`
@@ -120,8 +93,6 @@ export const createLogQueries = (pool: CommonQueryMethods) => {
     countLogs,
     findLogs,
     findLogById,
-    getDailyActiveUserCountsByTimeInterval,
-    countActiveUsersByTimeInterval,
     getHookExecutionStatsByHookId,
   };
 };

packages/core/src/queries/user.ts

@@ -253,12 +253,17 @@ export const createUserQueries = (pool: CommonQueryMethods) => {
     startTimeExclusive: number,
     endTimeInclusive: number
   ) =>
+    /**
+     * Since the DAU data stored in the `daily_active_users` table is in UTC, we need to convert
+     * the `created_at` time to UTC in the result. This keeps the result consistent with the daily
+     * active user counts.
+     */
     pool.any<{ date: string; count: number }>(sql`
-      select date(${fields.createdAt}), count(*)
+      select date(${fields.createdAt} at time zone 'UTC'), count(*)
       from ${table}
       where ${fields.createdAt} > to_timestamp(${startTimeExclusive}::double precision / 1000)
       and ${fields.createdAt} <= to_timestamp(${endTimeInclusive}::double precision / 1000)
-      group by date(${fields.createdAt})
+      group by date(${fields.createdAt} at time zone 'UTC')
     `);
 
   return {

packages/core/src/routes/dashboard.openapi.json

@@ -21,7 +21,7 @@
     "/api/dashboard/users/new": {
       "get": {
         "summary": "Get new user count",
-        "description": "Get new user count in the past 7 days.",
+        "description": "Get new user count in the past 7 days. Based on UTC time.",
         "parameters": [],
         "responses": {
           "200": {
@@ -33,12 +33,12 @@
     "/api/dashboard/users/active": {
       "get": {
         "summary": "Get active user data",
-        "description": "Get active user data, including daily active user (DAU), weekly active user (WAU) and monthly active user (MAU). It also includes an array of DAU in the past 30 days.",
+        "description": "Get active user data, including daily active user (DAU), weekly active user (WAU) and monthly active user (MAU). It also includes an array of DAU in the past 30 days. Based on UTC time.",
         "parameters": [
           {
             "in": "query",
             "name": "date",
-            "description": "The date to get active user data."
+            "description": "The date (based on UTC) to get active user data."
           }
         ],
         "responses": {

packages/core/src/routes/dashboard.test.ts

@@ -1,11 +1,12 @@
 // The FP version works better for `format()`
 /* eslint-disable import/no-duplicates */
 import { pickDefault } from '@logto/shared/esm';
-import { endOfDay, subDays } from 'date-fns';
+import { subDays } from 'date-fns';
 import { format } from 'date-fns/fp';
 
 import { MockTenant } from '#src/test-utils/tenant.js';
 import { createRequester } from '#src/utils/test-utils.js';
+import { getUtcEndOfTheDay } from '#src/utils/utc.js';
 /* eslint-enable import/no-duplicates */
 
 const { jest } = import.meta;
@@ -41,13 +42,13 @@ const users = {
 };
 const { countUsers, getDailyNewUserCountsByTimeInterval } = users;
 
-const logs = {
+const dailyActiveUsers = {
   getDailyActiveUserCountsByTimeInterval: jest.fn().mockResolvedValue(mockDailyActiveUserCounts),
   countActiveUsersByTimeInterval: jest.fn().mockResolvedValue({ count: mockActiveUserCount }),
 };
-const { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval } = logs;
+const { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval } = dailyActiveUsers;
 
-const tenantContext = new MockTenant(undefined, { logs, users });
+const tenantContext = new MockTenant(undefined, { dailyActiveUsers, users });
 const dashboardRoutes = await pickDefault(import('./dashboard.js'));
 
 describe('dashboardRoutes', () => {
@@ -72,14 +73,14 @@ describe('dashboardRoutes', () => {
 
   describe('GET /dashboard/users/new', () => {
     beforeEach(() => {
-      jest.useFakeTimers().setSystemTime(new Date('2022-05-14'));
+      jest.useFakeTimers().setSystemTime(Date.UTC(2022, 4, 14));
     });
 
     it('should call getDailyNewUserCountsByTimeInterval with the time interval (14 days ago 23:59:59.999, today 23:59:59.999]', async () => {
       await logRequest.get('/dashboard/users/new');
       expect(getDailyNewUserCountsByTimeInterval).toHaveBeenCalledWith(
-        subDays(endOfDay(Date.now()), 14).valueOf(),
-        endOfDay(Date.now()).valueOf()
+        getUtcEndOfTheDay(subDays(Date.now(), 14)),
+        getUtcEndOfTheDay(Date.now())
       );
     });
 
@@ -100,7 +101,7 @@ describe('dashboardRoutes', () => {
   });
 
   describe('GET /dashboard/users/active', () => {
-    const mockToday = new Date(2022, 4, 30);
+    const mockToday = Date.UTC(2022, 4, 30);
 
     beforeEach(() => {
       jest.useFakeTimers().setSystemTime(mockToday);
@@ -112,44 +113,44 @@ describe('dashboardRoutes', () => {
     });
 
     it('should call getDailyActiveUserCountsByTimeInterval with the time interval (2022-05-31, 2022-06-30] when the parameter `date` is 2022-06-30', async () => {
-      const targetDate = new Date(2022, 5, 30);
+      const targetDate = Date.UTC(2022, 5, 30);
       await logRequest.get(`/dashboard/users/active?date=${formatToQueryDate(targetDate)}`);
       expect(getDailyActiveUserCountsByTimeInterval).toHaveBeenCalledWith(
-        endOfDay(new Date(2022, 4, 31)).valueOf(),
-        endOfDay(targetDate).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 31)),
+        getUtcEndOfTheDay(targetDate)
       );
     });
 
     it('should call getDailyActiveUserCountsByTimeInterval with the time interval (30 days ago, tomorrow] when there is no parameter `date`', async () => {
       await logRequest.get('/dashboard/users/active');
       expect(getDailyActiveUserCountsByTimeInterval).toHaveBeenCalledWith(
-        endOfDay(new Date(2022, 3, 30)).valueOf(),
-        endOfDay(mockToday).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 3, 30)),
+        getUtcEndOfTheDay(mockToday)
       );
     });
 
     it('should call countActiveUsersByTimeInterval with correct parameters when the parameter `date` is 2022-06-30', async () => {
-      const targetDate = new Date(2022, 5, 30);
+      const targetDate = Date.UTC(2022, 5, 30);
       await logRequest.get(`/dashboard/users/active?date=${formatToQueryDate(targetDate)}`);
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         1,
-        endOfDay(new Date(2022, 5, 16)).valueOf(),
-        endOfDay(new Date(2022, 5, 23)).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 5, 16)),
+        getUtcEndOfTheDay(Date.UTC(2022, 5, 23))
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         2,
-        endOfDay(new Date(2022, 5, 23)).valueOf(),
-        endOfDay(targetDate).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 5, 23)),
+        getUtcEndOfTheDay(targetDate)
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         3,
-        endOfDay(new Date(2022, 4, 1)).valueOf(),
-        endOfDay(new Date(2022, 4, 31)).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 1)),
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 31))
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         4,
-        endOfDay(new Date(2022, 4, 31)).valueOf(),
-        endOfDay(targetDate).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 31)),
+        getUtcEndOfTheDay(targetDate)
       );
     });
 
@@ -157,23 +158,23 @@ describe('dashboardRoutes', () => {
       await logRequest.get('/dashboard/users/active');
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         1,
-        endOfDay(new Date(2022, 4, 16)).valueOf(),
-        endOfDay(new Date(2022, 4, 23)).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 16)),
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 23))
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         2,
-        endOfDay(new Date(2022, 4, 23)).valueOf(),
-        endOfDay(mockToday).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 4, 23)),
+        getUtcEndOfTheDay(mockToday)
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         3,
-        endOfDay(new Date(2022, 2, 31)).valueOf(),
-        endOfDay(new Date(2022, 3, 30)).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 2, 31)),
+        getUtcEndOfTheDay(Date.UTC(2022, 3, 30))
       );
       expect(countActiveUsersByTimeInterval).toHaveBeenNthCalledWith(
         4,
-        endOfDay(new Date(2022, 3, 30)).valueOf(),
-        endOfDay(mockToday).valueOf()
+        getUtcEndOfTheDay(Date.UTC(2022, 3, 30)),
+        getUtcEndOfTheDay(mockToday)
       );
     });
 

packages/core/src/routes/dashboard.ts

@@ -1,24 +1,21 @@
 import { dateRegex } from '@logto/core-kit';
 import { getNewUsersResponseGuard, getActiveUsersResponseGuard } from '@logto/schemas';
-import { endOfDay, format, subDays } from 'date-fns';
+import { subDays } from 'date-fns';
 import { number, object, string } from 'zod';
 
 import koaGuard from '#src/middleware/koa-guard.js';
+import { getUtcDateString, getUtcEndOfTheDay } from '#src/utils/utc.js';
 
 import type { AuthedRouter, RouterInitArgs } from './types.js';
 
-const getDateString = (date: Date | number) => format(date, 'yyyy-MM-dd');
-
 const indices = (length: number) => [...Array.from({ length }).keys()];
 
-const getEndOfDayTimestamp = (date: Date | number) => endOfDay(date).valueOf();
-
 export default function dashboardRoutes<T extends AuthedRouter>(
   ...[router, { queries }]: RouterInitArgs<T>
 ) {
   const {
-    logs: { countActiveUsersByTimeInterval, getDailyActiveUserCountsByTimeInterval },
     users: { countUsers, getDailyNewUserCountsByTimeInterval },
+    dailyActiveUsers: { getDailyActiveUserCountsByTimeInterval, countActiveUsersByTimeInterval },
   } = queries;
 
   router.get(
@@ -47,24 +44,24 @@ export default function dashboardRoutes<T extends AuthedRouter>(
       const today = Date.now();
       const dailyNewUserCounts = await getDailyNewUserCountsByTimeInterval(
         // (14 days ago 23:59:59.999, today 23:59:59.999]
-        getEndOfDayTimestamp(subDays(today, 14)),
-        getEndOfDayTimestamp(today)
+        getUtcEndOfTheDay(subDays(today, 14)),
+        getUtcEndOfTheDay(today)
       );
 
       const last14DaysNewUserCounts = new Map(
         dailyNewUserCounts.map(({ date, count }) => [date, count])
       );
 
-      const todayNewUserCount = last14DaysNewUserCounts.get(getDateString(today)) ?? 0;
+      const todayNewUserCount = last14DaysNewUserCounts.get(getUtcDateString(today)) ?? 0;
       const yesterday = subDays(today, 1);
-      const yesterdayNewUserCount = last14DaysNewUserCounts.get(getDateString(yesterday)) ?? 0;
+      const yesterdayNewUserCount = last14DaysNewUserCounts.get(getUtcDateString(yesterday)) ?? 0;
       const todayDelta = todayNewUserCount - yesterdayNewUserCount;
 
       const last7DaysNewUserCount = indices(7)
-        .map((index) => getDateString(subDays(today, index)))
+        .map((index) => getUtcDateString(subDays(today, index)))
         .reduce((sum, date) => sum + (last14DaysNewUserCounts.get(date) ?? 0), 0);
       const newUserCountFrom13DaysAgoTo7DaysAgo = indices(7)
-        .map((index) => getDateString(subDays(today, index + 7)))
+        .map((index) => getUtcDateString(subDays(today, index + 7)))
         .reduce((sum, date) => sum + (last14DaysNewUserCounts.get(date) ?? 0), 0);
       const last7DaysDelta = last7DaysNewUserCount - newUserCountFrom13DaysAgoTo7DaysAgo;
 
@@ -108,39 +105,39 @@ export default function dashboardRoutes<T extends AuthedRouter>(
       ] = await Promise.all([
         getDailyActiveUserCountsByTimeInterval(
           // (30 days ago 23:59:59.999, target day 23:59:59.999]
-          getEndOfDayTimestamp(subDays(targetDay, 30)),
-          getEndOfDayTimestamp(targetDay)
+          getUtcEndOfTheDay(subDays(targetDay, 30)),
+          getUtcEndOfTheDay(targetDay)
         ),
         countActiveUsersByTimeInterval(
           // (14 days ago 23:59:59.999, 7 days ago 23:59:59.999]
-          getEndOfDayTimestamp(subDays(targetDay, 14)),
-          getEndOfDayTimestamp(subDays(targetDay, 7))
+          getUtcEndOfTheDay(subDays(targetDay, 14)),
+          getUtcEndOfTheDay(subDays(targetDay, 7))
         ),
         countActiveUsersByTimeInterval(
           // (7 days ago 23:59:59.999, target day 23:59:59.999]
-          getEndOfDayTimestamp(subDays(targetDay, 7)),
-          getEndOfDayTimestamp(targetDay)
+          getUtcEndOfTheDay(subDays(targetDay, 7)),
+          getUtcEndOfTheDay(targetDay)
         ),
         countActiveUsersByTimeInterval(
           // (60 days ago 23:59:59.999, 30 days ago 23:59:59.999]
-          getEndOfDayTimestamp(subDays(targetDay, 60)),
-          getEndOfDayTimestamp(subDays(targetDay, 30))
+          getUtcEndOfTheDay(subDays(targetDay, 60)),
+          getUtcEndOfTheDay(subDays(targetDay, 30))
         ),
         countActiveUsersByTimeInterval(
           // (30 days ago 23:59:59.999, target day 23:59:59.999]
-          getEndOfDayTimestamp(subDays(targetDay, 30)),
-          getEndOfDayTimestamp(targetDay)
+          getUtcEndOfTheDay(subDays(targetDay, 30)),
+          getUtcEndOfTheDay(targetDay)
         ),
       ]);
 
-      const previousDate = getDateString(subDays(targetDay, 1));
-      const targetDate = getDateString(targetDay);
+      const previousDate = getUtcDateString(subDays(targetDay, 1));
+      const targetDate = getUtcDateString(targetDay);
 
       const previousDAU = last30DauCounts.find(({ date }) => date === previousDate)?.count ?? 0;
       const dau = last30DauCounts.find(({ date }) => date === targetDate)?.count ?? 0;
 
       const dauCurve = indices(30).map((index) => {
-        const dateString = getDateString(subDays(targetDay, 29 - index));
+        const dateString = getUtcDateString(subDays(targetDay, 29 - index));
         const count = last30DauCounts.find(({ date }) => date === dateString)?.count ?? 0;
 
         return { date: dateString, count };

packages/core/src/utils/utc.ts

@@ -0,0 +1,28 @@
+const getUtcDate = (date: Date | number) => {
+  const _date = new Date(date);
+  return new Date(Date.UTC(_date.getUTCFullYear(), _date.getUTCMonth(), _date.getUTCDate()));
+};
+
+/**
+ * Get the start timestamp of the day in UTC, with time set to 00:00:00.000
+ * @param date date
+ * @returns timestamp in milliseconds
+ */
+export const getUtcStartOfTheDay = (date: Date | number) =>
+  new Date(getUtcDate(date).setUTCHours(0, 0, 0, 0)).getTime();
+
+/**
+ * Get the end timestamp of the day in UTC, with time set to 23:59:59.999
+ * @param date date
+ * @returns timestamp in milliseconds
+ */
+export const getUtcEndOfTheDay = (date: Date | number) =>
+  new Date(getUtcDate(date).setUTCHours(23, 59, 59, 999)).getTime();
+
+/**
+ * Get date string in yyyy-MM-dd format in UTC
+ * @param date
+ * @returns date string in yyyy-MM-dd format
+ */
+export const getUtcDateString = (date: Date | number) =>
+  getUtcDate(date).toISOString().slice(0, 'yyyy-MM-dd'.length);