feat(v3.3): faster stats

prisma/migrations/20220103232702_stats/migration.sql

@@ -0,0 +1,8 @@
+-- CreateTable
+CREATE TABLE "Stats" (
+    "id" SERIAL NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "data" JSONB NOT NULL,
+
+    CONSTRAINT "Stats_pkey" PRIMARY KEY ("id")
+);

prisma/schema.prisma

@@ -75,3 +75,9 @@ model InvisibleUrl {
   urlId String
   url   Url    @relation(fields: [urlId], references: [id])
 }
+
+model Stats {
+  id         Int @id @default(autoincrement())
+  created_at DateTime @default(now())
+  data       Json
+}

server/index.js

@@ -1,9 +1,8 @@
 const next = require('next');
 const { createServer } = require('http');
-const { stat, mkdir } = require('fs/promises');
+const { stat, mkdir, readdir } = require('fs/promises');
 const { execSync } = require('child_process');
-const { extname } = require('path');
-const { red, green, bold } = require('colorette');
+const { extname, join } = require('path');
 const { PrismaClient } = require('@prisma/client');
 const validateConfig = require('./validateConfig');
 const Logger = require('../src/lib/logger');
@@ -125,6 +124,22 @@ function shouldUseYarn() {
     });
 
     srv.listen(config.core.port, config.core.host ?? '0.0.0.0');
+
+    const stats = await getStats(prisma, config);
+    await prisma.stats.create({
+      data: {
+        data: stats,
+      },
+    });
+    setInterval(async () => {
+      const stats = await getStats(prisma, config);
+      await prisma.stats.create({
+        data: {
+          data: stats,
+        },
+      });
+      if (config.core.logger) Logger.get('server').info('stats updated');
+    }, config.core.stats_interval * 1000);
   } catch (e) {
     if (e.message && e.message.startsWith('Could not find a production')) {
       Logger.get('web').error(`there is no production build - run \`${shouldUseYarn() ? 'yarn build' : 'npm build'}\``);
@@ -136,3 +151,80 @@ function shouldUseYarn() {
     }
   }
 })();
+
+async function sizeOfDir(directory) {
+  const files = await readdir(directory);
+  
+  let size = 0;
+  for (let i = 0, L = files.length; i !== L; ++i) {
+    const sta = await stat(join(directory, files[i]));
+    size += sta.size;
+  }
+
+  return size;
+}
+
+function bytesToRead(bytes) {
+  const units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'];
+  let num = 0;
+
+  while (bytes > 1024) {
+    bytes /= 1024;
+    ++num;
+  }
+
+  return `${bytes.toFixed(1)} ${units[num]}`;
+}
+
+
+async function getStats(prisma, config) {
+  const size = await sizeOfDir(join(process.cwd(), config.uploader.directory));
+  const byUser = await prisma.image.groupBy({
+    by: ['userId'],
+    _count: {
+      _all: true,
+    },
+  });
+  const count_users = await prisma.user.count();
+
+  const count_by_user = [];
+  for (let i = 0, L = byUser.length; i !== L; ++i) {
+    const user = await prisma.user.findFirst({
+      where: {
+        id: byUser[i].userId,
+      },
+    });
+
+    count_by_user.push({
+      username: user.username,
+      count: byUser[i]._count._all,
+    });
+  }
+
+  const count = await prisma.image.count();
+  const viewsCount = await prisma.image.groupBy({
+    by: ['views'],
+    _sum: {
+      views: true,
+    },
+  });
+
+  const typesCount = await prisma.image.groupBy({
+    by: ['mimetype'],
+    _count: {
+      mimetype: true,
+    },
+  });
+  const types_count = [];
+  for (let i = 0, L = typesCount.length; i !== L; ++i) types_count.push({ mimetype: typesCount[i].mimetype, count: typesCount[i]._count.mimetype });
+
+  return {
+    size: bytesToRead(size),
+    size_num: size,
+    count,
+    count_by_user: count_by_user.sort((a,b) => b.count-a.count),
+    count_users,
+    views_count: (viewsCount[0]?._sum?.views ?? 0),
+    types_count: types_count.sort((a,b) => b.count-a.count),
+  };
+}

server/validateConfig.js

@@ -10,6 +10,7 @@ const validator = yup.object({
     port: yup.number().default(3000),
     database_url: yup.string().required(),
     logger: yup.boolean().default(true),
+    stats_interval: yup.number().default(1800),
   }).required(),
   uploader: yup.object({
     route: yup.string().required(),

src/components/Layout.tsx

@@ -87,7 +87,7 @@ function CopyTokenDialog({ open, setOpen, token }) {
         </DialogTitle>
         <DialogContent>
           <DialogContentText id='copy-dialog-description'>
-              Make sure you don&apos;t share this token with anyone as they will be able to upload images on your behalf.
+              Make sure you don&apos;t share this token with anyone as they will be able to upload files on your behalf.
           </DialogContentText>
         </DialogContent>
         <DialogActions>

src/lib/readConfig.js

@@ -11,6 +11,7 @@ const envValues = [
   e('PORT', 'number', (c, v) => c.core.port = v),
   e('DATABASE_URL', 'string', (c, v) => c.core.database_url = v),
   e('LOGGER', 'boolean', (c, v) => c.core.logger = v ?? true),
+  e('STATS_INTERVAL', 'number', (c, v) => c.core.stats_interval = v),
 
   e('UPLOADER_ROUTE', 'string', (c, v) => c.uploader.route = v),
   e('UPLOADER_LENGTH', 'number', (c, v) => c.uploader.length = v),
@@ -48,6 +49,7 @@ function tryReadEnv() {
       port: undefined,
       database_url: undefined,
       logger: undefined,
+      stats_interval: undefined,
     },
     uploader: {
       route: undefined,

src/lib/types.ts

@@ -16,6 +16,9 @@ export interface ConfigCore {
 
   // Whether or not to log stuff
   logger: boolean;
+
+  // The interval to store stats
+  stats_interval: number;
 }
 
 export interface ConfigUploader {

src/lib/util.ts

@@ -4,6 +4,7 @@ import { readdir, stat } from 'fs/promises';
 import { join } from 'path';
 import prisma from './prisma';
 import { InvisibleImage, InvisibleUrl } from '@prisma/client';
+import config from './config';
 
 export async function hashPassword(s: string): Promise<string> {
   return await hash(s);

src/pages/api/stats.ts

@@ -1,62 +1,18 @@
-import { join } from 'path';
 import { NextApiReq, NextApiRes, withZipline } from 'middleware/withZipline';
 import prisma from 'lib/prisma';
-import { bytesToRead, sizeOfDir } from 'lib/util';
-import config from 'lib/config';
 
 async function handler(req: NextApiReq, res: NextApiRes) {
   const user = await req.user();
   if (!user) return res.forbid('not logged in');
 
-  const size = await sizeOfDir(join(process.cwd(), config.uploader.directory));
-  const byUser = await prisma.image.groupBy({
-    by: ['userId'],
-    _count: {
-      _all: true,
-    },
-  });
-  const count_users = await prisma.user.count();
-
-  const count_by_user = [];
-  for (let i = 0, L = byUser.length; i !== L; ++i) {
-    const user = await prisma.user.findFirst({
-      where: {
-        id: byUser[i].userId,
+  const stats = await prisma.stats.findFirst({
+    orderBy: {
+      created_at: 'desc',
     },
+    take: 1,
   });
 
-    count_by_user.push({
-      username: user.username,
-      count: byUser[i]._count._all,
-    });
-  }
-
-  const count = await prisma.image.count();
-  const viewsCount = await prisma.image.groupBy({
-    by: ['views'],
-    _sum: {
-      views: true,
-    },
-  });
-
-  const typesCount = await prisma.image.groupBy({
-    by: ['mimetype'],
-    _count: {
-      mimetype: true,
-    },
-  });
-  const types_count = [];
-  for (let i = 0, L = typesCount.length; i !== L; ++i) types_count.push({ mimetype: typesCount[i].mimetype, count: typesCount[i]._count.mimetype });
-
-  return res.json({
-    size: bytesToRead(size),
-    size_num: size,
-    count,
-    count_by_user: count_by_user.sort((a,b) => b.count-a.count),
-    count_users,
-    views_count: (viewsCount[0]?._sum?.views ?? 0),
-    types_count: types_count.sort((a,b) => b.count-a.count),
-  });
+  return res.json(stats.data);
 }
 
 export default withZipline(handler);

yarn.lock

@@ -1405,9 +1405,9 @@ camelcase@^5.3.1:
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
 
 caniuse-lite@^1.0.30001202, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001228:
-  version "1.0.30001237"
-  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz"
-  integrity sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==
+  version "1.0.30001295"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001295.tgz"
+  integrity sha512-lSP16vcyC0FEy0R4ECc9duSPoKoZy+YkpGkue9G4D81OfPnliopaZrU10+qtPdT8PbGXad/PNx43TIQrOmJZSQ==
 
 capitalize@1.0.0:
   version "1.0.0"