feat(cli): create translation command

iterate phrases files and translate to a new language using OpenAI API

packages/cli/package.json

@@ -45,6 +45,9 @@
   "dependencies": {
     "@logto/connector-kit": "workspace:^",
     "@logto/core-kit": "workspace:^",
+    "@logto/language-kit": "workspace:^",
+    "@logto/phrases": "workspace:^",
+    "@logto/phrases-ui": "workspace:^",
     "@logto/schemas": "workspace:^",
     "@logto/shared": "workspace:^",
     "@silverhand/essentials": "^2.5.0",

packages/cli/src/commands/translate/create.ts

@@ -0,0 +1,94 @@
+import { existsSync } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { isLanguageTag, type LanguageTag } from '@logto/language-kit';
+import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases';
+import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui';
+import { conditionalString, type Optional } from '@silverhand/essentials';
+import type { CommandModule } from 'yargs';
+
+import { log } from '../../utils.js';
+import { inquireInstancePath } from '../connector/utils.js';
+
+import { createOpenaiApi, translate } from './openai.js';
+import { baseLanguage, readBaseLocaleFiles } from './utils.js';
+
+const createFullTranslation = async (
+  instancePath: Optional<string>,
+  packageName: 'phrases' | 'phrases-ui',
+  languageTag: LanguageTag
+) => {
+  const directory = path.join(
+    await inquireInstancePath(instancePath),
+    'packages',
+    packageName,
+    'src/locales'
+  );
+  const files = await readBaseLocaleFiles(directory);
+
+  log.info(
+    'Found ' +
+      String(files.length) +
+      ' file' +
+      conditionalString(files.length !== 1 && 's') +
+      ' in ' +
+      packageName +
+      ' to translate'
+  );
+
+  const openai = createOpenaiApi();
+
+  /* eslint-disable no-await-in-loop */
+  for (const file of files) {
+    const relativePath = path.relative(path.join(directory, baseLanguage), file);
+    const targetPath = path.join(directory, languageTag, relativePath);
+
+    if (existsSync(targetPath)) {
+      log.info(`Target path ${targetPath} exists, skipping`);
+      continue;
+    }
+
+    log.info(`Translating ${file}`);
+    const result = await translate(openai, languageTag, file);
+
+    if (!result) {
+      log.error(`Unable to translate ${file}`);
+    }
+
+    await fs.mkdir(path.parse(targetPath).dir, { recursive: true });
+    await fs.writeFile(targetPath, result);
+    log.succeed(`Translated ${targetPath}`);
+  }
+  /* eslint-enable no-await-in-loop */
+};
+
+const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': string }> = {
+  command: ['create <language-tag>', 'c'],
+  describe: 'Create a new language translation',
+  builder: (yargs) =>
+    yargs.positional('language-tag', {
+      describe: 'The language tag to create, e.g. `af-ZA`.',
+      type: 'string',
+      demandOption: true,
+    }),
+  handler: async ({ path: inputPath, languageTag }) => {
+    if (!isLanguageTag(languageTag)) {
+      log.error('Invalid language tag. Run `logto translate list-tags` to see available list.');
+    }
+
+    if (isPhrasesBuiltInLanguageTag(languageTag)) {
+      log.info(languageTag + ' is a built-in tag of phrases, skipping');
+    } else {
+      await createFullTranslation(inputPath, 'phrases', languageTag);
+    }
+
+    if (isPhrasesUiBuiltInLanguageTag(languageTag)) {
+      log.info(languageTag + ' is a built-in tag of phrases-ui, skipping');
+    } else {
+      await createFullTranslation(inputPath, 'phrases-ui', languageTag);
+    }
+  },
+};
+
+export default create;

packages/cli/src/commands/translate/index.ts

@@ -0,0 +1,23 @@
+import { noop } from '@silverhand/essentials';
+import type { CommandModule } from 'yargs';
+
+import create from './create.js';
+import listTags from './list-tags.js';
+
+const translate: CommandModule = {
+  command: ['translate', 't'],
+  describe: 'Commands for Logto translation',
+  builder: (yargs) =>
+    yargs
+      .option('path', {
+        alias: 'p',
+        type: 'string',
+        describe: 'The path to your Logto instance directory',
+      })
+      .command(create)
+      .command(listTags)
+      .demandCommand(1),
+  handler: noop,
+};
+
+export default translate;

packages/cli/src/commands/translate/list-tags.ts

@@ -0,0 +1,24 @@
+import { languages } from '@logto/language-kit';
+import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases';
+import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui';
+import chalk from 'chalk';
+import type { CommandModule } from 'yargs';
+
+const listTags: CommandModule<Record<string, unknown>> = {
+  command: ['list-tags', 'list'],
+  describe: 'List all available language tags',
+
+  handler: async () => {
+    for (const tag of Object.keys(languages)) {
+      console.log(
+        ...[
+          tag,
+          isPhrasesBuiltInLanguageTag(tag) && chalk.blue('phrases'),
+          isPhrasesUiBuiltInLanguageTag(tag) && chalk.blue('phrases-ui'),
+        ].filter(Boolean)
+      );
+    }
+  },
+};
+
+export default listTags;

packages/cli/src/commands/translate/openai.ts

@@ -0,0 +1,95 @@
+import fs from 'node:fs/promises';
+
+import { type LanguageTag } from '@logto/language-kit';
+import { trySafe } from '@silverhand/essentials';
+import { type Got, got, HTTPError } from 'got';
+import { HttpsProxyAgent } from 'hpagent';
+import { z } from 'zod';
+
+import { getProxy, log } from '../../utils.js';
+
+export const createOpenaiApi = () => {
+  const proxy = getProxy();
+
+  return got.extend({
+    prefixUrl: 'https://api.openai.com/v1',
+    headers: {
+      Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`,
+    },
+    timeout: { request: 120_000 },
+    ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy, timeout: 120_000 }) } }),
+  });
+};
+
+const gptResponseGuard = z.object({
+  choices: z
+    .object({
+      message: z.object({ role: z.string(), content: z.string() }),
+      finish_reason: z.string(),
+    })
+    .array(),
+});
+
+export const translate = async (api: Got, languageTag: LanguageTag, filePath: string) => {
+  const fileContent = await fs.readFile(filePath, 'utf8');
+  const response = await trySafe(
+    api
+      .post('chat/completions', {
+        json: {
+          model: 'gpt-3.5-turbo',
+          messages: [
+            {
+              role: 'user',
+              content: `Translate the following code snippet to ${languageTag}, output ts code only: \n \`\`\`ts\n${fileContent}\n\`\`\``,
+            },
+          ],
+        },
+      })
+      .json(),
+    (error) => {
+      log.warn(`Error while translating ${filePath}:`, String(error));
+
+      if (error instanceof HTTPError) {
+        log.warn(error.response.body);
+      }
+    }
+  );
+
+  if (!response) {
+    return;
+  }
+
+  const guarded = gptResponseGuard.safeParse(response);
+
+  if (!guarded.success) {
+    log.warn(`Error while guarding response for ${filePath}:`, response);
+
+    return;
+  }
+
+  const [entity] = guarded.data.choices;
+
+  if (!entity) {
+    log.warn(`No choice found in response when translating ${filePath}`);
+
+    return;
+  }
+
+  if (entity.finish_reason !== 'stop') {
+    log.warn(`Unexpected finish reason ${entity.finish_reason} for ${filePath}`);
+  }
+
+  const { content } = entity.message;
+  const matched = /```ts\n(.*)```/s.exec(content)?.[1];
+
+  if (!matched) {
+    // Treat as pure code
+    if (['const ', 'import '].some((prefix) => content.startsWith(prefix))) {
+      return content;
+    }
+
+    log.warn('No matching code snippet from response:', content);
+  }
+
+  return matched;
+};

packages/cli/src/commands/translate/utils.ts

@@ -0,0 +1,35 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { type LanguageTag } from '@logto/language-kit';
+
+import { log } from '../../utils.js';
+
+export const baseLanguage = 'en' satisfies LanguageTag;
+
+export const readLocaleFiles = async (directory: string): Promise<string[]> => {
+  const entities = await fs.readdir(directory, { withFileTypes: true });
+
+  const result = await Promise.all(
+    entities.map(async (entity) => {
+      if (entity.isDirectory()) {
+        return readLocaleFiles(path.join(directory, entity.name));
+      }
+
+      return entity.name.endsWith('.ts') ? path.join(directory, entity.name) : [];
+    })
+  );
+
+  return result.flat();
+};
+
+export const readBaseLocaleFiles = async (directory: string): Promise<string[]> => {
+  const enDirectory = path.join(directory, baseLanguage);
+  const stat = await fs.stat(enDirectory);
+
+  if (!stat.isDirectory()) {
+    log.error(directory, 'has no `' + baseLanguage + '` directory');
+  }
+
+  return readLocaleFiles(enDirectory);
+};

packages/cli/src/index.ts

@@ -6,6 +6,7 @@ import { hideBin } from 'yargs/helpers';
 import connector from './commands/connector/index.js';
 import database from './commands/database/index.js';
 import install from './commands/install/index.js';
+import translate from './commands/translate/index.js';
 import { packageJson } from './package-json.js';
 import { cliConfig, ConfigKey } from './utils.js';
 
@@ -46,6 +47,7 @@ void yargs(hideBin(process.argv))
   .command(install)
   .command(database)
   .command(connector)
+  .command(translate)
   .demandCommand(1)
   .showHelpOnFail(false, `Specify ${chalk.green('--help')} for available options`)
   .strict()

packages/cli/src/utils.ts

@@ -45,10 +45,15 @@ export const log: Log = Object.freeze({
   },
 });
 
-export const downloadFile = async (url: string, destination: string) => {
+export const getProxy = () => {
   const { HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy } = process.env;
+
+  return HTTPS_PROXY ?? https_proxy ?? HTTP_PROXY ?? http_proxy;
+};
+
+export const downloadFile = async (url: string, destination: string) => {
   const file = createWriteStream(destination);
-  const proxy = HTTPS_PROXY ?? https_proxy ?? HTTP_PROXY ?? http_proxy;
+  const proxy = getProxy();
   const stream = got.stream(url, {
     ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy }) } }),
   });

pnpm-lock.yaml

@@ -29,6 +29,9 @@ importers:
     specifiers:
       '@logto/connector-kit': workspace:^
       '@logto/core-kit': workspace:^
+      '@logto/language-kit': workspace:^1.0.0
+      '@logto/phrases': workspace:^
+      '@logto/phrases-ui': workspace:^
       '@logto/schemas': workspace:^
       '@logto/shared': workspace:^
       '@silverhand/eslint-config': 3.0.0
@@ -69,6 +72,9 @@ importers:
     dependencies:
       '@logto/connector-kit': link:../toolkit/connector-kit
       '@logto/core-kit': link:../toolkit/core-kit
+      '@logto/language-kit': link:../toolkit/language-kit
+      '@logto/phrases': link:../phrases
+      '@logto/phrases-ui': link:../phrases-ui
       '@logto/schemas': link:../schemas
       '@logto/shared': link:../shared
       '@silverhand/essentials': 2.5.0