Merge pull request #3652 from logto-io/gao-add-translation-sync

feat(cli): translation sync command

.changeset/seven-dolphins-hammer.md

@@ -0,0 +1,5 @@
+---
+"@logto/cli": minor
+---
+
+`logto translate sync` to create missing files or translate untranslated phrases across all existing languages

packages/cli/package.json

@@ -60,6 +60,7 @@
     "nanoid": "^4.0.0",
     "ora": "^6.1.2",
     "p-limit": "^4.0.0",
+    "p-queue": "^7.3.4",
     "p-retry": "^5.1.2",
     "pg-protocol": "^1.6.0",
     "roarr": "^7.11.0",

packages/cli/src/commands/translate/create.ts

@@ -1,84 +1,12 @@
-import { existsSync } from 'node:fs';
+import { isLanguageTag } from '@logto/language-kit';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-
-import { isLanguageTag, type LanguageTag } from '@logto/language-kit';
 import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases';
 import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui';
-import { conditionalString, type Optional } from '@silverhand/essentials';
 import type { CommandModule } from 'yargs';
 
 import { log } from '../../utils.js';
 import { inquireInstancePath } from '../connector/utils.js';
 
-import { createOpenaiApi, translate } from './openai.js';
+import { createFullTranslation } from './utils.js';
-import { baseLanguage, readBaseLocaleFiles } from './utils.js';
-
-const createFullTranslation = async (
-  instancePath: Optional<string>,
-  packageName: 'phrases' | 'phrases-ui',
-  languageTag: LanguageTag
-) => {
-  const directory = path.join(
-    await inquireInstancePath(instancePath),
-    'packages',
-    packageName,
-    'src/locales'
-  );
-  const files = await readBaseLocaleFiles(directory);
-
-  log.info(
-    'Found ' +
-      String(files.length) +
-      ' file' +
-      conditionalString(files.length !== 1 && 's') +
-      ' in ' +
-      packageName +
-      ' to translate'
-  );
-
-  const openai = createOpenaiApi();
-
-  /* eslint-disable no-await-in-loop */
-  for (const file of files) {
-    const relativePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file);
-    const targetPath = path.join(directory, languageTag.toLowerCase(), relativePath);
-
-    const getTranslationPath = async () => {
-      if (existsSync(targetPath)) {
-        const currentContent = await fs.readFile(targetPath, 'utf8');
-
-        if (currentContent.includes('// UNTRANSLATED')) {
-          return targetPath;
-        }
-
-        log.info(`Target path ${targetPath} exists and has no untranslated mark, skipping`);
-
-        return;
-      }
-
-      return file;
-    };
-
-    const translationPath = await getTranslationPath();
-
-    if (!translationPath) {
-      continue;
-    }
-
-    log.info(`Translating ${translationPath}`);
-    const result = await translate(openai, languageTag, translationPath);
-
-    if (!result) {
-      log.error(`Unable to translate ${translationPath}`);
-    }
-
-    await fs.mkdir(path.parse(targetPath).dir, { recursive: true });
-    await fs.writeFile(targetPath, result);
-    log.succeed(`Translated ${targetPath}`);
-  }
-  /* eslint-enable no-await-in-loop */
-};
 
 const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': string }> = {
   command: ['create <language-tag>', 'c'],
@@ -94,15 +22,21 @@ const create: CommandModule<{ path?: string }, { path?: string; 'language-tag':
       log.error('Invalid language tag. Run `logto translate list-tags` to see available list.');
     }
 
+    const instancePath = await inquireInstancePath(inputPath);
+
     if (isPhrasesBuiltInLanguageTag(languageTag)) {
       log.info(languageTag + ' is a built-in tag of phrases, updating untranslated phrases');
     }
-    await createFullTranslation(inputPath, 'phrases', languageTag);
+    await createFullTranslation({ instancePath, packageName: 'phrases', languageTag });
 
     if (isPhrasesUiBuiltInLanguageTag(languageTag)) {
       log.info(languageTag + ' is a built-in tag of phrases-ui, updating untranslated phrases');
     }
-    await createFullTranslation(inputPath, 'phrases-ui', languageTag);
+    await createFullTranslation({
+      instancePath,
+      packageName: 'phrases-ui',
+      languageTag,
+    });
   },
 };
 

packages/cli/src/commands/translate/index.ts

@@ -3,6 +3,7 @@ import type { CommandModule } from 'yargs';
 
 import create from './create.js';
 import listTags from './list-tags.js';
+import sync from './sync.js';
 
 const translate: CommandModule = {
   command: ['translate', 't'],
@@ -16,6 +17,7 @@ const translate: CommandModule = {
       })
       .command(create)
       .command(listTags)
+      .command(sync)
       .demandCommand(1),
   handler: noop,
 };

packages/cli/src/commands/translate/sync.ts

@@ -0,0 +1,50 @@
+import { languages } from '@logto/language-kit';
+import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases';
+import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui';
+import PQueue from 'p-queue';
+import type { CommandModule } from 'yargs';
+
+import { inquireInstancePath } from '../connector/utils.js';
+
+import { type CreateFullTranslation, baseLanguage, createFullTranslation } from './utils.js';
+
+const sync: CommandModule<{ path?: string }, { path?: string }> = {
+  command: ['sync'],
+  describe: 'Translate all untranslated phrases.',
+  handler: async ({ path: inputPath }) => {
+    const queue = new PQueue({ concurrency: 5 });
+    const instancePath = await inquireInstancePath(inputPath);
+
+    for (const languageTag of Object.keys(languages)) {
+      if (languageTag === baseLanguage) {
+        continue;
+      }
+
+      const baseOptions = {
+        instancePath,
+        verbose: false,
+        queue,
+      } satisfies Partial<CreateFullTranslation>;
+
+      if (isPhrasesBuiltInLanguageTag(languageTag)) {
+        void createFullTranslation({
+          ...baseOptions,
+          packageName: 'phrases',
+          languageTag,
+        });
+      }
+
+      if (isPhrasesUiBuiltInLanguageTag(languageTag)) {
+        void createFullTranslation({
+          ...baseOptions,
+          packageName: 'phrases-ui',
+          languageTag,
+        });
+      }
+    }
+
+    await queue.onIdle();
+  },
+};
+
+export default sync;

packages/cli/src/commands/translate/utils.ts

@@ -1,10 +1,15 @@
+import { existsSync } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
 import { type LanguageTag } from '@logto/language-kit';
+import { conditionalString } from '@silverhand/essentials';
+import PQueue from 'p-queue';
 
 import { log } from '../../utils.js';
 
+import { createOpenaiApi, translate } from './openai.js';
+
 export const baseLanguage = 'en' satisfies LanguageTag;
 
 export const readLocaleFiles = async (directory: string): Promise<string[]> => {
@@ -33,3 +38,82 @@ export const readBaseLocaleFiles = async (directory: string): Promise<string[]>
 
   return readLocaleFiles(enDirectory);
 };
+
+export type CreateFullTranslation = {
+  instancePath: string;
+  packageName: 'phrases' | 'phrases-ui';
+  languageTag: LanguageTag;
+  verbose?: boolean;
+  queue?: PQueue;
+};
+
+export const createFullTranslation = async ({
+  instancePath,
+  packageName,
+  languageTag,
+  verbose = true,
+  queue = new PQueue({ concurrency: 5 }),
+}: CreateFullTranslation) => {
+  const directory = path.join(instancePath, 'packages', packageName, 'src/locales');
+  const files = await readBaseLocaleFiles(directory);
+
+  if (verbose) {
+    log.info(
+      'Found ' +
+        String(files.length) +
+        ' file' +
+        conditionalString(files.length !== 1 && 's') +
+        ' in ' +
+        packageName +
+        ' to translate'
+    );
+  }
+
+  const openai = createOpenaiApi();
+
+  /* eslint-disable no-await-in-loop */
+  for (const file of files) {
+    const basePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file);
+    const targetPath = path.join(directory, languageTag.toLowerCase(), basePath);
+
+    const getTranslationPath = async () => {
+      if (existsSync(targetPath)) {
+        const currentContent = await fs.readFile(targetPath, 'utf8');
+
+        if (currentContent.includes('// UNTRANSLATED')) {
+          return targetPath;
+        }
+
+        if (verbose) {
+          log.info(`Target path ${targetPath} exists and has no untranslated mark, skipping`);
+        }
+
+        return;
+      }
+
+      return file;
+    };
+
+    const translationPath = await getTranslationPath();
+
+    if (!translationPath) {
+      continue;
+    }
+
+    void queue.add(async () => {
+      log.info(`Translating ${translationPath}`);
+      const result = await translate(openai, languageTag, translationPath);
+
+      if (!result) {
+        log.error(`Unable to translate ${translationPath}`);
+      }
+
+      await fs.mkdir(path.parse(targetPath).dir, { recursive: true });
+      await fs.writeFile(targetPath, result);
+      log.succeed(`Translated ${targetPath}`);
+    });
+  }
+  /* eslint-enable no-await-in-loop */
+
+  return queue.onIdle();
+};

pnpm-lock.yaml

@@ -135,6 +135,9 @@ importers:
       p-limit:
         specifier: ^4.0.0
         version: 4.0.0
+      p-queue:
+        specifier: ^7.3.4
+        version: 7.3.4
       p-retry:
         specifier: ^5.1.2
         version: 5.1.2
@@ -11876,6 +11879,14 @@ packages:
       type-fest: 3.5.2
     dev: false
 
+  /p-queue@7.3.4:
+    resolution: {integrity: sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg==}
+    engines: {node: '>=12'}
+    dependencies:
+      eventemitter3: 4.0.7
+      p-timeout: 5.1.0
+    dev: false
+
   /p-retry@5.1.2:
     resolution: {integrity: sha512-couX95waDu98NfNZV+i/iLt+fdVxmI7CbrrdC2uDWfPdUAApyxT4wmDlyOtR5KtTDmkDO0zDScDjDou9YHhd9g==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -11884,6 +11895,11 @@ packages:
       retry: 0.13.1
     dev: false
 
+  /p-timeout@5.1.0:
+    resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==}
+    engines: {node: '>=12'}
+    dev: false
+
   /p-try@2.2.0:
     resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
     engines: {node: '>=6'}