refactor(cli): improve translation cli (#3695)

packages/cli/src/commands/translate/create.ts

@@ -30,7 +30,11 @@ const create: CommandModule<{ path?: string }, { path?: string; 'language-tag':
     if (isPhrasesBuiltInLanguageTag(languageTag)) {
       consoleLog.info(languageTag + ' is a built-in tag of phrases, updating untranslated phrases');
     }
-    await createFullTranslation({ instancePath, packageName: 'phrases', languageTag });
+    await createFullTranslation({
+      instancePath,
+      packageName: 'phrases',
+      languageTag,
+    });
 
     if (isPhrasesUiBuiltInLanguageTag(languageTag)) {
       consoleLog.info(

packages/cli/src/commands/translate/openai.ts

@@ -1,6 +1,6 @@
 import fs from 'node:fs/promises';
 
-import { languages, type LanguageTag } from '@logto/language-kit';
+import { type LanguageTag } from '@logto/language-kit';
 import { trySafe } from '@silverhand/essentials';
 import { type Got, got, HTTPError } from 'got';
 import { HttpsProxyAgent } from 'hpagent';
@@ -8,6 +8,8 @@ import { z } from 'zod';
 
 import { consoleLog, getProxy } from '../../utils.js';
 
+import { getTranslationPrompt } from './prompts.js';
+
 export const createOpenaiApi = () => {
   const proxy = getProxy();
 
@@ -30,8 +32,20 @@ const gptResponseGuard = z.object({
     .array(),
 });
 
-export const translate = async (api: Got, languageTag: LanguageTag, filePath: string) => {
-  const fileContent = await fs.readFile(filePath, 'utf8');
+type TranslateConfig = {
+  api: Got;
+  sourceFilePath: string;
+  targetLanguage: LanguageTag;
+  extraPrompt?: string;
+};
+
+export const translate = async ({
+  api,
+  targetLanguage,
+  sourceFilePath,
+  extraPrompt,
+}: TranslateConfig) => {
+  const sourceFileContent = await fs.readFile(sourceFilePath, 'utf8');
   const response = await trySafe(
     api
       .post('chat/completions', {
@@ -40,14 +54,18 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st
           messages: [
             {
               role: 'user',
-              content: `Given the following code snippet, only translate object values to ${languages[languageTag]}, keep all object keys original, output ts code only: \n \`\`\`ts\n${fileContent}\n\`\`\``,
+              content: getTranslationPrompt({
+                sourceFileContent,
+                targetLanguage,
+                extraPrompt,
+              }),
             },
           ],
         },
       })
       .json(),
     (error) => {
-      consoleLog.warn(`Error while translating ${filePath}:`, String(error));
+      consoleLog.warn(`Error while translating ${sourceFilePath}:`, String(error));
 
       if (error instanceof HTTPError) {
         consoleLog.warn(error.response.body);
@@ -62,7 +80,7 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st
   const guarded = gptResponseGuard.safeParse(response);
 
   if (!guarded.success) {
-    consoleLog.warn(`Error while guarding response for ${filePath}:`, response);
+    consoleLog.warn(`Error while guarding response for ${sourceFilePath}:`, response);
 
     return;
   }
@@ -70,13 +88,13 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st
   const [entity] = guarded.data.choices;
 
   if (!entity) {
-    consoleLog.warn(`No choice found in response when translating ${filePath}`);
+    consoleLog.warn(`No choice found in response when translating ${sourceFilePath}`);
 
     return;
   }
 
   if (entity.finish_reason !== 'stop') {
-    consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${filePath}`);
+    consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${sourceFilePath}`);
   }
 
   const { content } = entity.message;

packages/cli/src/commands/translate/prompts.ts

@@ -0,0 +1,52 @@
+import { languages, type LanguageTag } from '@logto/language-kit';
+import { conditionalString } from '@silverhand/essentials';
+
+type GetTranslationPromptProperties = {
+  sourceFileContent: string;
+  targetLanguage: LanguageTag;
+  extraPrompt?: string;
+};
+
+/**
+ * Note:
+ * The input token limit of GPT 3.5 is 2048, the following prompt tokens with sourceFileContent is about 1200.
+ * Remember to check the token limit before adding more prompt.
+ * Tokens can be counted in https://platform.openai.com/tokenizer
+ */
+export const getTranslationPrompt = ({
+  sourceFileContent,
+  targetLanguage,
+  extraPrompt,
+}: GetTranslationPromptProperties) => `Given the following code snippet:
+\`\`\`ts
+${sourceFileContent}
+\`\`\`
+only translate object values to ${
+  languages[targetLanguage]
+}, keep all object keys original, output ts code only, the code format should be strictly consistent, and should not contains the given code snippet.
+
+Take zh-cn as an example, if the input is:
+\`\`\`ts
+import others from './others.js';
+
+const translation = {
+  hello: '你好',
+  world: 'world', // UNTRANSLATED
+  others,
+};
+
+export default translation;
+\`\`\`
+the output should be:
+\`\`\`ts
+import others from './others.js';
+
+const translation = {
+  hello: '你好',
+  world: '世界',
+  others,
+};
+
+export default translation;
+\`\`\`
+${conditionalString(extraPrompt)}`;

packages/cli/src/commands/translate/sync.ts

@@ -6,7 +6,7 @@ import type { CommandModule } from 'yargs';
 
 import { inquireInstancePath } from '../connector/utils.js';
 
-import { type CreateFullTranslation, baseLanguage, createFullTranslation } from './utils.js';
+import { type TranslationOptions, baseLanguage, syncTranslation } from './utils.js';
 
 const sync: CommandModule<{ path?: string }, { path?: string }> = {
   command: ['sync'],
@@ -25,10 +25,10 @@ const sync: CommandModule<{ path?: string }, { path?: string }> = {
         instancePath,
         verbose: false,
         queue,
-      } satisfies Partial<CreateFullTranslation>;
+      } satisfies Partial<TranslationOptions>;
 
       if (isPhrasesBuiltInLanguageTag(languageTag)) {
-        void createFullTranslation({
+        void syncTranslation({
           ...baseOptions,
           packageName: 'phrases',
           languageTag,
@@ -36,7 +36,7 @@ const sync: CommandModule<{ path?: string }, { path?: string }> = {
       }
 
       if (isPhrasesUiBuiltInLanguageTag(languageTag)) {
-        void createFullTranslation({
+        void syncTranslation({
           ...baseOptions,
           packageName: 'phrases-ui',
           languageTag,

packages/cli/src/commands/translate/utils.ts

@@ -4,6 +4,7 @@ import path from 'node:path';
 
 import { type LanguageTag } from '@logto/language-kit';
 import { conditionalString } from '@silverhand/essentials';
+import { type Got } from 'got';
 import PQueue from 'p-queue';
 
 import { consoleLog } from '../../utils.js';
@@ -12,6 +13,8 @@ import { createOpenaiApi, translate } from './openai.js';
 
 export const baseLanguage = 'en' satisfies LanguageTag;
 
+const untranslatedMark = '// UNTRANSLATED';
+
 export const readLocaleFiles = async (directory: string): Promise<string[]> => {
   const entities = await fs.readdir(directory, { withFileTypes: true });
 
@@ -39,7 +42,7 @@ export const readBaseLocaleFiles = async (directory: string): Promise<string[]>
   return readLocaleFiles(enDirectory);
 };
 
-export type CreateFullTranslation = {
+export type TranslationOptions = {
   instancePath: string;
   packageName: 'phrases' | 'phrases-ui';
   languageTag: LanguageTag;
@@ -53,16 +56,59 @@ export const createFullTranslation = async ({
   languageTag,
   verbose = true,
   queue = new PQueue({ concurrency: 5 }),
-}: CreateFullTranslation) => {
-  const directory = path.join(instancePath, 'packages', packageName, 'src/locales');
-  const files = await readBaseLocaleFiles(directory);
+}: TranslationOptions) => {
+  const localeFiles = await getBaseAndTargetLocaleFiles(instancePath, packageName, languageTag);
 
   if (verbose) {
     consoleLog.info(
       'Found ' +
-        String(files.length) +
+        String(localeFiles.length) +
         ' file' +
-        conditionalString(files.length !== 1 && 's') +
+        conditionalString(localeFiles.length !== 1 && 's') +
+        ' in ' +
+        packageName +
+        ' to create'
+    );
+  }
+
+  const openai = createOpenaiApi();
+
+  for (const { baseLocaleFile, targetLocaleFile } of localeFiles) {
+    if (existsSync(targetLocaleFile)) {
+      if (verbose) {
+        consoleLog.info(`Target locale file ${targetLocaleFile} exists, skipping`);
+      }
+
+      continue;
+    }
+
+    void createLocaleFile({
+      api: openai,
+      baseLocaleFile,
+      targetPath: targetLocaleFile,
+      targetLanguage: languageTag,
+      queue,
+    });
+  }
+
+  return queue.onIdle();
+};
+
+export const syncTranslation = async ({
+  instancePath,
+  packageName,
+  languageTag,
+  verbose = true,
+  queue = new PQueue({ concurrency: 5 }),
+}: TranslationOptions) => {
+  const localeFiles = await getBaseAndTargetLocaleFiles(instancePath, packageName, languageTag);
+
+  if (verbose) {
+    consoleLog.info(
+      'Found ' +
+        String(localeFiles.length) +
+        ' file' +
+        conditionalString(localeFiles.length !== 1 && 's') +
         ' in ' +
         packageName +
         ' to translate'
@@ -72,50 +118,102 @@ export const createFullTranslation = async ({
   const openai = createOpenaiApi();
 
   /* eslint-disable no-await-in-loop */
-  for (const file of files) {
-    const basePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file);
-    const targetPath = path.join(directory, languageTag.toLowerCase(), basePath);
+  for (const { baseLocaleFile, targetLocaleFile } of localeFiles) {
+    if (!existsSync(targetLocaleFile)) {
+      void createLocaleFile({
+        api: openai,
+        baseLocaleFile,
+        targetPath: targetLocaleFile,
+        targetLanguage: languageTag,
+        queue,
+      });
 
-    const getTranslationPath = async () => {
-      if (existsSync(targetPath)) {
-        const currentContent = await fs.readFile(targetPath, 'utf8');
+      continue;
+    }
 
-        if (currentContent.includes('// UNTRANSLATED')) {
-          return targetPath;
-        }
+    const currentContent = await fs.readFile(targetLocaleFile, 'utf8');
 
-        if (verbose) {
-          consoleLog.info(
-            `Target path ${targetPath} exists and has no untranslated mark, skipping`
-          );
-        }
-
-        return;
+    if (!currentContent.includes(untranslatedMark)) {
+      if (verbose) {
+        consoleLog.info(
+          `Target path ${targetLocaleFile} exists and has no untranslated mark, skipping`
+        );
       }
-
-      return file;
-    };
-
-    const translationPath = await getTranslationPath();
-
-    if (!translationPath) {
       continue;
     }
 
     void queue.add(async () => {
-      consoleLog.info(`Translating ${translationPath}`);
-      const result = await translate(openai, languageTag, translationPath);
+      consoleLog.info(`Translating ${targetLocaleFile}`);
+      const result = await translate({
+        api: openai,
+        sourceFilePath: targetLocaleFile,
+        targetLanguage: languageTag,
+        extraPrompt: `Object values without an "${untranslatedMark}" mark should be skipped and keep its original value. Remember to remove the "${untranslatedMark}" mark with the spaces before and after it in the output content.`,
+      });
 
       if (!result) {
-        consoleLog.fatal(`Unable to translate ${translationPath}`);
+        consoleLog.fatal(`Unable to translate ${targetLocaleFile}`);
       }
 
-      await fs.mkdir(path.parse(targetPath).dir, { recursive: true });
-      await fs.writeFile(targetPath, result);
-      consoleLog.succeed(`Translated ${targetPath}`);
+      await fs.unlink(targetLocaleFile);
+      await fs.writeFile(targetLocaleFile, result);
+      consoleLog.succeed(`Translated ${targetLocaleFile}`);
     });
   }
   /* eslint-enable no-await-in-loop */
 
   return queue.onIdle();
 };
+
+const getBaseAndTargetLocaleFiles = async (
+  instancePath: string,
+  packageName: string,
+  languageTag: LanguageTag
+) => {
+  const directory = path.join(instancePath, 'packages', packageName, 'src/locales');
+  const baseLocaleFiles = await readBaseLocaleFiles(directory);
+
+  return baseLocaleFiles.map((baseLocaleFile) => {
+    const basePath = path.relative(
+      path.join(directory, baseLanguage.toLowerCase()),
+      baseLocaleFile
+    );
+
+    return {
+      baseLocaleFile,
+      targetLocaleFile: path.join(directory, languageTag.toLowerCase(), basePath),
+    };
+  });
+};
+
+type OperateLocaleFileOptions = {
+  api: Got;
+  baseLocaleFile: string;
+  targetPath: string;
+  targetLanguage: LanguageTag;
+  queue: PQueue;
+};
+
+const createLocaleFile = async ({
+  api,
+  baseLocaleFile,
+  targetPath,
+  targetLanguage,
+  queue,
+}: OperateLocaleFileOptions) =>
+  queue.add(async () => {
+    consoleLog.info(`Creating the translation for ${targetPath}`);
+    const result = await translate({
+      api,
+      sourceFilePath: baseLocaleFile,
+      targetLanguage,
+    });
+
+    if (!result) {
+      consoleLog.fatal(`Unable to create the translation for ${targetPath}`);
+    }
+
+    await fs.mkdir(path.parse(targetPath).dir, { recursive: true });
+    await fs.writeFile(targetPath, result);
+    consoleLog.succeed(`The translation for ${targetPath} created`);
+  });

packages/phrases/src/locales/fr/translation/admin-console/components.ts

@@ -9,7 +9,7 @@ const components = {
       'La taille du fichier est trop grande. Veuillez télécharger un fichier de moins de {{size, number}}Ko.',
     error_file_type:
       "Le type de fichier n'est pas pris en charge. Uniquement {{extensions, list(style: narrow; type: conjunction;)}}.",
-    error_file_count: 'You can only upload 1 file.', // UNTRANSLATED
+    error_file_count: 'Vous ne pouvez télécharger qu’un seul fichier.',
   },
 };
 

packages/phrases/src/locales/ja/translation/admin-console/components.ts

@@ -8,7 +8,7 @@ const components = {
     error_file_size: '{{size, number}}KB以下のファイルをアップロードしてください。',
     error_file_type:
       '{{extensions, list(style: narrow; type: conjunction;)}}のみサポートされます。',
-    error_file_count: 'You can only upload 1 file.', // UNTRANSLATED
+    error_file_count: '1つのファイルしかアップロードできません。',
   },
 };
 

packages/phrases/src/locales/ko/translation/admin-console/components.ts

@@ -8,7 +8,7 @@ const components = {
     error_file_size: '파일 크기가 너무 커요. {{size, number}}KB 미만의 파일을 업로드해 주세요.',
     error_file_type:
       '지원되지 않는 파일 유형이에요. {{extensions, list(style: narrow; type: conjunction;)}} 파일만 사용 가능해요.',
-    error_file_count: 'You can only upload 1 file.', // UNTRANSLATED
+    error_file_count: '파일은 1개만 업로드 가능합니다.',
   },
 };
 

packages/phrases/src/locales/ru/translation/admin-console/components.ts

@@ -9,7 +9,7 @@ const components = {
       'Размер файла слишком большой. Пожалуйста, загрузите файл размером менее {{size, number}} КБ.',
     error_file_type:
       'Тип файла не поддерживается. Допустимы только файлы типа {{extensions, list(style: narrow; type: conjunction;)}}.',
-    error_file_count: 'You can only upload 1 file.', // UNTRANSLATED
+    error_file_count: 'Вы можете загрузить только 1 файл.',
   },
 };