From 9cedac95cb416892063036561a3aa663aae21168 Mon Sep 17 00:00:00 2001 From: Xiao Yijun Date: Fri, 14 Apr 2023 14:00:39 +0800 Subject: [PATCH] refactor(cli): improve translation cli (#3695) --- packages/cli/src/commands/translate/create.ts | 6 +- packages/cli/src/commands/translate/openai.ts | 34 +++- .../cli/src/commands/translate/prompts.ts | 52 ++++++ packages/cli/src/commands/translate/sync.ts | 8 +- packages/cli/src/commands/translate/utils.ts | 168 ++++++++++++++---- .../translation/admin-console/components.ts | 2 +- .../translation/admin-console/components.ts | 2 +- .../translation/admin-console/components.ts | 2 +- .../translation/admin-console/components.ts | 2 +- 9 files changed, 224 insertions(+), 52 deletions(-) create mode 100644 packages/cli/src/commands/translate/prompts.ts diff --git a/packages/cli/src/commands/translate/create.ts b/packages/cli/src/commands/translate/create.ts index 28840f6b5..968480ff6 100644 --- a/packages/cli/src/commands/translate/create.ts +++ b/packages/cli/src/commands/translate/create.ts @@ -30,7 +30,11 @@ const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': if (isPhrasesBuiltInLanguageTag(languageTag)) { consoleLog.info(languageTag + ' is a built-in tag of phrases, updating untranslated phrases'); } - await createFullTranslation({ instancePath, packageName: 'phrases', languageTag }); + await createFullTranslation({ + instancePath, + packageName: 'phrases', + languageTag, + }); if (isPhrasesUiBuiltInLanguageTag(languageTag)) { consoleLog.info( diff --git a/packages/cli/src/commands/translate/openai.ts b/packages/cli/src/commands/translate/openai.ts index a40811b0a..b424892e6 100644 --- a/packages/cli/src/commands/translate/openai.ts +++ b/packages/cli/src/commands/translate/openai.ts @@ -1,6 +1,6 @@ import fs from 'node:fs/promises'; -import { languages, type LanguageTag } from '@logto/language-kit'; +import { type LanguageTag } from '@logto/language-kit'; import { trySafe } from '@silverhand/essentials'; import { type Got, got, HTTPError } from 'got'; import { HttpsProxyAgent } from 'hpagent'; @@ -8,6 +8,8 @@ import { z } from 'zod'; import { consoleLog, getProxy } from '../../utils.js'; +import { getTranslationPrompt } from './prompts.js'; + export const createOpenaiApi = () => { const proxy = getProxy(); @@ -30,8 +32,20 @@ const gptResponseGuard = z.object({ .array(), }); -export const translate = async (api: Got, languageTag: LanguageTag, filePath: string) => { - const fileContent = await fs.readFile(filePath, 'utf8'); +type TranslateConfig = { + api: Got; + sourceFilePath: string; + targetLanguage: LanguageTag; + extraPrompt?: string; +}; + +export const translate = async ({ + api, + targetLanguage, + sourceFilePath, + extraPrompt, +}: TranslateConfig) => { + const sourceFileContent = await fs.readFile(sourceFilePath, 'utf8'); const response = await trySafe( api .post('chat/completions', { @@ -40,14 +54,18 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st messages: [ { role: 'user', - content: `Given the following code snippet, only translate object values to ${languages[languageTag]}, keep all object keys original, output ts code only: \n \`\`\`ts\n${fileContent}\n\`\`\``, + content: getTranslationPrompt({ + sourceFileContent, + targetLanguage, + extraPrompt, + }), }, ], }, }) .json(), (error) => { - consoleLog.warn(`Error while translating ${filePath}:`, String(error)); + consoleLog.warn(`Error while translating ${sourceFilePath}:`, String(error)); if (error instanceof HTTPError) { consoleLog.warn(error.response.body); @@ -62,7 +80,7 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st const guarded = gptResponseGuard.safeParse(response); if (!guarded.success) { - consoleLog.warn(`Error while guarding response for ${filePath}:`, response); + consoleLog.warn(`Error while guarding response for ${sourceFilePath}:`, response); return; } @@ -70,13 +88,13 @@ export const translate = async (api: Got, languageTag: LanguageTag, filePath: st const [entity] = guarded.data.choices; if (!entity) { - consoleLog.warn(`No choice found in response when translating ${filePath}`); + consoleLog.warn(`No choice found in response when translating ${sourceFilePath}`); return; } if (entity.finish_reason !== 'stop') { - consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${filePath}`); + consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${sourceFilePath}`); } const { content } = entity.message; diff --git a/packages/cli/src/commands/translate/prompts.ts b/packages/cli/src/commands/translate/prompts.ts new file mode 100644 index 000000000..dc95c9047 --- /dev/null +++ b/packages/cli/src/commands/translate/prompts.ts @@ -0,0 +1,52 @@ +import { languages, type LanguageTag } from '@logto/language-kit'; +import { conditionalString } from '@silverhand/essentials'; + +type GetTranslationPromptProperties = { + sourceFileContent: string; + targetLanguage: LanguageTag; + extraPrompt?: string; +}; + +/** + * Note: + * The input token limit of GPT 3.5 is 2048, the following prompt tokens with sourceFileContent is about 1200. + * Remember to check the token limit before adding more prompt. + * Tokens can be counted in https://platform.openai.com/tokenizer + */ +export const getTranslationPrompt = ({ + sourceFileContent, + targetLanguage, + extraPrompt, +}: GetTranslationPromptProperties) => `Given the following code snippet: +\`\`\`ts +${sourceFileContent} +\`\`\` +only translate object values to ${ + languages[targetLanguage] +}, keep all object keys original, output ts code only, the code format should be strictly consistent, and should not contains the given code snippet. + +Take zh-cn as an example, if the input is: +\`\`\`ts +import others from './others.js'; + +const translation = { + hello: '你好', + world: 'world', // UNTRANSLATED + others, +}; + +export default translation; +\`\`\` +the output should be: +\`\`\`ts +import others from './others.js'; + +const translation = { + hello: '你好', + world: '世界', + others, +}; + +export default translation; +\`\`\` +${conditionalString(extraPrompt)}`; diff --git a/packages/cli/src/commands/translate/sync.ts b/packages/cli/src/commands/translate/sync.ts index 2d7cce681..fcd3c7d60 100644 --- a/packages/cli/src/commands/translate/sync.ts +++ b/packages/cli/src/commands/translate/sync.ts @@ -6,7 +6,7 @@ import type { CommandModule } from 'yargs'; import { inquireInstancePath } from '../connector/utils.js'; -import { type CreateFullTranslation, baseLanguage, createFullTranslation } from './utils.js'; +import { type TranslationOptions, baseLanguage, syncTranslation } from './utils.js'; const sync: CommandModule<{ path?: string }, { path?: string }> = { command: ['sync'], @@ -25,10 +25,10 @@ const sync: CommandModule<{ path?: string }, { path?: string }> = { instancePath, verbose: false, queue, - } satisfies Partial; + } satisfies Partial; if (isPhrasesBuiltInLanguageTag(languageTag)) { - void createFullTranslation({ + void syncTranslation({ ...baseOptions, packageName: 'phrases', languageTag, @@ -36,7 +36,7 @@ const sync: CommandModule<{ path?: string }, { path?: string }> = { } if (isPhrasesUiBuiltInLanguageTag(languageTag)) { - void createFullTranslation({ + void syncTranslation({ ...baseOptions, packageName: 'phrases-ui', languageTag, diff --git a/packages/cli/src/commands/translate/utils.ts b/packages/cli/src/commands/translate/utils.ts index 6a0619903..147733bb0 100644 --- a/packages/cli/src/commands/translate/utils.ts +++ b/packages/cli/src/commands/translate/utils.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { type LanguageTag } from '@logto/language-kit'; import { conditionalString } from '@silverhand/essentials'; +import { type Got } from 'got'; import PQueue from 'p-queue'; import { consoleLog } from '../../utils.js'; @@ -12,6 +13,8 @@ import { createOpenaiApi, translate } from './openai.js'; export const baseLanguage = 'en' satisfies LanguageTag; +const untranslatedMark = '// UNTRANSLATED'; + export const readLocaleFiles = async (directory: string): Promise => { const entities = await fs.readdir(directory, { withFileTypes: true }); @@ -39,7 +42,7 @@ export const readBaseLocaleFiles = async (directory: string): Promise return readLocaleFiles(enDirectory); }; -export type CreateFullTranslation = { +export type TranslationOptions = { instancePath: string; packageName: 'phrases' | 'phrases-ui'; languageTag: LanguageTag; @@ -53,16 +56,59 @@ export const createFullTranslation = async ({ languageTag, verbose = true, queue = new PQueue({ concurrency: 5 }), -}: CreateFullTranslation) => { - const directory = path.join(instancePath, 'packages', packageName, 'src/locales'); - const files = await readBaseLocaleFiles(directory); +}: TranslationOptions) => { + const localeFiles = await getBaseAndTargetLocaleFiles(instancePath, packageName, languageTag); if (verbose) { consoleLog.info( 'Found ' + - String(files.length) + + String(localeFiles.length) + ' file' + - conditionalString(files.length !== 1 && 's') + + conditionalString(localeFiles.length !== 1 && 's') + + ' in ' + + packageName + + ' to create' + ); + } + + const openai = createOpenaiApi(); + + for (const { baseLocaleFile, targetLocaleFile } of localeFiles) { + if (existsSync(targetLocaleFile)) { + if (verbose) { + consoleLog.info(`Target locale file ${targetLocaleFile} exists, skipping`); + } + + continue; + } + + void createLocaleFile({ + api: openai, + baseLocaleFile, + targetPath: targetLocaleFile, + targetLanguage: languageTag, + queue, + }); + } + + return queue.onIdle(); +}; + +export const syncTranslation = async ({ + instancePath, + packageName, + languageTag, + verbose = true, + queue = new PQueue({ concurrency: 5 }), +}: TranslationOptions) => { + const localeFiles = await getBaseAndTargetLocaleFiles(instancePath, packageName, languageTag); + + if (verbose) { + consoleLog.info( + 'Found ' + + String(localeFiles.length) + + ' file' + + conditionalString(localeFiles.length !== 1 && 's') + ' in ' + packageName + ' to translate' @@ -72,50 +118,102 @@ export const createFullTranslation = async ({ const openai = createOpenaiApi(); /* eslint-disable no-await-in-loop */ - for (const file of files) { - const basePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file); - const targetPath = path.join(directory, languageTag.toLowerCase(), basePath); + for (const { baseLocaleFile, targetLocaleFile } of localeFiles) { + if (!existsSync(targetLocaleFile)) { + void createLocaleFile({ + api: openai, + baseLocaleFile, + targetPath: targetLocaleFile, + targetLanguage: languageTag, + queue, + }); - const getTranslationPath = async () => { - if (existsSync(targetPath)) { - const currentContent = await fs.readFile(targetPath, 'utf8'); + continue; + } - if (currentContent.includes('// UNTRANSLATED')) { - return targetPath; - } + const currentContent = await fs.readFile(targetLocaleFile, 'utf8'); - if (verbose) { - consoleLog.info( - `Target path ${targetPath} exists and has no untranslated mark, skipping` - ); - } - - return; + if (!currentContent.includes(untranslatedMark)) { + if (verbose) { + consoleLog.info( + `Target path ${targetLocaleFile} exists and has no untranslated mark, skipping` + ); } - - return file; - }; - - const translationPath = await getTranslationPath(); - - if (!translationPath) { continue; } void queue.add(async () => { - consoleLog.info(`Translating ${translationPath}`); - const result = await translate(openai, languageTag, translationPath); + consoleLog.info(`Translating ${targetLocaleFile}`); + const result = await translate({ + api: openai, + sourceFilePath: targetLocaleFile, + targetLanguage: languageTag, + extraPrompt: `Object values without an "${untranslatedMark}" mark should be skipped and keep its original value. Remember to remove the "${untranslatedMark}" mark with the spaces before and after it in the output content.`, + }); if (!result) { - consoleLog.fatal(`Unable to translate ${translationPath}`); + consoleLog.fatal(`Unable to translate ${targetLocaleFile}`); } - await fs.mkdir(path.parse(targetPath).dir, { recursive: true }); - await fs.writeFile(targetPath, result); - consoleLog.succeed(`Translated ${targetPath}`); + await fs.unlink(targetLocaleFile); + await fs.writeFile(targetLocaleFile, result); + consoleLog.succeed(`Translated ${targetLocaleFile}`); }); } /* eslint-enable no-await-in-loop */ return queue.onIdle(); }; + +const getBaseAndTargetLocaleFiles = async ( + instancePath: string, + packageName: string, + languageTag: LanguageTag +) => { + const directory = path.join(instancePath, 'packages', packageName, 'src/locales'); + const baseLocaleFiles = await readBaseLocaleFiles(directory); + + return baseLocaleFiles.map((baseLocaleFile) => { + const basePath = path.relative( + path.join(directory, baseLanguage.toLowerCase()), + baseLocaleFile + ); + + return { + baseLocaleFile, + targetLocaleFile: path.join(directory, languageTag.toLowerCase(), basePath), + }; + }); +}; + +type OperateLocaleFileOptions = { + api: Got; + baseLocaleFile: string; + targetPath: string; + targetLanguage: LanguageTag; + queue: PQueue; +}; + +const createLocaleFile = async ({ + api, + baseLocaleFile, + targetPath, + targetLanguage, + queue, +}: OperateLocaleFileOptions) => + queue.add(async () => { + consoleLog.info(`Creating the translation for ${targetPath}`); + const result = await translate({ + api, + sourceFilePath: baseLocaleFile, + targetLanguage, + }); + + if (!result) { + consoleLog.fatal(`Unable to create the translation for ${targetPath}`); + } + + await fs.mkdir(path.parse(targetPath).dir, { recursive: true }); + await fs.writeFile(targetPath, result); + consoleLog.succeed(`The translation for ${targetPath} created`); + }); diff --git a/packages/phrases/src/locales/fr/translation/admin-console/components.ts b/packages/phrases/src/locales/fr/translation/admin-console/components.ts index 13c663fd0..11d109027 100644 --- a/packages/phrases/src/locales/fr/translation/admin-console/components.ts +++ b/packages/phrases/src/locales/fr/translation/admin-console/components.ts @@ -9,7 +9,7 @@ const components = { 'La taille du fichier est trop grande. Veuillez télécharger un fichier de moins de {{size, number}}Ko.', error_file_type: "Le type de fichier n'est pas pris en charge. Uniquement {{extensions, list(style: narrow; type: conjunction;)}}.", - error_file_count: 'You can only upload 1 file.', // UNTRANSLATED + error_file_count: 'Vous ne pouvez télécharger qu’un seul fichier.', }, }; diff --git a/packages/phrases/src/locales/ja/translation/admin-console/components.ts b/packages/phrases/src/locales/ja/translation/admin-console/components.ts index 3a1e2094c..cf3218679 100644 --- a/packages/phrases/src/locales/ja/translation/admin-console/components.ts +++ b/packages/phrases/src/locales/ja/translation/admin-console/components.ts @@ -8,7 +8,7 @@ const components = { error_file_size: '{{size, number}}KB以下のファイルをアップロードしてください。', error_file_type: '{{extensions, list(style: narrow; type: conjunction;)}}のみサポートされます。', - error_file_count: 'You can only upload 1 file.', // UNTRANSLATED + error_file_count: '1つのファイルしかアップロードできません。', }, }; diff --git a/packages/phrases/src/locales/ko/translation/admin-console/components.ts b/packages/phrases/src/locales/ko/translation/admin-console/components.ts index 927cdab8f..23db663aa 100644 --- a/packages/phrases/src/locales/ko/translation/admin-console/components.ts +++ b/packages/phrases/src/locales/ko/translation/admin-console/components.ts @@ -8,7 +8,7 @@ const components = { error_file_size: '파일 크기가 너무 커요. {{size, number}}KB 미만의 파일을 업로드해 주세요.', error_file_type: '지원되지 않는 파일 유형이에요. {{extensions, list(style: narrow; type: conjunction;)}} 파일만 사용 가능해요.', - error_file_count: 'You can only upload 1 file.', // UNTRANSLATED + error_file_count: '파일은 1개만 업로드 가능합니다.', }, }; diff --git a/packages/phrases/src/locales/ru/translation/admin-console/components.ts b/packages/phrases/src/locales/ru/translation/admin-console/components.ts index 66a438177..e59eb4aae 100644 --- a/packages/phrases/src/locales/ru/translation/admin-console/components.ts +++ b/packages/phrases/src/locales/ru/translation/admin-console/components.ts @@ -9,7 +9,7 @@ const components = { 'Размер файла слишком большой. Пожалуйста, загрузите файл размером менее {{size, number}} КБ.', error_file_type: 'Тип файла не поддерживается. Допустимы только файлы типа {{extensions, list(style: narrow; type: conjunction;)}}.', - error_file_count: 'You can only upload 1 file.', // UNTRANSLATED + error_file_count: 'Вы можете загрузить только 1 файл.', }, };