From 3f19aa259e1714bbf9edb0679cbde54e77e7cc3c Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Fri, 31 Mar 2023 23:21:03 +0800 Subject: [PATCH 1/2] feat(cli): translation sync command to create missing files or translate untranslated phrases across all existing languages --- packages/cli/package.json | 1 + packages/cli/src/commands/translate/create.ts | 86 +++---------------- packages/cli/src/commands/translate/index.ts | 2 + packages/cli/src/commands/translate/sync.ts | 50 +++++++++++ packages/cli/src/commands/translate/utils.ts | 84 ++++++++++++++++++ pnpm-lock.yaml | 16 ++++ 6 files changed, 163 insertions(+), 76 deletions(-) create mode 100644 packages/cli/src/commands/translate/sync.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index cb953d864..d160dd41d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -60,6 +60,7 @@ "nanoid": "^4.0.0", "ora": "^6.1.2", "p-limit": "^4.0.0", + "p-queue": "^7.3.4", "p-retry": "^5.1.2", "pg-protocol": "^1.6.0", "roarr": "^7.11.0", diff --git a/packages/cli/src/commands/translate/create.ts b/packages/cli/src/commands/translate/create.ts index 60cafd8c3..d02eedcbf 100644 --- a/packages/cli/src/commands/translate/create.ts +++ b/packages/cli/src/commands/translate/create.ts @@ -1,84 +1,12 @@ -import { existsSync } from 'node:fs'; -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { isLanguageTag, type LanguageTag } from '@logto/language-kit'; +import { isLanguageTag } from '@logto/language-kit'; import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases'; import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui'; -import { conditionalString, type Optional } from '@silverhand/essentials'; import type { CommandModule } from 'yargs'; import { log } from '../../utils.js'; import { inquireInstancePath } from '../connector/utils.js'; -import { createOpenaiApi, translate } from './openai.js'; -import { baseLanguage, readBaseLocaleFiles } from './utils.js'; - -const createFullTranslation = async ( - instancePath: Optional, - packageName: 'phrases' | 'phrases-ui', - languageTag: LanguageTag -) => { - const directory = path.join( - await inquireInstancePath(instancePath), - 'packages', - packageName, - 'src/locales' - ); - const files = await readBaseLocaleFiles(directory); - - log.info( - 'Found ' + - String(files.length) + - ' file' + - conditionalString(files.length !== 1 && 's') + - ' in ' + - packageName + - ' to translate' - ); - - const openai = createOpenaiApi(); - - /* eslint-disable no-await-in-loop */ - for (const file of files) { - const relativePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file); - const targetPath = path.join(directory, languageTag.toLowerCase(), relativePath); - - const getTranslationPath = async () => { - if (existsSync(targetPath)) { - const currentContent = await fs.readFile(targetPath, 'utf8'); - - if (currentContent.includes('// UNTRANSLATED')) { - return targetPath; - } - - log.info(`Target path ${targetPath} exists and has no untranslated mark, skipping`); - - return; - } - - return file; - }; - - const translationPath = await getTranslationPath(); - - if (!translationPath) { - continue; - } - - log.info(`Translating ${translationPath}`); - const result = await translate(openai, languageTag, translationPath); - - if (!result) { - log.error(`Unable to translate ${translationPath}`); - } - - await fs.mkdir(path.parse(targetPath).dir, { recursive: true }); - await fs.writeFile(targetPath, result); - log.succeed(`Translated ${targetPath}`); - } - /* eslint-enable no-await-in-loop */ -}; +import { createFullTranslation } from './utils.js'; const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': string }> = { command: ['create ', 'c'], @@ -94,15 +22,21 @@ const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': log.error('Invalid language tag. Run `logto translate list-tags` to see available list.'); } + const instancePath = await inquireInstancePath(inputPath); + if (isPhrasesBuiltInLanguageTag(languageTag)) { log.info(languageTag + ' is a built-in tag of phrases, updating untranslated phrases'); } - await createFullTranslation(inputPath, 'phrases', languageTag); + await createFullTranslation({ instancePath, packageName: 'phrases', languageTag }); if (isPhrasesUiBuiltInLanguageTag(languageTag)) { log.info(languageTag + ' is a built-in tag of phrases-ui, updating untranslated phrases'); } - await createFullTranslation(inputPath, 'phrases-ui', languageTag); + await createFullTranslation({ + instancePath, + packageName: 'phrases-ui', + languageTag, + }); }, }; diff --git a/packages/cli/src/commands/translate/index.ts b/packages/cli/src/commands/translate/index.ts index 1a9bfee70..d302ead0f 100644 --- a/packages/cli/src/commands/translate/index.ts +++ b/packages/cli/src/commands/translate/index.ts @@ -3,6 +3,7 @@ import type { CommandModule } from 'yargs'; import create from './create.js'; import listTags from './list-tags.js'; +import sync from './sync.js'; const translate: CommandModule = { command: ['translate', 't'], @@ -16,6 +17,7 @@ const translate: CommandModule = { }) .command(create) .command(listTags) + .command(sync) .demandCommand(1), handler: noop, }; diff --git a/packages/cli/src/commands/translate/sync.ts b/packages/cli/src/commands/translate/sync.ts new file mode 100644 index 000000000..928fe5300 --- /dev/null +++ b/packages/cli/src/commands/translate/sync.ts @@ -0,0 +1,50 @@ +import { languages } from '@logto/language-kit'; +import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases'; +import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui'; +import PQueue from 'p-queue'; +import type { CommandModule } from 'yargs'; + +import { inquireInstancePath } from '../connector/utils.js'; + +import { type CreateFullTranslation, baseLanguage, createFullTranslation } from './utils.js'; + +const sync: CommandModule<{ path?: string }, { path?: string }> = { + command: ['sync'], + describe: 'Translate all untranslated phrases.', + handler: async ({ path: inputPath }) => { + const queue = new PQueue({ concurrency: 5 }); + const instancePath = await inquireInstancePath(inputPath); + + for (const languageTag of Object.keys(languages)) { + if (languageTag === baseLanguage) { + continue; + } + + const baseOptions = { + instancePath, + verbose: false, + queue, + } satisfies Partial; + + if (isPhrasesBuiltInLanguageTag(languageTag)) { + void createFullTranslation({ + ...baseOptions, + packageName: 'phrases', + languageTag, + }); + } + + if (isPhrasesUiBuiltInLanguageTag(languageTag)) { + void createFullTranslation({ + ...baseOptions, + packageName: 'phrases-ui', + languageTag, + }); + } + } + + await queue.onIdle(); + }, +}; + +export default sync; diff --git a/packages/cli/src/commands/translate/utils.ts b/packages/cli/src/commands/translate/utils.ts index 7bc0eb831..e2ea17e7c 100644 --- a/packages/cli/src/commands/translate/utils.ts +++ b/packages/cli/src/commands/translate/utils.ts @@ -1,10 +1,15 @@ +import { existsSync } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; import { type LanguageTag } from '@logto/language-kit'; +import { conditionalString } from '@silverhand/essentials'; +import PQueue from 'p-queue'; import { log } from '../../utils.js'; +import { createOpenaiApi, translate } from './openai.js'; + export const baseLanguage = 'en' satisfies LanguageTag; export const readLocaleFiles = async (directory: string): Promise => { @@ -33,3 +38,82 @@ export const readBaseLocaleFiles = async (directory: string): Promise return readLocaleFiles(enDirectory); }; + +export type CreateFullTranslation = { + instancePath: string; + packageName: 'phrases' | 'phrases-ui'; + languageTag: LanguageTag; + verbose?: boolean; + queue?: PQueue; +}; + +export const createFullTranslation = async ({ + instancePath, + packageName, + languageTag, + verbose = true, + queue = new PQueue({ concurrency: 5 }), +}: CreateFullTranslation) => { + const directory = path.join(instancePath, 'packages', packageName, 'src/locales'); + const files = await readBaseLocaleFiles(directory); + + if (verbose) { + log.info( + 'Found ' + + String(files.length) + + ' file' + + conditionalString(files.length !== 1 && 's') + + ' in ' + + packageName + + ' to translate' + ); + } + + const openai = createOpenaiApi(); + + /* eslint-disable no-await-in-loop */ + for (const file of files) { + const basePath = path.relative(path.join(directory, baseLanguage.toLowerCase()), file); + const targetPath = path.join(directory, languageTag.toLowerCase(), basePath); + + const getTranslationPath = async () => { + if (existsSync(targetPath)) { + const currentContent = await fs.readFile(targetPath, 'utf8'); + + if (currentContent.includes('// UNTRANSLATED')) { + return targetPath; + } + + if (verbose) { + log.info(`Target path ${targetPath} exists and has no untranslated mark, skipping`); + } + + return; + } + + return file; + }; + + const translationPath = await getTranslationPath(); + + if (!translationPath) { + continue; + } + + void queue.add(async () => { + log.info(`Translating ${translationPath}`); + const result = await translate(openai, languageTag, translationPath); + + if (!result) { + log.error(`Unable to translate ${translationPath}`); + } + + await fs.mkdir(path.parse(targetPath).dir, { recursive: true }); + await fs.writeFile(targetPath, result); + log.succeed(`Translated ${targetPath}`); + }); + } + /* eslint-enable no-await-in-loop */ + + return queue.onIdle(); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 427a75b52..2503545d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -135,6 +135,9 @@ importers: p-limit: specifier: ^4.0.0 version: 4.0.0 + p-queue: + specifier: ^7.3.4 + version: 7.3.4 p-retry: specifier: ^5.1.2 version: 5.1.2 @@ -11876,6 +11879,14 @@ packages: type-fest: 3.5.2 dev: false + /p-queue@7.3.4: + resolution: {integrity: sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg==} + engines: {node: '>=12'} + dependencies: + eventemitter3: 4.0.7 + p-timeout: 5.1.0 + dev: false + /p-retry@5.1.2: resolution: {integrity: sha512-couX95waDu98NfNZV+i/iLt+fdVxmI7CbrrdC2uDWfPdUAApyxT4wmDlyOtR5KtTDmkDO0zDScDjDou9YHhd9g==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -11884,6 +11895,11 @@ packages: retry: 0.13.1 dev: false + /p-timeout@5.1.0: + resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} + engines: {node: '>=12'} + dev: false + /p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} From 736d6d212b1ba898d13f9ddf9f25e009735ae009 Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Fri, 31 Mar 2023 23:22:46 +0800 Subject: [PATCH 2/2] chore: add changeset --- .changeset/seven-dolphins-hammer.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/seven-dolphins-hammer.md diff --git a/.changeset/seven-dolphins-hammer.md b/.changeset/seven-dolphins-hammer.md new file mode 100644 index 000000000..a723a90f7 --- /dev/null +++ b/.changeset/seven-dolphins-hammer.md @@ -0,0 +1,5 @@ +--- +"@logto/cli": minor +--- + +`logto translate sync` to create missing files or translate untranslated phrases across all existing languages