From 4ec5e0550f99862d38b936dd6b1f6c0c066af2a0 Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Tue, 28 Mar 2023 21:45:45 +0800 Subject: [PATCH 1/3] feat(cli): create translation command iterate phrases files and translate to a new language using OpenAI API --- packages/cli/package.json | 3 + packages/cli/src/commands/translate/create.ts | 94 ++++++++++++++++++ packages/cli/src/commands/translate/index.ts | 23 +++++ .../cli/src/commands/translate/list-tags.ts | 24 +++++ packages/cli/src/commands/translate/openai.ts | 95 +++++++++++++++++++ packages/cli/src/commands/translate/utils.ts | 35 +++++++ packages/cli/src/index.ts | 2 + packages/cli/src/utils.ts | 9 +- pnpm-lock.yaml | 6 ++ 9 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/commands/translate/create.ts create mode 100644 packages/cli/src/commands/translate/index.ts create mode 100644 packages/cli/src/commands/translate/list-tags.ts create mode 100644 packages/cli/src/commands/translate/openai.ts create mode 100644 packages/cli/src/commands/translate/utils.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 0795db1eb..0ee9cf404 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -45,6 +45,9 @@ "dependencies": { "@logto/connector-kit": "workspace:^", "@logto/core-kit": "workspace:^", + "@logto/language-kit": "workspace:^", + "@logto/phrases": "workspace:^", + "@logto/phrases-ui": "workspace:^", "@logto/schemas": "workspace:^", "@logto/shared": "workspace:^", "@silverhand/essentials": "^2.5.0", diff --git a/packages/cli/src/commands/translate/create.ts b/packages/cli/src/commands/translate/create.ts new file mode 100644 index 000000000..c3c8cef1a --- /dev/null +++ b/packages/cli/src/commands/translate/create.ts @@ -0,0 +1,94 @@ +import { existsSync } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { isLanguageTag, type LanguageTag } from '@logto/language-kit'; +import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases'; +import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui'; +import { conditionalString, type Optional } from '@silverhand/essentials'; +import type { CommandModule } from 'yargs'; + +import { log } from '../../utils.js'; +import { inquireInstancePath } from '../connector/utils.js'; + +import { createOpenaiApi, translate } from './openai.js'; +import { baseLanguage, readBaseLocaleFiles } from './utils.js'; + +const createFullTranslation = async ( + instancePath: Optional, + packageName: 'phrases' | 'phrases-ui', + languageTag: LanguageTag +) => { + const directory = path.join( + await inquireInstancePath(instancePath), + 'packages', + packageName, + 'src/locales' + ); + const files = await readBaseLocaleFiles(directory); + + log.info( + 'Found ' + + String(files.length) + + ' file' + + conditionalString(files.length !== 1 && 's') + + ' in ' + + packageName + + ' to translate' + ); + + const openai = createOpenaiApi(); + + /* eslint-disable no-await-in-loop */ + for (const file of files) { + const relativePath = path.relative(path.join(directory, baseLanguage), file); + const targetPath = path.join(directory, languageTag, relativePath); + + if (existsSync(targetPath)) { + log.info(`Target path ${targetPath} exists, skipping`); + continue; + } + + log.info(`Translating ${file}`); + const result = await translate(openai, languageTag, file); + + if (!result) { + log.error(`Unable to translate ${file}`); + } + + await fs.mkdir(path.parse(targetPath).dir, { recursive: true }); + await fs.writeFile(targetPath, result); + log.succeed(`Translated ${targetPath}`); + } + /* eslint-enable no-await-in-loop */ +}; + +const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': string }> = { + command: ['create ', 'c'], + describe: 'Create a new language translation', + builder: (yargs) => + yargs.positional('language-tag', { + describe: 'The language tag to create, e.g. `af-ZA`.', + type: 'string', + demandOption: true, + }), + handler: async ({ path: inputPath, languageTag }) => { + if (!isLanguageTag(languageTag)) { + log.error('Invalid language tag. Run `logto translate list-tags` to see available list.'); + } + + if (isPhrasesBuiltInLanguageTag(languageTag)) { + log.info(languageTag + ' is a built-in tag of phrases, skipping'); + } else { + await createFullTranslation(inputPath, 'phrases', languageTag); + } + + if (isPhrasesUiBuiltInLanguageTag(languageTag)) { + log.info(languageTag + ' is a built-in tag of phrases-ui, skipping'); + } else { + await createFullTranslation(inputPath, 'phrases-ui', languageTag); + } + }, +}; + +export default create; diff --git a/packages/cli/src/commands/translate/index.ts b/packages/cli/src/commands/translate/index.ts new file mode 100644 index 000000000..1a9bfee70 --- /dev/null +++ b/packages/cli/src/commands/translate/index.ts @@ -0,0 +1,23 @@ +import { noop } from '@silverhand/essentials'; +import type { CommandModule } from 'yargs'; + +import create from './create.js'; +import listTags from './list-tags.js'; + +const translate: CommandModule = { + command: ['translate', 't'], + describe: 'Commands for Logto translation', + builder: (yargs) => + yargs + .option('path', { + alias: 'p', + type: 'string', + describe: 'The path to your Logto instance directory', + }) + .command(create) + .command(listTags) + .demandCommand(1), + handler: noop, +}; + +export default translate; diff --git a/packages/cli/src/commands/translate/list-tags.ts b/packages/cli/src/commands/translate/list-tags.ts new file mode 100644 index 000000000..5fab85ea1 --- /dev/null +++ b/packages/cli/src/commands/translate/list-tags.ts @@ -0,0 +1,24 @@ +import { languages } from '@logto/language-kit'; +import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phrases'; +import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-ui'; +import chalk from 'chalk'; +import type { CommandModule } from 'yargs'; + +const listTags: CommandModule> = { + command: ['list-tags', 'list'], + describe: 'List all available language tags', + + handler: async () => { + for (const tag of Object.keys(languages)) { + console.log( + ...[ + tag, + isPhrasesBuiltInLanguageTag(tag) && chalk.blue('phrases'), + isPhrasesUiBuiltInLanguageTag(tag) && chalk.blue('phrases-ui'), + ].filter(Boolean) + ); + } + }, +}; + +export default listTags; diff --git a/packages/cli/src/commands/translate/openai.ts b/packages/cli/src/commands/translate/openai.ts new file mode 100644 index 000000000..2901a697b --- /dev/null +++ b/packages/cli/src/commands/translate/openai.ts @@ -0,0 +1,95 @@ +import fs from 'node:fs/promises'; + +import { type LanguageTag } from '@logto/language-kit'; +import { trySafe } from '@silverhand/essentials'; +import { type Got, got, HTTPError } from 'got'; +import { HttpsProxyAgent } from 'hpagent'; +import { z } from 'zod'; + +import { getProxy, log } from '../../utils.js'; + +export const createOpenaiApi = () => { + const proxy = getProxy(); + + return got.extend({ + prefixUrl: 'https://api.openai.com/v1', + headers: { + Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`, + }, + timeout: { request: 120_000 }, + ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy, timeout: 120_000 }) } }), + }); +}; + +const gptResponseGuard = z.object({ + choices: z + .object({ + message: z.object({ role: z.string(), content: z.string() }), + finish_reason: z.string(), + }) + .array(), +}); + +export const translate = async (api: Got, languageTag: LanguageTag, filePath: string) => { + const fileContent = await fs.readFile(filePath, 'utf8'); + const response = await trySafe( + api + .post('chat/completions', { + json: { + model: 'gpt-3.5-turbo', + messages: [ + { + role: 'user', + content: `Translate the following code snippet to ${languageTag}, output ts code only: \n \`\`\`ts\n${fileContent}\n\`\`\``, + }, + ], + }, + }) + .json(), + (error) => { + log.warn(`Error while translating ${filePath}:`, String(error)); + + if (error instanceof HTTPError) { + log.warn(error.response.body); + } + } + ); + + if (!response) { + return; + } + + const guarded = gptResponseGuard.safeParse(response); + + if (!guarded.success) { + log.warn(`Error while guarding response for ${filePath}:`, response); + + return; + } + + const [entity] = guarded.data.choices; + + if (!entity) { + log.warn(`No choice found in response when translating ${filePath}`); + + return; + } + + if (entity.finish_reason !== 'stop') { + log.warn(`Unexpected finish reason ${entity.finish_reason} for ${filePath}`); + } + + const { content } = entity.message; + const matched = /```ts\n(.*)```/s.exec(content)?.[1]; + + if (!matched) { + // Treat as pure code + if (['const ', 'import '].some((prefix) => content.startsWith(prefix))) { + return content; + } + + log.warn('No matching code snippet from response:', content); + } + + return matched; +}; diff --git a/packages/cli/src/commands/translate/utils.ts b/packages/cli/src/commands/translate/utils.ts new file mode 100644 index 000000000..13f7e76e0 --- /dev/null +++ b/packages/cli/src/commands/translate/utils.ts @@ -0,0 +1,35 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { type LanguageTag } from '@logto/language-kit'; + +import { log } from '../../utils.js'; + +export const baseLanguage = 'en' satisfies LanguageTag; + +export const readLocaleFiles = async (directory: string): Promise => { + const entities = await fs.readdir(directory, { withFileTypes: true }); + + const result = await Promise.all( + entities.map(async (entity) => { + if (entity.isDirectory()) { + return readLocaleFiles(path.join(directory, entity.name)); + } + + return entity.name.endsWith('.ts') ? path.join(directory, entity.name) : []; + }) + ); + + return result.flat(); +}; + +export const readBaseLocaleFiles = async (directory: string): Promise => { + const enDirectory = path.join(directory, baseLanguage); + const stat = await fs.stat(enDirectory); + + if (!stat.isDirectory()) { + log.error(directory, 'has no `' + baseLanguage + '` directory'); + } + + return readLocaleFiles(enDirectory); +}; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 4b17f1a16..547cca81f 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,6 +6,7 @@ import { hideBin } from 'yargs/helpers'; import connector from './commands/connector/index.js'; import database from './commands/database/index.js'; import install from './commands/install/index.js'; +import translate from './commands/translate/index.js'; import { packageJson } from './package-json.js'; import { cliConfig, ConfigKey } from './utils.js'; @@ -46,6 +47,7 @@ void yargs(hideBin(process.argv)) .command(install) .command(database) .command(connector) + .command(translate) .demandCommand(1) .showHelpOnFail(false, `Specify ${chalk.green('--help')} for available options`) .strict() diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 16e790792..b8f5ea7e3 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -45,10 +45,15 @@ export const log: Log = Object.freeze({ }, }); -export const downloadFile = async (url: string, destination: string) => { +export const getProxy = () => { const { HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy } = process.env; + + return HTTPS_PROXY ?? https_proxy ?? HTTP_PROXY ?? http_proxy; +}; + +export const downloadFile = async (url: string, destination: string) => { const file = createWriteStream(destination); - const proxy = HTTPS_PROXY ?? https_proxy ?? HTTP_PROXY ?? http_proxy; + const proxy = getProxy(); const stream = got.stream(url, { ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy }) } }), }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f44e16db6..344032fd6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: specifiers: '@logto/connector-kit': workspace:^ '@logto/core-kit': workspace:^ + '@logto/language-kit': workspace:^1.0.0 + '@logto/phrases': workspace:^ + '@logto/phrases-ui': workspace:^ '@logto/schemas': workspace:^ '@logto/shared': workspace:^ '@silverhand/eslint-config': 3.0.0 @@ -69,6 +72,9 @@ importers: dependencies: '@logto/connector-kit': link:../toolkit/connector-kit '@logto/core-kit': link:../toolkit/core-kit + '@logto/language-kit': link:../toolkit/language-kit + '@logto/phrases': link:../phrases + '@logto/phrases-ui': link:../phrases-ui '@logto/schemas': link:../schemas '@logto/shared': link:../shared '@silverhand/essentials': 2.5.0 From e2ec1f93e4962c7f8233519690d0334be5ea4f04 Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Tue, 28 Mar 2023 21:48:54 +0800 Subject: [PATCH 2/3] chore: add changeset --- .changeset/thick-pumpkins-count.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thick-pumpkins-count.md diff --git a/.changeset/thick-pumpkins-count.md b/.changeset/thick-pumpkins-count.md new file mode 100644 index 000000000..5dab2fccf --- /dev/null +++ b/.changeset/thick-pumpkins-count.md @@ -0,0 +1,5 @@ +--- +"@logto/cli": minor +--- + +add `translate create` and `translate list-tags` commands From 5e67a785c6faa2eb47ed9ca9bbe41f6d86ec3ce9 Mon Sep 17 00:00:00 2001 From: Gao Sun Date: Wed, 29 Mar 2023 00:20:41 +0800 Subject: [PATCH 3/3] chore: fix lockfile --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 344032fd6..59be9880c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,7 +29,7 @@ importers: specifiers: '@logto/connector-kit': workspace:^ '@logto/core-kit': workspace:^ - '@logto/language-kit': workspace:^1.0.0 + '@logto/language-kit': workspace:^ '@logto/phrases': workspace:^ '@logto/phrases-ui': workspace:^ '@logto/schemas': workspace:^