From 0183d0c33a78cad81d240b14a14643be22867e39 Mon Sep 17 00:00:00 2001 From: Charles Zhao Date: Mon, 19 Aug 2024 12:16:10 +0800 Subject: [PATCH] refactor: split translate cmd from logto cli (#6451) * refactor: split translate cmd from logto cli * chore: add changeset * refactor(cli): remove translate command from cli package --- .changeset/silly-hotels-greet.md | 11 ++ commitlint.config.ts | 2 +- package.json | 4 +- packages/cli/package.json | 7 +- packages/cli/src/commands/translate/index.ts | 32 ---- packages/cli/src/commands/translate/openai.ts | 109 ----------- packages/cli/src/index.ts | 2 - packages/cli/src/utils.ts | 32 +--- packages/translate/.gitignore | 1 + packages/translate/bin/index.js | 3 + packages/translate/package.json | 80 ++++++++ packages/translate/src/constants.ts | 5 + .../translate => translate/src}/create.ts | 5 +- packages/translate/src/index.ts | 56 ++++++ .../translate => translate/src}/list-tags.ts | 2 +- .../utils.ts => translate/src/openai.ts} | 133 +++++++++---- .../translate => translate/src}/prompts.ts | 0 .../src}/sync-keys/index.ts | 2 +- .../src}/sync-keys/utils.ts | 2 +- .../translate => translate/src}/sync.ts | 10 +- packages/translate/src/utils.ts | 174 ++++++++++++++++++ packages/translate/tsconfig.build.json | 4 + packages/translate/tsconfig.json | 10 + pnpm-lock.yaml | 115 +++++++++--- 24 files changed, 553 insertions(+), 248 deletions(-) create mode 100644 .changeset/silly-hotels-greet.md delete mode 100644 packages/cli/src/commands/translate/index.ts delete mode 100644 packages/cli/src/commands/translate/openai.ts create mode 100644 packages/translate/.gitignore create mode 100755 packages/translate/bin/index.js create mode 100644 packages/translate/package.json create mode 100644 packages/translate/src/constants.ts rename packages/{cli/src/commands/translate => translate/src}/create.ts (93%) create mode 100644 packages/translate/src/index.ts rename packages/{cli/src/commands/translate => translate/src}/list-tags.ts (94%) rename packages/{cli/src/commands/translate/utils.ts => translate/src/openai.ts} (61%) rename packages/{cli/src/commands/translate => translate/src}/prompts.ts (100%) rename packages/{cli/src/commands/translate => translate/src}/sync-keys/index.ts (99%) rename packages/{cli/src/commands/translate => translate/src}/sync-keys/utils.ts (99%) rename packages/{cli/src/commands/translate => translate/src}/sync.ts (92%) create mode 100644 packages/translate/src/utils.ts create mode 100644 packages/translate/tsconfig.build.json create mode 100644 packages/translate/tsconfig.json diff --git a/.changeset/silly-hotels-greet.md b/.changeset/silly-hotels-greet.md new file mode 100644 index 000000000..ddb4f4c1b --- /dev/null +++ b/.changeset/silly-hotels-greet.md @@ -0,0 +1,11 @@ +--- +"@logto/cli": minor +"@logto/translate": minor +--- + +split translate command from @logto/cli and create a standalone package + +The "translate" command has greatly increased the size of the "@logto/cli" package, as it involves TypeScript code manipulation and has to use "typescrpt" as a "dependency". +In fact, only a small number of developers who want to contribute Logto will use this command, so we believe it's best to separate the less frequently used "translate" command from the "cli" package to keep it small and simple. + +Please also be noted that this change is actually a breaking change for those who use the "translate" command. However, the CLI has to be bundle-released with the "Logto" open-source distribution, and we feel it is still too early to make a major version bump for Logto. Therefore, the "minor" bump is used this time. diff --git a/commitlint.config.ts b/commitlint.config.ts index c89743462..f75d4df49 100644 --- a/commitlint.config.ts +++ b/commitlint.config.ts @@ -7,7 +7,7 @@ const config: UserConfig = { extends: ['@commitlint/config-conventional'], rules: { 'type-enum': [2, 'always', [...conventional.rules['type-enum'][2], 'api', 'release']], - 'scope-enum': [2, 'always', ['connector', 'console', 'core', 'demo-app', 'test', 'phrases', 'schemas', 'shared', 'experience', 'deps', 'deps-dev', 'cli', 'toolkit', 'cloud', 'app-insights', 'elements']], + 'scope-enum': [2, 'always', ['connector', 'console', 'core', 'demo-app', 'test', 'phrases', 'schemas', 'shared', 'experience', 'deps', 'deps-dev', 'cli', 'toolkit', 'cloud', 'app-insights', 'elements', 'translate']], // Slightly increase the tolerance to allow the appending PR number ...(isCi && { 'header-max-length': [2, 'always', 110] }), 'body-max-line-length': [2, 'always', 110], diff --git a/package.json b/package.json index 254f4b842..39609f1c8 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "start:dev": "pnpm -r --parallel --filter=!@logto/integration-tests --filter \"!./packages/connectors/connector-*\" dev", "start": "cd packages/core && NODE_ENV=production node .", "cli": "logto", + "translate": "logto-translate", "changeset": "changeset", "alteration": "logto db alt", "connectors": "pnpm -r --filter \"./packages/connectors/connector-*\"", @@ -52,6 +53,7 @@ } }, "dependencies": { - "@logto/cli": "workspace:^1.1.0" + "@logto/cli": "workspace:^1.1.0", + "@logto/translate": "workspace:^0.0.0" } } diff --git a/packages/cli/package.json b/packages/cli/package.json index 10c7ea85d..547904e64 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -44,9 +44,6 @@ "dependencies": { "@logto/connector-kit": "workspace:^4.0.0", "@logto/core-kit": "workspace:^2.5.0", - "@logto/language-kit": "workspace:^1.1.0", - "@logto/phrases": "workspace:^1.13.0", - "@logto/phrases-experience": "workspace:^1.7.0", "@logto/schemas": "workspace:1.19.0", "@logto/shared": "workspace:^3.1.1", "@silverhand/essentials": "^2.9.1", @@ -62,13 +59,10 @@ "nanoid": "^5.0.1", "ora": "^8.0.1", "p-limit": "^6.0.0", - "p-queue": "^8.0.0", "p-retry": "^6.0.0", "pg-protocol": "^1.6.0", - "roarr": "^7.11.0", "semver": "^7.3.8", "tar": "^7.0.0", - "typescript": "^5.5.3", "yargs": "^17.6.0", "zod": "^3.23.8" }, @@ -87,6 +81,7 @@ "lint-staged": "^15.0.0", "prettier": "^3.0.0", "sinon": "^18.0.0", + "typescript": "^5.5.3", "vitest": "^2.0.0" }, "eslintConfig": { diff --git a/packages/cli/src/commands/translate/index.ts b/packages/cli/src/commands/translate/index.ts deleted file mode 100644 index 5f0abbd8b..000000000 --- a/packages/cli/src/commands/translate/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { noop } from '@silverhand/essentials'; -import type { CommandModule } from 'yargs'; - -import create from './create.js'; -import listTags from './list-tags.js'; -import syncKeys from './sync-keys/index.js'; -import sync from './sync.js'; - -const translate: CommandModule = { - command: ['translate', 't'], - describe: 'Commands for Logto translation', - builder: (yargs) => - yargs - .option('path', { - alias: 'p', - type: 'string', - describe: 'The path to your Logto instance directory', - }) - .option('skip-core-check', { - alias: 'sc', - type: 'boolean', - describe: 'Skip checking if the core package is existed', - }) - .command(create) - .command(listTags) - .command(sync) - .command(syncKeys) - .demandCommand(1), - handler: noop, -}; - -export default translate; diff --git a/packages/cli/src/commands/translate/openai.ts b/packages/cli/src/commands/translate/openai.ts deleted file mode 100644 index 26e30efb2..000000000 --- a/packages/cli/src/commands/translate/openai.ts +++ /dev/null @@ -1,109 +0,0 @@ -import fs from 'node:fs/promises'; - -import { type LanguageTag } from '@logto/language-kit'; -import { trySafe } from '@silverhand/essentials'; -import { type Got, got, HTTPError } from 'got'; -import { HttpsProxyAgent } from 'hpagent'; -import { z } from 'zod'; - -import { consoleLog, getProxy } from '../../utils.js'; - -import { getTranslationPromptMessages } from './prompts.js'; - -export const createOpenaiApi = () => { - const proxy = getProxy(); - - return got.extend({ - prefixUrl: process.env.OPENAI_API_PROXY_ENDPOINT ?? 'https://api.openai.com/v1', - headers: { - Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`, - }, - timeout: { request: 300_000 }, - ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy, timeout: 300_000 }) } }), - }); -}; - -const gptResponseGuard = z.object({ - choices: z - .object({ - message: z.object({ role: z.string(), content: z.string() }), - finish_reason: z.string(), - }) - .array(), -}); - -type TranslateConfig = { - api: Got; - sourceFilePath: string; - targetLanguage: LanguageTag; - extraPrompt?: string; -}; - -export const translate = async ({ - api, - targetLanguage, - sourceFilePath, - extraPrompt, -}: TranslateConfig) => { - const sourceFileContent = await fs.readFile(sourceFilePath, 'utf8'); - const response = await trySafe( - api - .post('chat/completions', { - json: { - // The full list of OPENAI model can be found at https://platform.openai.com/docs/models. - model: process.env.OPENAI_MODEL_NAME ?? 'gpt-3.5-turbo-0125', - messages: getTranslationPromptMessages({ - sourceFileContent, - targetLanguage, - extraPrompt, - }), - }, - }) - .json(), - (error) => { - consoleLog.warn(`Error while translating ${sourceFilePath}:`, String(error)); - - if (error instanceof HTTPError) { - consoleLog.warn(error.response.body); - } - } - ); - - if (!response) { - return; - } - - const guarded = gptResponseGuard.safeParse(response); - - if (!guarded.success) { - consoleLog.warn(`Error while guarding response for ${sourceFilePath}:`, response); - - return; - } - - const [entity] = guarded.data.choices; - - if (!entity) { - consoleLog.warn(`No choice found in response when translating ${sourceFilePath}`); - - return; - } - - if (entity.finish_reason !== 'stop') { - consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${sourceFilePath}`); - } - - const { content } = entity.message; - const matched = /```(?:ts)?\n(.*)```/s.exec(content)?.[1]; - - if (!matched) { - // Treat as pure code - if (['const ', 'import '].some((prefix) => content.startsWith(prefix))) { - return content; - } - - consoleLog.warn('No matching code snippet from response:', content); - } - - return matched; -}; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 6ea9e51e2..faaddbff4 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,7 +6,6 @@ import { hideBin } from 'yargs/helpers'; import connector from './commands/connector/index.js'; import database from './commands/database/index.js'; import install from './commands/install/index.js'; -import translate from './commands/translate/index.js'; import tunnel from './commands/tunnel/index.js'; import { packageJson } from './package-json.js'; import { cliConfig, ConfigKey, consoleLog } from './utils.js'; @@ -48,7 +47,6 @@ void yargs(hideBin(process.argv)) .command(install) .command(database) .command(connector) - .command(translate) .command(tunnel) .demandCommand(1) .showHelpOnFail(false, `Specify ${chalk.green('--help')} for available options`) diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 66ad32d78..20375964d 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -1,9 +1,8 @@ -import { execSync, execFile } from 'node:child_process'; +import { execSync } from 'node:child_process'; import { createWriteStream, existsSync } from 'node:fs'; import { readdir, readFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import path from 'node:path'; -import { promisify } from 'node:util'; import { ConsoleLog } from '@logto/shared'; import type { Optional } from '@silverhand/essentials'; @@ -271,32 +270,3 @@ export const getConnectorPackagesFromDirectory = async (directory: string) => { (packageInfo): packageInfo is ConnectorPackage => typeof packageInfo.name === 'string' ); }; - -const execPromise = promisify(execFile); - -export const lintLocaleFiles = async ( - /** Logto instance path */ - instancePath: string, - /** Target package name, ignore to lint both `phrases` and `phrases-experience` packages */ - packageName?: string -) => { - const spinner = ora({ - text: 'Running `eslint --fix` for locales', - }).start(); - - const targetPackages = packageName ? [packageName] : ['phrases', 'phrases-experience']; - - await Promise.all( - targetPackages.map(async (packageName) => { - const phrasesPath = path.join(instancePath, 'packages', packageName); - const localesPath = path.join(phrasesPath, 'src/locales'); - await execPromise( - 'pnpm', - ['eslint', '--ext', '.ts', path.relative(phrasesPath, localesPath), '--fix'], - { cwd: phrasesPath } - ); - }) - ); - - spinner.succeed('Ran `eslint --fix` for locales'); -}; diff --git a/packages/translate/.gitignore b/packages/translate/.gitignore new file mode 100644 index 000000000..228f29f8d --- /dev/null +++ b/packages/translate/.gitignore @@ -0,0 +1 @@ +src/package-json.ts diff --git a/packages/translate/bin/index.js b/packages/translate/bin/index.js new file mode 100755 index 000000000..f053a34bd --- /dev/null +++ b/packages/translate/bin/index.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node +// eslint-disable-next-line import/no-unassigned-import +import '../lib/index.js'; diff --git a/packages/translate/package.json b/packages/translate/package.json new file mode 100644 index 000000000..a51a5bb9a --- /dev/null +++ b/packages/translate/package.json @@ -0,0 +1,80 @@ +{ + "name": "@logto/translate", + "version": "0.0.0", + "description": "A CLI tool that helps translate phrases and experience-phrases to i18n resources.", + "author": "Silverhand Inc. ", + "homepage": "https://github.com/logto-io/logto#readme", + "license": "MPL-2.0", + "type": "module", + "publishConfig": { + "access": "public" + }, + "main": "lib/index.js", + "bin": { + "logto-translate": "bin/index.js" + }, + "files": [ + "bin", + "lib" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/logto-io/logto.git" + }, + "scripts": { + "precommit": "lint-staged", + "prepare:package-json": "node -p \"'export const packageJson = ' + JSON.stringify(require('./package.json'), undefined, 2) + ';'\" > src/package-json.ts", + "build": "rm -rf lib && pnpm prepare:package-json && tsc -p tsconfig.build.json", + "dev": "tsc -p tsconfig.build.json --watch --preserveWatchOutput --incremental", + "start": "node .", + "start:dev": "pnpm build && node .", + "lint": "eslint --ext .ts src", + "lint:report": "pnpm lint --format json --output-file report.json", + "prepack": "pnpm build" + }, + "engines": { + "node": "^20.9.0" + }, + "bugs": { + "url": "https://github.com/logto-io/logto/issues" + }, + "dependencies": { + "@logto/core-kit": "workspace:^2.5.0", + "@logto/language-kit": "workspace:^1.1.0", + "@logto/phrases": "workspace:^1.13.0", + "@logto/phrases-experience": "workspace:^1.7.0", + "@logto/shared": "workspace:^3.1.1", + "@silverhand/essentials": "^2.9.1", + "chalk": "^5.3.0", + "dotenv": "^16.4.5", + "got": "^14.0.0", + "hpagent": "^1.2.0", + "inquirer": "^9.0.0", + "ora": "^8.0.1", + "p-queue": "^8.0.0", + "typescript": "^5.5.3", + "yargs": "^17.6.0", + "zod": "^3.23.8" + }, + "devDependencies": { + "@silverhand/eslint-config": "6.0.1", + "@silverhand/ts-config": "6.0.0", + "@types/inquirer": "^9.0.0", + "@types/node": "^20.9.5", + "@types/yargs": "^17.0.13", + "@vitest/coverage-v8": "^2.0.0", + "eslint": "^8.56.0", + "lint-staged": "^15.0.0", + "prettier": "^3.0.0" + }, + "eslintConfig": { + "extends": "@silverhand", + "rules": { + "no-console": "error" + }, + "ignorePatterns": [ + "src/package-json.ts" + ] + }, + "prettier": "@silverhand/eslint-config/.prettierrc" +} diff --git a/packages/translate/src/constants.ts b/packages/translate/src/constants.ts new file mode 100644 index 000000000..6bb6e7c8e --- /dev/null +++ b/packages/translate/src/constants.ts @@ -0,0 +1,5 @@ +import os from 'node:os'; +import path from 'node:path'; + +export const defaultPath = path.join(os.homedir(), 'logto'); +export const coreDirectory = 'packages/core'; diff --git a/packages/cli/src/commands/translate/create.ts b/packages/translate/src/create.ts similarity index 93% rename from packages/cli/src/commands/translate/create.ts rename to packages/translate/src/create.ts index 3e6f1a6ae..8dc8c81b5 100644 --- a/packages/cli/src/commands/translate/create.ts +++ b/packages/translate/src/create.ts @@ -3,9 +3,8 @@ import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phra import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/phrases-experience'; import type { CommandModule } from 'yargs'; -import { consoleLog, inquireInstancePath } from '../../utils.js'; - -import { createFullTranslation } from './utils.js'; +import { createFullTranslation } from './openai.js'; +import { consoleLog, inquireInstancePath } from './utils.js'; const create: CommandModule<{ path?: string }, { path?: string; 'language-tag': string }> = { command: ['create ', 'c'], diff --git a/packages/translate/src/index.ts b/packages/translate/src/index.ts new file mode 100644 index 000000000..a5d3fdf15 --- /dev/null +++ b/packages/translate/src/index.ts @@ -0,0 +1,56 @@ +import chalk from 'chalk'; +import dotenv from 'dotenv'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; + +import create from './create.js'; +import listTags from './list-tags.js'; +import { packageJson } from './package-json.js'; +import syncKeys from './sync-keys/index.js'; +import sync from './sync.js'; +import { consoleLog } from './utils.js'; + +void yargs(hideBin(process.argv)) + .version(false) + .option('env', { + alias: ['e', 'env-file'], + describe: 'The path to your `.env` file', + type: 'string', + }) + .option('version', { + alias: 'v', + describe: 'Print Logto translate CLI version', + type: 'boolean', + global: false, + }) + .middleware(({ version }) => { + if (version) { + consoleLog.plain(packageJson.name + ' v' + packageJson.version); + // eslint-disable-next-line unicorn/no-process-exit + process.exit(0); + } + }, true) + .middleware(({ env }) => { + dotenv.config({ path: env }); + }) + .option('path', { + alias: 'p', + type: 'string', + describe: 'The path to your Logto instance directory', + }) + .option('skip-core-check', { + alias: 'sc', + type: 'boolean', + describe: 'Skip checking if the core package is existed', + }) + .command(create) + .command(listTags) + .command(sync) + .command(syncKeys) + .demandCommand(1) + .showHelpOnFail(false, `Specify ${chalk.green('--help')} for available options`) + .strict() + .parserConfiguration({ + 'dot-notation': false, + }) + .parse(); diff --git a/packages/cli/src/commands/translate/list-tags.ts b/packages/translate/src/list-tags.ts similarity index 94% rename from packages/cli/src/commands/translate/list-tags.ts rename to packages/translate/src/list-tags.ts index 498fbf562..6a1b3ab57 100644 --- a/packages/cli/src/commands/translate/list-tags.ts +++ b/packages/translate/src/list-tags.ts @@ -4,7 +4,7 @@ import { isBuiltInLanguageTag as isPhrasesUiBuiltInLanguageTag } from '@logto/ph import chalk from 'chalk'; import type { CommandModule } from 'yargs'; -import { consoleLog } from '../../utils.js'; +import { consoleLog } from './utils.js'; const listTags: CommandModule> = { command: ['list-tags', 'list'], diff --git a/packages/cli/src/commands/translate/utils.ts b/packages/translate/src/openai.ts similarity index 61% rename from packages/cli/src/commands/translate/utils.ts rename to packages/translate/src/openai.ts index 8a7c818af..ef4e02a72 100644 --- a/packages/cli/src/commands/translate/utils.ts +++ b/packages/translate/src/openai.ts @@ -3,50 +3,117 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { type LanguageTag } from '@logto/language-kit'; -import { conditionalString } from '@silverhand/essentials'; -import { type Got } from 'got'; +import { conditionalString, trySafe } from '@silverhand/essentials'; +import { type Got, got, HTTPError } from 'got'; +import { HttpsProxyAgent } from 'hpagent'; import PQueue from 'p-queue'; +import { z } from 'zod'; -import { consoleLog } from '../../utils.js'; +import { getTranslationPromptMessages, untranslatedMark } from './prompts.js'; +import { + baseLanguage, + consoleLog, + getProxy, + readBaseLocaleFiles, + type TranslationOptions, +} from './utils.js'; -import { createOpenaiApi, translate } from './openai.js'; -import { untranslatedMark } from './prompts.js'; +export const createOpenaiApi = () => { + const proxy = getProxy(); -export const baseLanguage = 'en' satisfies LanguageTag; + return got.extend({ + prefixUrl: process.env.OPENAI_API_PROXY_ENDPOINT ?? 'https://api.openai.com/v1', + headers: { + Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`, + }, + timeout: { request: 300_000 }, + ...(proxy && { agent: { https: new HttpsProxyAgent({ proxy, timeout: 300_000 }) } }), + }); +}; -export const readLocaleFiles = async (directory: string): Promise => { - const entities = await fs.readdir(directory, { withFileTypes: true }); - - const result = await Promise.all( - entities.map(async (entity) => { - if (entity.isDirectory()) { - return readLocaleFiles(path.join(directory, entity.name)); - } - - return entity.name.endsWith('.ts') ? path.join(directory, entity.name) : []; +const gptResponseGuard = z.object({ + choices: z + .object({ + message: z.object({ role: z.string(), content: z.string() }), + finish_reason: z.string(), }) + .array(), +}); + +type TranslateConfig = { + api: Got; + sourceFilePath: string; + targetLanguage: LanguageTag; + extraPrompt?: string; +}; + +export const translate = async ({ + api, + targetLanguage, + sourceFilePath, + extraPrompt, +}: TranslateConfig) => { + const sourceFileContent = await fs.readFile(sourceFilePath, 'utf8'); + const response = await trySafe( + api + .post('chat/completions', { + json: { + // The full list of OPENAI model can be found at https://platform.openai.com/docs/models. + model: process.env.OPENAI_MODEL_NAME ?? 'gpt-3.5-turbo-0125', + messages: getTranslationPromptMessages({ + sourceFileContent, + targetLanguage, + extraPrompt, + }), + }, + }) + .json(), + (error) => { + consoleLog.warn(`Error while translating ${sourceFilePath}:`, String(error)); + + if (error instanceof HTTPError) { + consoleLog.warn(error.response.body); + } + } ); - return result.flat(); -}; - -export const readBaseLocaleFiles = async (directory: string): Promise => { - const enDirectory = path.join(directory, baseLanguage.toLowerCase()); - const stat = await fs.stat(enDirectory); - - if (!stat.isDirectory()) { - consoleLog.fatal(directory, 'has no `' + baseLanguage.toLowerCase() + '` directory'); + if (!response) { + return; } - return readLocaleFiles(enDirectory); -}; + const guarded = gptResponseGuard.safeParse(response); -export type TranslationOptions = { - instancePath: string; - packageName: string; - languageTag: LanguageTag; - verbose?: boolean; - queue?: PQueue; + if (!guarded.success) { + consoleLog.warn(`Error while guarding response for ${sourceFilePath}:`, response); + + return; + } + + const [entity] = guarded.data.choices; + + if (!entity) { + consoleLog.warn(`No choice found in response when translating ${sourceFilePath}`); + + return; + } + + if (entity.finish_reason !== 'stop') { + consoleLog.warn(`Unexpected finish reason ${entity.finish_reason} for ${sourceFilePath}`); + } + + const { content } = entity.message; + const matched = /```(?:ts)?\n(.*)```/s.exec(content)?.[1]; + + if (!matched) { + // Treat as pure code + if (['const ', 'import '].some((prefix) => content.startsWith(prefix))) { + return content; + } + + consoleLog.warn('No matching code snippet from response:', content); + } + + return matched; }; export const createFullTranslation = async ({ diff --git a/packages/cli/src/commands/translate/prompts.ts b/packages/translate/src/prompts.ts similarity index 100% rename from packages/cli/src/commands/translate/prompts.ts rename to packages/translate/src/prompts.ts diff --git a/packages/cli/src/commands/translate/sync-keys/index.ts b/packages/translate/src/sync-keys/index.ts similarity index 99% rename from packages/cli/src/commands/translate/sync-keys/index.ts rename to packages/translate/src/sync-keys/index.ts index f836a933d..10b855e92 100644 --- a/packages/cli/src/commands/translate/sync-keys/index.ts +++ b/packages/translate/src/sync-keys/index.ts @@ -5,7 +5,7 @@ import { isLanguageTag } from '@logto/language-kit'; import ora from 'ora'; import { type CommandModule } from 'yargs'; -import { consoleLog, inquireInstancePath, lintLocaleFiles } from '../../../utils.js'; +import { consoleLog, inquireInstancePath, lintLocaleFiles } from '../utils.js'; import { parseLocaleFiles, syncPhraseKeysAndFileStructure } from './utils.js'; diff --git a/packages/cli/src/commands/translate/sync-keys/utils.ts b/packages/translate/src/sync-keys/utils.ts similarity index 99% rename from packages/cli/src/commands/translate/sync-keys/utils.ts rename to packages/translate/src/sync-keys/utils.ts index 1bbd77607..617e903b9 100644 --- a/packages/cli/src/commands/translate/sync-keys/utils.ts +++ b/packages/translate/src/sync-keys/utils.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { tryThat } from '@silverhand/essentials'; import ts from 'typescript'; -import { consoleLog } from '../../../utils.js'; +import { consoleLog } from '../utils.js'; const getValue = (property: ts.PropertyAssignment): string => { if ( diff --git a/packages/cli/src/commands/translate/sync.ts b/packages/translate/src/sync.ts similarity index 92% rename from packages/cli/src/commands/translate/sync.ts rename to packages/translate/src/sync.ts index 6fa3ef6d2..f62cb81f5 100644 --- a/packages/cli/src/commands/translate/sync.ts +++ b/packages/translate/src/sync.ts @@ -3,9 +3,13 @@ import { isBuiltInLanguageTag as isPhrasesBuiltInLanguageTag } from '@logto/phra import PQueue from 'p-queue'; import type { CommandModule } from 'yargs'; -import { inquireInstancePath, lintLocaleFiles } from '../../utils.js'; - -import { type TranslationOptions, baseLanguage, syncTranslation } from './utils.js'; +import { syncTranslation } from './openai.js'; +import { + inquireInstancePath, + lintLocaleFiles, + type TranslationOptions, + baseLanguage, +} from './utils.js'; const sync: CommandModule< { path?: string; skipCoreCheck?: boolean }, diff --git a/packages/translate/src/utils.ts b/packages/translate/src/utils.ts new file mode 100644 index 000000000..c6bd208ff --- /dev/null +++ b/packages/translate/src/utils.ts @@ -0,0 +1,174 @@ +import { execFile } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import fs, { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +import { type LanguageTag } from '@logto/language-kit'; +import { ConsoleLog } from '@logto/shared'; +import { assert, conditional } from '@silverhand/essentials'; +import chalk from 'chalk'; +import inquirer from 'inquirer'; +import ora from 'ora'; +import type PQueue from 'p-queue'; +import { z } from 'zod'; + +import { coreDirectory, defaultPath } from './constants.js'; + +// The explicit type annotation is required to make `.fatal()` +// works correctly without `return`: +// +// ```ts +// const foo: number | undefined; +// consoleLog.fatal(); +// typeof foo // Still `number | undefined` without explicit type annotation +// ``` +// +// For now I have no idea why. +export const consoleLog: ConsoleLog = new ConsoleLog(); + +export const getProxy = () => { + const { HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy } = process.env; + + return HTTPS_PROXY ?? https_proxy ?? HTTP_PROXY ?? http_proxy; +}; + +export const getPathInModule = (moduleName: string, relativePath = '/') => + // https://stackoverflow.com/a/49455609/12514940 + path.join( + path.dirname(createRequire(import.meta.url).resolve(`${moduleName}/package.json`)), + relativePath + ); + +export const isTty = () => process.stdin.isTTY; + +const buildPathErrorMessage = (value: string) => + `The path ${chalk.green(value)} does not contain a Logto instance. Please try another.`; + +const validatePath = async (value: string) => { + const corePackageJsonPath = path.resolve(path.join(value, coreDirectory, 'package.json')); + + if (!existsSync(corePackageJsonPath)) { + return buildPathErrorMessage(value); + } + + const packageJson = await readFile(corePackageJsonPath, { encoding: 'utf8' }); + const packageName = await z + .object({ name: z.string() }) + .parseAsync(JSON.parse(packageJson)) + .then(({ name }) => name) + .catch(() => ''); + + if (packageName !== '@logto/core') { + return buildPathErrorMessage(value); + } + + return true; +}; + +export const inquireInstancePath = async (initialPath?: string, skipCoreCheck?: boolean) => { + const inquire = async () => { + if (!initialPath && (skipCoreCheck ?? (await validatePath('.')) === true)) { + return path.resolve('.'); + } + + if (!isTty()) { + assert(initialPath, new Error('Path is missing')); + + return initialPath; + } + + const { instancePath } = await inquirer.prompt<{ instancePath: string }>( + { + name: 'instancePath', + message: 'Where is your Logto instance?', + type: 'input', + default: defaultPath, + filter: (value: string) => value.trim(), + validate: conditional(!skipCoreCheck && validatePath), + }, + { instancePath: initialPath } + ); + + return instancePath; + }; + + const instancePath = await inquire(); + + if (!skipCoreCheck) { + const validated = await validatePath(instancePath); + + if (validated !== true) { + consoleLog.fatal(validated); + } + } + + return instancePath; +}; + +const execPromise = promisify(execFile); + +export const lintLocaleFiles = async ( + /** Logto instance path */ + instancePath: string, + /** Target package name, ignore to lint both `phrases` and `phrases-experience` packages */ + packageName?: string +) => { + const spinner = ora({ + text: 'Running `eslint --fix` for locales', + }).start(); + + const targetPackages = packageName ? [packageName] : ['phrases', 'phrases-experience']; + + await Promise.all( + targetPackages.map(async (packageName) => { + const phrasesPath = path.join(instancePath, 'packages', packageName); + const localesPath = path.join(phrasesPath, 'src/locales'); + await execPromise( + 'pnpm', + ['eslint', '--ext', '.ts', path.relative(phrasesPath, localesPath), '--fix'], + { cwd: phrasesPath } + ); + }) + ); + + spinner.succeed('Ran `eslint --fix` for locales'); +}; + +export const baseLanguage = 'en' satisfies LanguageTag; + +export const readLocaleFiles = async (directory: string): Promise => { + const entities = await fs.readdir(directory, { withFileTypes: true }); + + const result = await Promise.all( + entities.map(async (entity) => { + if (entity.isDirectory()) { + return readLocaleFiles(path.join(directory, entity.name)); + } + + return entity.name.endsWith('.ts') ? path.join(directory, entity.name) : []; + }) + ); + + return result.flat(); +}; + +export const readBaseLocaleFiles = async (directory: string): Promise => { + const enDirectory = path.join(directory, baseLanguage.toLowerCase()); + const stat = await fs.stat(enDirectory); + + if (!stat.isDirectory()) { + consoleLog.fatal(directory, 'has no `' + baseLanguage.toLowerCase() + '` directory'); + } + + return readLocaleFiles(enDirectory); +}; + +export type TranslationOptions = { + instancePath: string; + packageName: string; + languageTag: LanguageTag; + verbose?: boolean; + queue?: PQueue; +}; diff --git a/packages/translate/tsconfig.build.json b/packages/translate/tsconfig.build.json new file mode 100644 index 000000000..b2142cfd9 --- /dev/null +++ b/packages/translate/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig", + "include": ["src"], +} diff --git a/packages/translate/tsconfig.json b/packages/translate/tsconfig.json new file mode 100644 index 000000000..225658205 --- /dev/null +++ b/packages/translate/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@silverhand/ts-config/tsconfig.base", + "compilerOptions": { + "outDir": "lib", + "types": ["node"] + }, + "include": [ + "src" + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10acf63e8..7aa7d468e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@logto/cli': specifier: workspace:^1.1.0 version: link:packages/cli + '@logto/translate': + specifier: workspace:^0.0.0 + version: link:packages/translate devDependencies: '@changesets/cli': specifier: ^2.26.2 @@ -94,15 +97,6 @@ importers: '@logto/core-kit': specifier: workspace:^2.5.0 version: link:../toolkit/core-kit - '@logto/language-kit': - specifier: workspace:^1.1.0 - version: link:../toolkit/language-kit - '@logto/phrases': - specifier: workspace:^1.13.0 - version: link:../phrases - '@logto/phrases-experience': - specifier: workspace:^1.7.0 - version: link:../phrases-experience '@logto/schemas': specifier: workspace:1.19.0 version: link:../schemas @@ -148,27 +142,18 @@ importers: p-limit: specifier: ^6.0.0 version: 6.0.0 - p-queue: - specifier: ^8.0.0 - version: 8.0.1 p-retry: specifier: ^6.0.0 version: 6.0.0 pg-protocol: specifier: ^1.6.0 version: 1.6.0 - roarr: - specifier: ^7.11.0 - version: 7.11.0 semver: specifier: ^7.3.8 version: 7.5.2 tar: specifier: ^7.0.0 version: 7.0.1 - typescript: - specifier: ^5.5.3 - version: 5.5.3 yargs: specifier: ^17.6.0 version: 17.6.0 @@ -218,6 +203,9 @@ importers: sinon: specifier: ^18.0.0 version: 18.0.0 + typescript: + specifier: ^5.5.3 + version: 5.5.3 vitest: specifier: ^2.0.0 version: 2.0.0(@types/node@20.10.4)(happy-dom@14.12.3)(jsdom@20.0.2)(lightningcss@1.25.1)(sass@1.77.8) @@ -3838,6 +3826,85 @@ importers: specifier: ^2.0.0 version: 2.0.0(@types/node@20.10.4)(happy-dom@14.12.3)(jsdom@20.0.2)(lightningcss@1.25.1)(sass@1.77.8) + packages/translate: + dependencies: + '@logto/core-kit': + specifier: workspace:^2.5.0 + version: link:../toolkit/core-kit + '@logto/language-kit': + specifier: workspace:^1.1.0 + version: link:../toolkit/language-kit + '@logto/phrases': + specifier: workspace:^1.13.0 + version: link:../phrases + '@logto/phrases-experience': + specifier: workspace:^1.7.0 + version: link:../phrases-experience + '@logto/shared': + specifier: workspace:^3.1.1 + version: link:../shared + '@silverhand/essentials': + specifier: ^2.9.1 + version: 2.9.1 + chalk: + specifier: ^5.3.0 + version: 5.3.0 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 + got: + specifier: ^14.0.0 + version: 14.0.0 + hpagent: + specifier: ^1.2.0 + version: 1.2.0 + inquirer: + specifier: ^9.0.0 + version: 9.1.4 + ora: + specifier: ^8.0.1 + version: 8.0.1 + p-queue: + specifier: ^8.0.0 + version: 8.0.1 + typescript: + specifier: ^5.5.3 + version: 5.5.3 + yargs: + specifier: ^17.6.0 + version: 17.7.2 + zod: + specifier: ^3.23.8 + version: 3.23.8 + devDependencies: + '@silverhand/eslint-config': + specifier: 6.0.1 + version: 6.0.1(eslint@8.57.0)(prettier@3.0.0)(typescript@5.5.3) + '@silverhand/ts-config': + specifier: 6.0.0 + version: 6.0.0(typescript@5.5.3) + '@types/inquirer': + specifier: ^9.0.0 + version: 9.0.3 + '@types/node': + specifier: ^20.9.5 + version: 20.12.7 + '@types/yargs': + specifier: ^17.0.13 + version: 17.0.13 + '@vitest/coverage-v8': + specifier: ^2.0.0 + version: 2.0.0(vitest@2.0.0(@types/node@20.12.7)(happy-dom@14.12.3)(jsdom@20.0.2)(lightningcss@1.25.1)(sass@1.77.8)) + eslint: + specifier: ^8.56.0 + version: 8.57.0 + lint-staged: + specifier: ^15.0.0 + version: 15.0.2 + prettier: + specifier: ^3.0.0 + version: 3.0.0 + packages: '@75lb/deep-merge@1.1.1': @@ -16209,7 +16276,7 @@ snapshots: '@typescript-eslint/type-utils': 7.7.0(eslint@8.57.0)(typescript@5.5.3) '@typescript-eslint/utils': 7.7.0(eslint@8.57.0)(typescript@5.5.3) '@typescript-eslint/visitor-keys': 7.7.0 - debug: 4.3.4 + debug: 4.3.5 eslint: 8.57.0 graphemer: 1.4.0 ignore: 5.3.1 @@ -16227,7 +16294,7 @@ snapshots: '@typescript-eslint/types': 7.7.0 '@typescript-eslint/typescript-estree': 7.7.0(typescript@5.5.3) '@typescript-eslint/visitor-keys': 7.7.0 - debug: 4.3.4 + debug: 4.3.5 eslint: 8.57.0 optionalDependencies: typescript: 5.5.3 @@ -18120,7 +18187,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.7.0(eslint@8.57.0)(typescript@5.5.3))(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: - debug: 4.3.4 + debug: 4.3.5 enhanced-resolve: 5.16.0 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.7.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.7.0(eslint@8.57.0)(typescript@5.5.3))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) @@ -18272,7 +18339,7 @@ snapshots: eslint-plugin-sql@2.1.0(eslint@8.57.0): dependencies: astring: 1.8.3 - debug: 4.3.4 + debug: 4.3.5 eslint: 8.57.0 lodash: 4.17.21 pg-formatter: 1.3.0 @@ -18282,7 +18349,7 @@ snapshots: eslint-plugin-unicorn@52.0.0(eslint@8.57.0): dependencies: - '@babel/helper-validator-identifier': 7.22.20 + '@babel/helper-validator-identifier': 7.24.7 '@eslint-community/eslint-utils': 4.4.0(eslint@8.57.0) '@eslint/eslintrc': 2.1.4 ci-info: 4.0.0 @@ -24495,7 +24562,7 @@ snapshots: yargs@17.7.2: dependencies: cliui: 8.0.1 - escalade: 3.1.1 + escalade: 3.1.2 get-caller-file: 2.0.5 require-directory: 2.1.1 string-width: 4.2.3