Install package (#484)
This commit is contained in:
Chris
GitHub
parent
8bc951b0ff
commit
3e58377ec9
6 changed files with 39 additions and 922 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -47,3 +47,4 @@ test-results/
|
||||||
|
|
||||||
# Temporary files
|
# Temporary files
|
||||||
tmp/
|
tmp/
|
||||||
|
*.bak
|
||||||
|
|
|
||||||
10
linguaisync.config.js
Normal file
10
linguaisync.config.js
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
const path = require('path');
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
localesDir: path.join(__dirname, 'public/locales'),
|
||||||
|
baseLanguage: 'en',
|
||||||
|
translationFiles: ['translation.json', 'quotes.json'],
|
||||||
|
batchSize: 20,
|
||||||
|
model: 'gpt-4o-mini',
|
||||||
|
temperature: 0.3,
|
||||||
|
};
|
||||||
29
package-lock.json
generated
29
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "tududi",
|
"name": "tududi",
|
||||||
"version": "v0.85.1",
|
"version": "v0.86-beta.1",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "tududi",
|
"name": "tududi",
|
||||||
"version": "v0.85.1",
|
"version": "v0.86-beta.1",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@dnd-kit/core": "^6.3.1",
|
"@dnd-kit/core": "^6.3.1",
|
||||||
|
|
@ -31,6 +31,7 @@
|
||||||
"i18next-browser-languagedetector": "^8.0.4",
|
"i18next-browser-languagedetector": "^8.0.4",
|
||||||
"i18next-http-backend": "^3.0.2",
|
"i18next-http-backend": "^3.0.2",
|
||||||
"js-yaml": "~4.1.0",
|
"js-yaml": "~4.1.0",
|
||||||
|
"linguaisync": "^0.1.2",
|
||||||
"lodash": "~4.17.21",
|
"lodash": "~4.17.21",
|
||||||
"moment-timezone": "~0.6.0",
|
"moment-timezone": "~0.6.0",
|
||||||
"morgan": "~1.10.0",
|
"morgan": "~1.10.0",
|
||||||
|
|
@ -12484,6 +12485,30 @@
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/linguaisync": {
|
||||||
|
"version": "0.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/linguaisync/-/linguaisync-0.1.2.tgz",
|
||||||
|
"integrity": "sha512-7iH/O6Nonm5aaLorqBtgBA715BihTbBR4d4LKei4prETiOxWxAluFdyBXVkyIml5wg3NgQ7TyOwqV5X5Nj1EdQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"commander": "^12.0.0"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"linguaisync": "bin/sync-translations.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=14.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/linguaisync/node_modules/commander": {
|
||||||
|
"version": "12.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
|
||||||
|
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/loader-runner": {
|
"node_modules/loader-runner": {
|
||||||
"version": "4.3.0",
|
"version": "4.3.0",
|
||||||
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
|
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
|
||||||
|
|
|
||||||
|
|
@ -50,11 +50,6 @@
|
||||||
"migration:undo": "cd backend && npx sequelize-cli db:migrate:undo",
|
"migration:undo": "cd backend && npx sequelize-cli db:migrate:undo",
|
||||||
"migration:undo:all": "cd backend && npx sequelize-cli db:migrate:undo:all",
|
"migration:undo:all": "cd backend && npx sequelize-cli db:migrate:undo:all",
|
||||||
"migration:status": "cd backend && npx sequelize-cli db:migrate:status",
|
"migration:status": "cd backend && npx sequelize-cli db:migrate:status",
|
||||||
"translations:sync": "cd scripts && ./sync-translations.js",
|
|
||||||
"translations:sync-all": "cd scripts && ./sync-translations.js --all",
|
|
||||||
"translations:dry-run": "cd scripts && ./sync-translations.js --all --dry-run",
|
|
||||||
"translations:check": "cd scripts && ./sync-translations.js --all --dry-run --verbose",
|
|
||||||
"translations:export": "cd scripts && ./sync-translations.js --all --dry-run --output missing-translations.json",
|
|
||||||
"clean": "rimraf dist",
|
"clean": "rimraf dist",
|
||||||
"lint": "npm run frontend:lint && npm run backend:lint",
|
"lint": "npm run frontend:lint && npm run backend:lint",
|
||||||
"lint:fix": "npm run frontend:lint:fix && npm run backend:lint:fix",
|
"lint:fix": "npm run frontend:lint:fix && npm run backend:lint:fix",
|
||||||
|
|
@ -135,6 +130,7 @@
|
||||||
"i18next-browser-languagedetector": "^8.0.4",
|
"i18next-browser-languagedetector": "^8.0.4",
|
||||||
"i18next-http-backend": "^3.0.2",
|
"i18next-http-backend": "^3.0.2",
|
||||||
"js-yaml": "~4.1.0",
|
"js-yaml": "~4.1.0",
|
||||||
|
"linguaisync": "^0.1.2",
|
||||||
"lodash": "~4.17.21",
|
"lodash": "~4.17.21",
|
||||||
"moment-timezone": "~0.6.0",
|
"moment-timezone": "~0.6.0",
|
||||||
"morgan": "~1.10.0",
|
"morgan": "~1.10.0",
|
||||||
|
|
|
||||||
|
|
@ -1,231 +0,0 @@
|
||||||
# Translation Synchronization Script
|
|
||||||
|
|
||||||
This script helps maintain consistent translations across all language files by comparing them with the English base translation and using OpenAI to fill in missing translations.
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
### Prerequisites
|
|
||||||
|
|
||||||
1. **Node.js** (version 14 or higher)
|
|
||||||
2. **OpenAI API Key** - Get one from [OpenAI Platform](https://platform.openai.com/api-keys)
|
|
||||||
|
|
||||||
### Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd scripts
|
|
||||||
npm install
|
|
||||||
```
|
|
||||||
|
|
||||||
### Environment Setup
|
|
||||||
|
|
||||||
Set your OpenAI API key as an environment variable:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Linux/macOS
|
|
||||||
export OPENAI_API_KEY="your-openai-api-key-here"
|
|
||||||
|
|
||||||
# Windows
|
|
||||||
set OPENAI_API_KEY=your-openai-api-key-here
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Check All Languages
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./sync-translations.js --all
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check Specific Languages
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./sync-translations.js --lang=jp,el,de
|
|
||||||
./sync-translations.js --lang=es
|
|
||||||
```
|
|
||||||
|
|
||||||
### Dry Run (Preview Changes)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./sync-translations.js --all --dry-run
|
|
||||||
./sync-translations.js --lang=it --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
### Detailed Analysis
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Show detailed missing translations with English values
|
|
||||||
./sync-translations.js --lang=el --dry-run --verbose
|
|
||||||
|
|
||||||
# Export missing translations to JSON file
|
|
||||||
./sync-translations.js --all --dry-run --output missing-translations.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
1. **Base Comparison**: Uses English (`en`) as the base language
|
|
||||||
2. **Multi-File Support**: Processes both `translation.json` and `quotes.json` files
|
|
||||||
3. **Key Analysis**: Recursively compares all keys and nested objects
|
|
||||||
4. **Missing Detection**: Identifies missing translations in target languages
|
|
||||||
5. **Batch Processing**: Groups missing translations into batches of 20
|
|
||||||
6. **AI Translation**: Uses OpenAI GPT-4 to translate missing content
|
|
||||||
7. **File Updates**: Updates translation files with new translations
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- ✅ **Multi-File Support**: Handles both `translation.json` and `quotes.json` files
|
|
||||||
- ✅ **Recursive Analysis**: Handles nested translation objects
|
|
||||||
- ✅ **Batch Processing**: Efficient API usage with configurable batch sizes
|
|
||||||
- ✅ **Placeholder Preservation**: Maintains `{{variables}}` and formatting
|
|
||||||
- ✅ **Context Awareness**: Provides context for better translations
|
|
||||||
- ✅ **Error Handling**: Graceful error handling and reporting
|
|
||||||
- ✅ **Dry Run Mode**: Preview changes before applying (no API key required)
|
|
||||||
- ✅ **Progress Tracking**: Clear progress indicators
|
|
||||||
- ✅ **File Backup**: Maintains original file structure
|
|
||||||
|
|
||||||
## Supported Languages
|
|
||||||
|
|
||||||
- `ar` - Arabic (العربية) 🇸🇦
|
|
||||||
- `bg` - Bulgarian (Български) 🇧🇬
|
|
||||||
- `da` - Danish (Dansk) 🇩🇰
|
|
||||||
- `de` - German (Deutsch) 🇩🇪
|
|
||||||
- `el` - Greek (Ελληνικά) 🇬🇷
|
|
||||||
- `es` - Spanish (Español) 🇪🇸
|
|
||||||
- `fi` - Finnish (Suomi) 🇫🇮
|
|
||||||
- `fr` - French (Français) 🇫🇷
|
|
||||||
- `id` - Indonesian (Bahasa Indonesia) 🇮🇩
|
|
||||||
- `it` - Italian (Italiano) 🇮🇹
|
|
||||||
- `jp` - Japanese (日本語) 🇯🇵
|
|
||||||
- `ko` - Korean (한국어) 🇰🇷
|
|
||||||
- `nl` - Dutch (Nederlands) 🇳🇱
|
|
||||||
- `no` - Norwegian (Norsk) 🇳🇴
|
|
||||||
- `pl` - Polish (Polski) 🇵🇱
|
|
||||||
- `pt` - Portuguese (Português) 🇵🇹
|
|
||||||
- `ro` - Romanian (Română) 🇷🇴
|
|
||||||
- `ru` - Russian (Русский) 🇷🇺
|
|
||||||
- `sl` - Slovenian (Slovenščina) 🇸🇮
|
|
||||||
- `sv` - Swedish (Svenska) 🇸🇪
|
|
||||||
- `tr` - Turkish (Türkçe) 🇹🇷
|
|
||||||
- `ua` - Ukrainian (Українська) 🇺🇦
|
|
||||||
- `vi` - Vietnamese (Tiếng Việt) 🇻🇳
|
|
||||||
- `zh` - Chinese (中文) 🇨🇳
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
### Using NPM Scripts (Recommended)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Quick dry run of all languages
|
|
||||||
npm run translations:dry-run
|
|
||||||
|
|
||||||
# Detailed analysis with all missing translations
|
|
||||||
npm run translations:check
|
|
||||||
|
|
||||||
# Export missing translations to JSON file
|
|
||||||
npm run translations:export
|
|
||||||
|
|
||||||
# Update all languages (requires OPENAI_API_KEY)
|
|
||||||
npm run translations:sync-all
|
|
||||||
```
|
|
||||||
|
|
||||||
### Full Synchronization
|
|
||||||
```bash
|
|
||||||
# Update all languages with missing translations
|
|
||||||
./sync-translations.js --all
|
|
||||||
```
|
|
||||||
|
|
||||||
### Targeted Updates
|
|
||||||
```bash
|
|
||||||
# Update only Japanese and Italian
|
|
||||||
./sync-translations.js --lang=jp,it
|
|
||||||
|
|
||||||
# Preview changes for German
|
|
||||||
./sync-translations.js --lang=de --dry-run
|
|
||||||
|
|
||||||
# Detailed analysis of missing Spanish translations
|
|
||||||
./sync-translations.js --lang=es --dry-run --verbose
|
|
||||||
|
|
||||||
# Export all missing translations to review
|
|
||||||
./sync-translations.js --all --dry-run --output review.json
|
|
||||||
```
|
|
||||||
|
|
||||||
### Sample Output
|
|
||||||
```
|
|
||||||
🌍 Translation Synchronization Tool
|
|
||||||
|
|
||||||
📁 Locales directory: /path/to/public/locales
|
|
||||||
🏴 Base language: en
|
|
||||||
🔄 Processing all available languages: es, de, el, jp, ua, it
|
|
||||||
|
|
||||||
📊 Analyzing es (Spanish)...
|
|
||||||
✅ es: All translations are up to date!
|
|
||||||
|
|
||||||
📊 Analyzing de (German)...
|
|
||||||
📝 Found 5 missing translation(s) for de
|
|
||||||
📦 Processing 1 batch(es) of translations...
|
|
||||||
Batch 1/1 (5 items)...
|
|
||||||
✅ de: Applied 5/5 translations
|
|
||||||
|
|
||||||
🎉 Completed! Successfully updated 2/6 languages
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
### Batch Size
|
|
||||||
Modify `BATCH_SIZE` in the script to change how many translations are sent per API request (default: 50).
|
|
||||||
|
|
||||||
### API Model
|
|
||||||
The script uses GPT-4 by default. You can change this in the `requestTranslations` function.
|
|
||||||
|
|
||||||
### Rate Limiting
|
|
||||||
The script includes a 1-second delay between API requests to respect rate limits.
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **API Key Not Set**
|
|
||||||
```
|
|
||||||
❌ Error: OPENAI_API_KEY environment variable is required
|
|
||||||
```
|
|
||||||
Solution: Set the environment variable with your OpenAI API key.
|
|
||||||
|
|
||||||
2. **Invalid Language Code**
|
|
||||||
```
|
|
||||||
❌ Invalid language codes: xx
|
|
||||||
```
|
|
||||||
Solution: Use valid language codes from the supported list.
|
|
||||||
|
|
||||||
3. **API Rate Limits**
|
|
||||||
If you hit rate limits, the script will show an error. Wait a moment and try again.
|
|
||||||
|
|
||||||
### File Structure
|
|
||||||
|
|
||||||
The script expects the following directory structure:
|
|
||||||
```
|
|
||||||
public/
|
|
||||||
├── locales/
|
|
||||||
│ ├── en/
|
|
||||||
│ │ ├── translation.json
|
|
||||||
│ │ └── quotes.json
|
|
||||||
│ ├── es/
|
|
||||||
│ │ ├── translation.json
|
|
||||||
│ │ └── quotes.json
|
|
||||||
│ ├── de/
|
|
||||||
│ │ ├── translation.json
|
|
||||||
│ │ └── quotes.json
|
|
||||||
│ └── ...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
To add support for new languages:
|
|
||||||
|
|
||||||
1. Add the language code and name to `LANGUAGE_NAMES` object
|
|
||||||
2. Create the language directory in `public/locales/`
|
|
||||||
3. Run the script to generate initial translations
|
|
||||||
|
|
||||||
## Security
|
|
||||||
|
|
||||||
- API keys are passed securely through environment variables
|
|
||||||
- No sensitive data is logged or stored
|
|
||||||
- Translation requests use HTTPS
|
|
||||||
|
|
@ -1,684 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
const fs = require('fs');
|
|
||||||
const path = require('path');
|
|
||||||
const { program } = require('commander');
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
const LOCALES_DIR = path.join(__dirname, '../public/locales');
|
|
||||||
const BASE_LANGUAGE = 'en';
|
|
||||||
const BATCH_SIZE = 20;
|
|
||||||
const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
|
|
||||||
// Base translation files to check
|
|
||||||
const BASE_TRANSLATION_FILES = ['translation.json', 'quotes.json'];
|
|
||||||
|
|
||||||
// Language mappings for OpenAI
|
|
||||||
const LANGUAGE_NAMES = {
|
|
||||||
es: 'Spanish',
|
|
||||||
de: 'German',
|
|
||||||
el: 'Greek',
|
|
||||||
jp: 'Japanese',
|
|
||||||
ua: 'Ukrainian',
|
|
||||||
it: 'Italian',
|
|
||||||
fr: 'French',
|
|
||||||
ru: 'Russian',
|
|
||||||
tr: 'Turkish',
|
|
||||||
ko: 'Korean',
|
|
||||||
vi: 'Vietnamese',
|
|
||||||
ar: 'Arabic',
|
|
||||||
nl: 'Dutch',
|
|
||||||
ro: 'Romanian',
|
|
||||||
zh: 'Mandarin Chinese',
|
|
||||||
pt: 'Portuguese',
|
|
||||||
id: 'Indonesian',
|
|
||||||
no: 'Norwegian',
|
|
||||||
fi: 'Finnish',
|
|
||||||
da: 'Danish',
|
|
||||||
sv: 'Swedish',
|
|
||||||
pl: 'Polish',
|
|
||||||
bg: 'Bulgarian',
|
|
||||||
sl: 'Slovenian',
|
|
||||||
};
|
|
||||||
|
|
||||||
// Get OpenAI API key from environment
|
|
||||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load and parse a JSON translation file
|
|
||||||
*/
|
|
||||||
function loadTranslationFile(language, filename = 'translation.json') {
|
|
||||||
const filePath = path.join(LOCALES_DIR, language, filename);
|
|
||||||
try {
|
|
||||||
const content = fs.readFileSync(filePath, 'utf8');
|
|
||||||
return JSON.parse(content);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(
|
|
||||||
`⚠️ Warning: Could not load ${filePath}: ${error.message}`
|
|
||||||
);
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Save translation file with proper formatting
|
|
||||||
*/
|
|
||||||
function saveTranslationFile(language, data, filename = 'translation.json') {
|
|
||||||
const filePath = path.join(LOCALES_DIR, language, filename);
|
|
||||||
const dirPath = path.dirname(filePath);
|
|
||||||
|
|
||||||
// Ensure directory exists
|
|
||||||
if (!fs.existsSync(dirPath)) {
|
|
||||||
fs.mkdirSync(dirPath, { recursive: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
const jsonString = JSON.stringify(data, null, 2);
|
|
||||||
fs.writeFileSync(filePath, jsonString + '\n', 'utf8');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all available language codes
|
|
||||||
*/
|
|
||||||
function getAvailableLanguages() {
|
|
||||||
try {
|
|
||||||
return fs.readdirSync(LOCALES_DIR).filter((dir) => {
|
|
||||||
const stat = fs.statSync(path.join(LOCALES_DIR, dir));
|
|
||||||
return stat.isDirectory() && dir !== BASE_LANGUAGE;
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error reading locales directory: ${error.message}`);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get available translation files for the base language
|
|
||||||
*/
|
|
||||||
function getAvailableTranslationFiles() {
|
|
||||||
const availableFiles = [];
|
|
||||||
|
|
||||||
for (const filename of BASE_TRANSLATION_FILES) {
|
|
||||||
const filePath = path.join(LOCALES_DIR, BASE_LANGUAGE, filename);
|
|
||||||
if (fs.existsSync(filePath)) {
|
|
||||||
availableFiles.push(filename);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return availableFiles;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Recursively compare objects and find missing keys
|
|
||||||
*/
|
|
||||||
function findMissingKeys(baseObj, targetObj, currentPath = '') {
|
|
||||||
const missing = [];
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(baseObj)) {
|
|
||||||
const fullPath = currentPath ? `${currentPath}.${key}` : key;
|
|
||||||
|
|
||||||
if (!(key in targetObj)) {
|
|
||||||
// If the missing key is an object, recursively add all its nested keys
|
|
||||||
if (typeof value === 'object' && value !== null) {
|
|
||||||
missing.push(...findMissingKeys(value, {}, fullPath));
|
|
||||||
} else {
|
|
||||||
missing.push({
|
|
||||||
path: fullPath,
|
|
||||||
value: value,
|
|
||||||
isNested: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (
|
|
||||||
typeof value === 'object' &&
|
|
||||||
value !== null &&
|
|
||||||
typeof targetObj[key] === 'object' &&
|
|
||||||
targetObj[key] !== null
|
|
||||||
) {
|
|
||||||
// Recursively check nested objects
|
|
||||||
missing.push(...findMissingKeys(value, targetObj[key], fullPath));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return missing;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set nested object property using dot notation
|
|
||||||
*/
|
|
||||||
function setNestedProperty(obj, path, value) {
|
|
||||||
const keys = path.split('.');
|
|
||||||
let current = obj;
|
|
||||||
|
|
||||||
for (let i = 0; i < keys.length - 1; i++) {
|
|
||||||
const key = keys[i];
|
|
||||||
if (!(key in current) || typeof current[key] !== 'object') {
|
|
||||||
current[key] = {};
|
|
||||||
}
|
|
||||||
current = current[key];
|
|
||||||
}
|
|
||||||
|
|
||||||
current[keys[keys.length - 1]] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create translation batches for OpenAI API
|
|
||||||
*/
|
|
||||||
function createTranslationBatches(missingKeys, batchSize = BATCH_SIZE) {
|
|
||||||
const batches = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < missingKeys.length; i += batchSize) {
|
|
||||||
const batch = missingKeys.slice(i, i + batchSize);
|
|
||||||
batches.push(batch);
|
|
||||||
}
|
|
||||||
|
|
||||||
return batches;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Request translations from OpenAI API
|
|
||||||
*/
|
|
||||||
async function requestTranslations(batch, targetLanguage) {
|
|
||||||
const languageName = LANGUAGE_NAMES[targetLanguage] || targetLanguage;
|
|
||||||
|
|
||||||
// Prepare the translation request
|
|
||||||
const translationItems = batch.map((item) => ({
|
|
||||||
key: item.path,
|
|
||||||
english:
|
|
||||||
typeof item.value === 'string'
|
|
||||||
? item.value
|
|
||||||
: JSON.stringify(item.value),
|
|
||||||
}));
|
|
||||||
|
|
||||||
const prompt = `You are a professional translator. Translate the following English text to ${languageName}.
|
|
||||||
|
|
||||||
IMPORTANT INSTRUCTIONS:
|
|
||||||
1. Maintain the exact same structure and formatting
|
|
||||||
2. Preserve all placeholders like {{variable}}, {{count}}, etc.
|
|
||||||
3. Keep HTML tags intact if present
|
|
||||||
4. For technical terms, use appropriate ${languageName} equivalents
|
|
||||||
5. Maintain the tone and context appropriate for a task management application
|
|
||||||
6. If the English value is an array, return an array in the translation (NOT an object with numbered keys)
|
|
||||||
7. If the English value is an object, return an object in the translation
|
|
||||||
8. Return ONLY a JSON object with the translations
|
|
||||||
|
|
||||||
Translate these English texts:
|
|
||||||
${JSON.stringify(translationItems, null, 2)}
|
|
||||||
|
|
||||||
Return format:
|
|
||||||
{
|
|
||||||
"translations": [
|
|
||||||
{
|
|
||||||
"key": "path.to.key",
|
|
||||||
"translation": "translated text in ${languageName}"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}`;
|
|
||||||
|
|
||||||
// Debug: Log request details for troubleshooting
|
|
||||||
console.log(` 🔍 Debug: Translating ${batch.length} items to ${languageName}`);
|
|
||||||
console.log(` 📊 Prompt length: ${prompt.length} characters`);
|
|
||||||
|
|
||||||
// Check payload size
|
|
||||||
const requestPayload = {
|
|
||||||
model: 'gpt-4o-mini',
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
role: 'system',
|
|
||||||
content: `You are a professional translator specializing in software localization. Always return valid JSON in the exact format requested.`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
role: 'user',
|
|
||||||
content: prompt,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
temperature: 0.3,
|
|
||||||
max_tokens: 2000,
|
|
||||||
};
|
|
||||||
const payloadSize = JSON.stringify(requestPayload).length;
|
|
||||||
console.log(` 📦 Request payload size: ${payloadSize} bytes`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await fetch(OPENAI_API_URL, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
Authorization: `Bearer ${OPENAI_API_KEY}`,
|
|
||||||
},
|
|
||||||
body: JSON.stringify(requestPayload),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
let errorDetails = '';
|
|
||||||
try {
|
|
||||||
const errorBody = await response.json();
|
|
||||||
errorDetails = ` - ${JSON.stringify(errorBody)}`;
|
|
||||||
} catch (e) {
|
|
||||||
// If response body isn't JSON, try to get text
|
|
||||||
try {
|
|
||||||
errorDetails = ` - ${await response.text()}`;
|
|
||||||
} catch (e2) {
|
|
||||||
errorDetails = ' - Unable to read error response';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error(
|
|
||||||
`OpenAI API error: ${response.status} ${response.statusText}${errorDetails}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
const content = data.choices[0].message.content;
|
|
||||||
|
|
||||||
// Parse the JSON response
|
|
||||||
const translationResponse = JSON.parse(content);
|
|
||||||
return translationResponse.translations || [];
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error requesting translations: ${error.message}`);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process a single language
|
|
||||||
*/
|
|
||||||
async function processLanguage(language) {
|
|
||||||
console.log(
|
|
||||||
`\n📊 Analyzing ${language} (${LANGUAGE_NAMES[language] || language})...`
|
|
||||||
);
|
|
||||||
|
|
||||||
let totalMissingKeys = 0;
|
|
||||||
let totalAppliedCount = 0;
|
|
||||||
let processedFiles = 0;
|
|
||||||
|
|
||||||
// Get available translation files from base language
|
|
||||||
const availableFiles = getAvailableTranslationFiles();
|
|
||||||
|
|
||||||
// Process each translation file
|
|
||||||
for (const filename of availableFiles) {
|
|
||||||
console.log(`\n 📄 Processing ${filename}...`);
|
|
||||||
|
|
||||||
// Load base and target translation files
|
|
||||||
const baseTranslations = loadTranslationFile(BASE_LANGUAGE, filename);
|
|
||||||
const targetTranslations = loadTranslationFile(language, filename);
|
|
||||||
|
|
||||||
if (Object.keys(baseTranslations).length === 0) {
|
|
||||||
console.warn(
|
|
||||||
`⚠️ Warning: Could not load base ${filename} for ${BASE_LANGUAGE}, skipping...`
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find missing keys
|
|
||||||
const missingKeys = findMissingKeys(baseTranslations, targetTranslations);
|
|
||||||
|
|
||||||
if (missingKeys.length === 0) {
|
|
||||||
console.log(` ✅ ${filename}: All translations are up to date!`);
|
|
||||||
processedFiles++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
` 📝 Found ${missingKeys.length} missing translation(s) in ${filename}`
|
|
||||||
);
|
|
||||||
totalMissingKeys += missingKeys.length;
|
|
||||||
|
|
||||||
// Create batches for API requests
|
|
||||||
const batches = createTranslationBatches(missingKeys);
|
|
||||||
console.log(` 📦 Processing ${batches.length} batch(es) of translations...`);
|
|
||||||
|
|
||||||
let allTranslations = [];
|
|
||||||
|
|
||||||
// Process each batch
|
|
||||||
for (let i = 0; i < batches.length; i++) {
|
|
||||||
const batch = batches[i];
|
|
||||||
console.log(
|
|
||||||
` Batch ${i + 1}/${batches.length} (${batch.length} items)...`
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const translations = await requestTranslations(batch, language);
|
|
||||||
allTranslations.push(...translations);
|
|
||||||
|
|
||||||
// Add a small delay between requests to be respectful to the API
|
|
||||||
if (i < batches.length - 1) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error(
|
|
||||||
` ❌ Error processing batch ${i + 1}: ${error.message}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (allTranslations.length === 0) {
|
|
||||||
console.error(` ❌ No translations received for ${filename}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply translations to the target object
|
|
||||||
const updatedTranslations = { ...targetTranslations };
|
|
||||||
let appliedCount = 0;
|
|
||||||
|
|
||||||
for (const translation of allTranslations) {
|
|
||||||
try {
|
|
||||||
// Try to parse as JSON if it looks like an object
|
|
||||||
let translatedValue = translation.translation;
|
|
||||||
if (
|
|
||||||
translatedValue.startsWith('{') ||
|
|
||||||
translatedValue.startsWith('[')
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
translatedValue = JSON.parse(translatedValue);
|
|
||||||
|
|
||||||
// Check if this should be an array but was translated as a numbered object
|
|
||||||
if (typeof translatedValue === 'object' && translatedValue !== null && !Array.isArray(translatedValue)) {
|
|
||||||
const keys = Object.keys(translatedValue);
|
|
||||||
const isNumberedObject = keys.every((key, index) => key === index.toString());
|
|
||||||
|
|
||||||
if (isNumberedObject && keys.length > 0) {
|
|
||||||
// Convert numbered object back to array
|
|
||||||
translatedValue = keys.map(key => translatedValue[key]);
|
|
||||||
console.log(` 🔧 Fixed numbered object to array for ${translation.key}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Keep as string if JSON parsing fails
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
setNestedProperty(
|
|
||||||
updatedTranslations,
|
|
||||||
translation.key,
|
|
||||||
translatedValue
|
|
||||||
);
|
|
||||||
appliedCount++;
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(
|
|
||||||
` ⚠️ Warning: Could not apply translation for ${translation.key}: ${error.message}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save the updated translations
|
|
||||||
saveTranslationFile(language, updatedTranslations, filename);
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
` ✅ ${filename}: Applied ${appliedCount}/${missingKeys.length} translations`
|
|
||||||
);
|
|
||||||
|
|
||||||
totalAppliedCount += appliedCount;
|
|
||||||
processedFiles++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (processedFiles === 0) {
|
|
||||||
console.error(`❌ ${language}: No files could be processed`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (totalMissingKeys === 0) {
|
|
||||||
console.log(`✅ ${language}: All translation files are up to date!`);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`✅ ${language}: Applied ${totalAppliedCount}/${totalMissingKeys} total translations across ${processedFiles} files`
|
|
||||||
);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Main execution function
|
|
||||||
*/
|
|
||||||
async function main() {
|
|
||||||
program
|
|
||||||
.name('sync-translations')
|
|
||||||
.description(
|
|
||||||
'Synchronize translation files with English base and fill missing translations using OpenAI'
|
|
||||||
)
|
|
||||||
.option('--all', 'Update all available languages')
|
|
||||||
.option(
|
|
||||||
'--lang <languages>',
|
|
||||||
'Comma-separated list of language codes to update (e.g., jp,el,de)'
|
|
||||||
)
|
|
||||||
.option(
|
|
||||||
'--dry-run',
|
|
||||||
'Show what would be updated without making changes'
|
|
||||||
)
|
|
||||||
.option(
|
|
||||||
'--verbose',
|
|
||||||
'Show detailed missing translations in dry run mode'
|
|
||||||
)
|
|
||||||
.option('--output <file>', 'Save missing translations to a JSON file')
|
|
||||||
.parse();
|
|
||||||
|
|
||||||
const options = program.opts();
|
|
||||||
|
|
||||||
console.log('🌍 Translation Synchronization Tool\n');
|
|
||||||
console.log(`📁 Locales directory: ${LOCALES_DIR}`);
|
|
||||||
console.log(`🏴 Base language: ${BASE_LANGUAGE}`);
|
|
||||||
|
|
||||||
// Show available translation files
|
|
||||||
const availableFiles = getAvailableTranslationFiles();
|
|
||||||
console.log(`📄 Available translation files: ${availableFiles.join(', ')}`);
|
|
||||||
|
|
||||||
// Determine which languages to process
|
|
||||||
let languagesToProcess = [];
|
|
||||||
|
|
||||||
if (options.all) {
|
|
||||||
languagesToProcess = getAvailableLanguages();
|
|
||||||
console.log(
|
|
||||||
`🔄 Processing all available languages: ${languagesToProcess.join(', ')}`
|
|
||||||
);
|
|
||||||
} else if (options.lang) {
|
|
||||||
languagesToProcess = options.lang.split(',').map((lang) => lang.trim());
|
|
||||||
console.log(
|
|
||||||
`🎯 Processing specified languages: ${languagesToProcess.join(', ')}`
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
console.log('❓ No languages specified. Use --all or --lang=<codes>');
|
|
||||||
console.log(
|
|
||||||
'\nAvailable languages:',
|
|
||||||
getAvailableLanguages().join(', ')
|
|
||||||
);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (languagesToProcess.length === 0) {
|
|
||||||
console.log('❌ No valid languages to process');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate that all specified languages exist
|
|
||||||
const availableLanguages = getAvailableLanguages();
|
|
||||||
const invalidLanguages = languagesToProcess.filter(
|
|
||||||
(lang) => !availableLanguages.includes(lang)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (invalidLanguages.length > 0) {
|
|
||||||
console.error(
|
|
||||||
`❌ Invalid language codes: ${invalidLanguages.join(', ')}`
|
|
||||||
);
|
|
||||||
console.error(`Available languages: ${availableLanguages.join(', ')}`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.dryRun) {
|
|
||||||
console.log('\n🧪 DRY RUN MODE - No files will be modified\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process each language
|
|
||||||
let successCount = 0;
|
|
||||||
const allMissingTranslations = {};
|
|
||||||
|
|
||||||
for (const language of languagesToProcess) {
|
|
||||||
try {
|
|
||||||
if (options.dryRun) {
|
|
||||||
// In dry run, just analyze without making API calls
|
|
||||||
console.log(
|
|
||||||
`\n📊 Analyzing ${language} (${LANGUAGE_NAMES[language] || language})...`
|
|
||||||
);
|
|
||||||
|
|
||||||
let totalMissingKeys = 0;
|
|
||||||
let allMissingForLanguage = [];
|
|
||||||
|
|
||||||
// Get available translation files from base language
|
|
||||||
const availableFiles = getAvailableTranslationFiles();
|
|
||||||
|
|
||||||
// Process each translation file
|
|
||||||
for (const filename of availableFiles) {
|
|
||||||
console.log(`\n 📄 Checking ${filename}...`);
|
|
||||||
|
|
||||||
const baseTranslations = loadTranslationFile(BASE_LANGUAGE, filename);
|
|
||||||
const targetTranslations = loadTranslationFile(language, filename);
|
|
||||||
|
|
||||||
if (Object.keys(baseTranslations).length === 0) {
|
|
||||||
console.warn(
|
|
||||||
` ⚠️ Warning: Could not load base ${filename} for ${BASE_LANGUAGE}, skipping...`
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const missingKeys = findMissingKeys(
|
|
||||||
baseTranslations,
|
|
||||||
targetTranslations
|
|
||||||
);
|
|
||||||
|
|
||||||
if (missingKeys.length === 0) {
|
|
||||||
console.log(` ✅ ${filename}: All translations are up to date!`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
` 📝 Found ${missingKeys.length} missing translation(s) in ${filename}`
|
|
||||||
);
|
|
||||||
|
|
||||||
totalMissingKeys += missingKeys.length;
|
|
||||||
|
|
||||||
// Add filename prefix to paths for clarity
|
|
||||||
const prefixedMissingKeys = missingKeys.map(item => ({
|
|
||||||
...item,
|
|
||||||
path: `${filename}:${item.path}`,
|
|
||||||
filename: filename
|
|
||||||
}));
|
|
||||||
|
|
||||||
allMissingForLanguage.push(...prefixedMissingKeys);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`\n📊 ${language}: Would update ${totalMissingKeys} missing translation(s) across ${availableFiles.length} files`
|
|
||||||
);
|
|
||||||
|
|
||||||
// Store missing translations for potential output
|
|
||||||
if (totalMissingKeys > 0) {
|
|
||||||
allMissingTranslations[language] = {
|
|
||||||
languageName: LANGUAGE_NAMES[language] || language,
|
|
||||||
missingCount: totalMissingKeys,
|
|
||||||
missing: allMissingForLanguage.map((item) => ({
|
|
||||||
path: item.path,
|
|
||||||
englishValue: item.value,
|
|
||||||
isNested: item.isNested,
|
|
||||||
filename: item.filename,
|
|
||||||
})),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (totalMissingKeys > 0) {
|
|
||||||
if (options.verbose) {
|
|
||||||
console.log(
|
|
||||||
`\n🔍 Detailed missing translations for ${language}:`
|
|
||||||
);
|
|
||||||
console.log('─'.repeat(60));
|
|
||||||
|
|
||||||
allMissingForLanguage.forEach((item, index) => {
|
|
||||||
const displayValue =
|
|
||||||
typeof item.value === 'string'
|
|
||||||
? item.value.length > 100
|
|
||||||
? item.value.substring(0, 97) + '...'
|
|
||||||
: item.value
|
|
||||||
: JSON.stringify(item.value);
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`${(index + 1).toString().padStart(3)}. ${item.path}`
|
|
||||||
);
|
|
||||||
console.log(` 📝 EN: "${displayValue}"`);
|
|
||||||
console.log(
|
|
||||||
` 🎯 ${LANGUAGE_NAMES[language] || language}: [MISSING]`
|
|
||||||
);
|
|
||||||
|
|
||||||
if (index < allMissingForLanguage.length - 1) {
|
|
||||||
console.log('');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
console.log('─'.repeat(60));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Check API key for actual processing
|
|
||||||
if (!OPENAI_API_KEY) {
|
|
||||||
console.error('❌ Error: OPENAI_API_KEY environment variable is required for actual translation updates');
|
|
||||||
console.error('Please set it with: export OPENAI_API_KEY="your-api-key"');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const success = await processLanguage(language);
|
|
||||||
if (success) successCount++;
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error processing ${language}: ${error.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!options.dryRun) {
|
|
||||||
console.log(
|
|
||||||
`\n🎉 Completed! Successfully updated ${successCount}/${languagesToProcess.length} languages`
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
// Show summary for dry run
|
|
||||||
const totalMissing = Object.values(allMissingTranslations).reduce((sum, lang) => sum + lang.missingCount, 0);
|
|
||||||
if (totalMissing > 0) {
|
|
||||||
console.log(`\n📋 Summary: ${totalMissing} total missing translations across ${Object.keys(allMissingTranslations).length} languages`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle output file for dry run
|
|
||||||
if (options.output && Object.keys(allMissingTranslations).length > 0) {
|
|
||||||
const outputData = {
|
|
||||||
generatedAt: new Date().toISOString(),
|
|
||||||
baseLanguage: BASE_LANGUAGE,
|
|
||||||
languages: allMissingTranslations,
|
|
||||||
summary: {
|
|
||||||
totalLanguages: Object.keys(allMissingTranslations).length,
|
|
||||||
totalMissingTranslations: totalMissing,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
fs.writeFileSync(
|
|
||||||
options.output,
|
|
||||||
JSON.stringify(outputData, null, 2)
|
|
||||||
);
|
|
||||||
console.log(`💾 Missing translations saved to: ${options.output}`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error saving output file: ${error.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle uncaught errors
|
|
||||||
process.on('unhandledRejection', (error) => {
|
|
||||||
console.error('❌ Unhandled error:', error);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Run the script
|
|
||||||
if (require.main === module) {
|
|
||||||
main().catch((error) => {
|
|
||||||
console.error('❌ Script failed:', error);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
loadTranslationFile,
|
|
||||||
findMissingKeys,
|
|
||||||
createTranslationBatches,
|
|
||||||
processLanguage,
|
|
||||||
};
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue