Install package (#484)
This commit is contained in:
Chris
GitHub
parent
8bc951b0ff
commit
3e58377ec9
6 changed files with 39 additions and 922 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -47,3 +47,4 @@ test-results/
|
|||
|
||||
# Temporary files
|
||||
tmp/
|
||||
*.bak
|
||||
|
|
|
|||
10
linguaisync.config.js
Normal file
10
linguaisync.config.js
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
const path = require('path');
|
||||
|
||||
module.exports = {
|
||||
localesDir: path.join(__dirname, 'public/locales'),
|
||||
baseLanguage: 'en',
|
||||
translationFiles: ['translation.json', 'quotes.json'],
|
||||
batchSize: 20,
|
||||
model: 'gpt-4o-mini',
|
||||
temperature: 0.3,
|
||||
};
|
||||
29
package-lock.json
generated
29
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "tududi",
|
||||
"version": "v0.85.1",
|
||||
"version": "v0.86-beta.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "tududi",
|
||||
"version": "v0.85.1",
|
||||
"version": "v0.86-beta.1",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
|
|
@ -31,6 +31,7 @@
|
|||
"i18next-browser-languagedetector": "^8.0.4",
|
||||
"i18next-http-backend": "^3.0.2",
|
||||
"js-yaml": "~4.1.0",
|
||||
"linguaisync": "^0.1.2",
|
||||
"lodash": "~4.17.21",
|
||||
"moment-timezone": "~0.6.0",
|
||||
"morgan": "~1.10.0",
|
||||
|
|
@ -12484,6 +12485,30 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/linguaisync": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/linguaisync/-/linguaisync-0.1.2.tgz",
|
||||
"integrity": "sha512-7iH/O6Nonm5aaLorqBtgBA715BihTbBR4d4LKei4prETiOxWxAluFdyBXVkyIml5wg3NgQ7TyOwqV5X5Nj1EdQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"commander": "^12.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"linguaisync": "bin/sync-translations.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/linguaisync/node_modules/commander": {
|
||||
"version": "12.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
|
||||
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/loader-runner": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
|
||||
|
|
|
|||
|
|
@ -50,11 +50,6 @@
|
|||
"migration:undo": "cd backend && npx sequelize-cli db:migrate:undo",
|
||||
"migration:undo:all": "cd backend && npx sequelize-cli db:migrate:undo:all",
|
||||
"migration:status": "cd backend && npx sequelize-cli db:migrate:status",
|
||||
"translations:sync": "cd scripts && ./sync-translations.js",
|
||||
"translations:sync-all": "cd scripts && ./sync-translations.js --all",
|
||||
"translations:dry-run": "cd scripts && ./sync-translations.js --all --dry-run",
|
||||
"translations:check": "cd scripts && ./sync-translations.js --all --dry-run --verbose",
|
||||
"translations:export": "cd scripts && ./sync-translations.js --all --dry-run --output missing-translations.json",
|
||||
"clean": "rimraf dist",
|
||||
"lint": "npm run frontend:lint && npm run backend:lint",
|
||||
"lint:fix": "npm run frontend:lint:fix && npm run backend:lint:fix",
|
||||
|
|
@ -135,6 +130,7 @@
|
|||
"i18next-browser-languagedetector": "^8.0.4",
|
||||
"i18next-http-backend": "^3.0.2",
|
||||
"js-yaml": "~4.1.0",
|
||||
"linguaisync": "^0.1.2",
|
||||
"lodash": "~4.17.21",
|
||||
"moment-timezone": "~0.6.0",
|
||||
"morgan": "~1.10.0",
|
||||
|
|
|
|||
|
|
@ -1,231 +0,0 @@
|
|||
# Translation Synchronization Script
|
||||
|
||||
This script helps maintain consistent translations across all language files by comparing them with the English base translation and using OpenAI to fill in missing translations.
|
||||
|
||||
## Setup
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. **Node.js** (version 14 or higher)
|
||||
2. **OpenAI API Key** - Get one from [OpenAI Platform](https://platform.openai.com/api-keys)
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
cd scripts
|
||||
npm install
|
||||
```
|
||||
|
||||
### Environment Setup
|
||||
|
||||
Set your OpenAI API key as an environment variable:
|
||||
|
||||
```bash
|
||||
# Linux/macOS
|
||||
export OPENAI_API_KEY="your-openai-api-key-here"
|
||||
|
||||
# Windows
|
||||
set OPENAI_API_KEY=your-openai-api-key-here
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Check All Languages
|
||||
|
||||
```bash
|
||||
./sync-translations.js --all
|
||||
```
|
||||
|
||||
### Check Specific Languages
|
||||
|
||||
```bash
|
||||
./sync-translations.js --lang=jp,el,de
|
||||
./sync-translations.js --lang=es
|
||||
```
|
||||
|
||||
### Dry Run (Preview Changes)
|
||||
|
||||
```bash
|
||||
./sync-translations.js --all --dry-run
|
||||
./sync-translations.js --lang=it --dry-run
|
||||
```
|
||||
|
||||
### Detailed Analysis
|
||||
|
||||
```bash
|
||||
# Show detailed missing translations with English values
|
||||
./sync-translations.js --lang=el --dry-run --verbose
|
||||
|
||||
# Export missing translations to JSON file
|
||||
./sync-translations.js --all --dry-run --output missing-translations.json
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Base Comparison**: Uses English (`en`) as the base language
|
||||
2. **Multi-File Support**: Processes both `translation.json` and `quotes.json` files
|
||||
3. **Key Analysis**: Recursively compares all keys and nested objects
|
||||
4. **Missing Detection**: Identifies missing translations in target languages
|
||||
5. **Batch Processing**: Groups missing translations into batches of 20
|
||||
6. **AI Translation**: Uses OpenAI GPT-4 to translate missing content
|
||||
7. **File Updates**: Updates translation files with new translations
|
||||
|
||||
## Features
|
||||
|
||||
- ✅ **Multi-File Support**: Handles both `translation.json` and `quotes.json` files
|
||||
- ✅ **Recursive Analysis**: Handles nested translation objects
|
||||
- ✅ **Batch Processing**: Efficient API usage with configurable batch sizes
|
||||
- ✅ **Placeholder Preservation**: Maintains `{{variables}}` and formatting
|
||||
- ✅ **Context Awareness**: Provides context for better translations
|
||||
- ✅ **Error Handling**: Graceful error handling and reporting
|
||||
- ✅ **Dry Run Mode**: Preview changes before applying (no API key required)
|
||||
- ✅ **Progress Tracking**: Clear progress indicators
|
||||
- ✅ **File Backup**: Maintains original file structure
|
||||
|
||||
## Supported Languages
|
||||
|
||||
- `ar` - Arabic (العربية) 🇸🇦
|
||||
- `bg` - Bulgarian (Български) 🇧🇬
|
||||
- `da` - Danish (Dansk) 🇩🇰
|
||||
- `de` - German (Deutsch) 🇩🇪
|
||||
- `el` - Greek (Ελληνικά) 🇬🇷
|
||||
- `es` - Spanish (Español) 🇪🇸
|
||||
- `fi` - Finnish (Suomi) 🇫🇮
|
||||
- `fr` - French (Français) 🇫🇷
|
||||
- `id` - Indonesian (Bahasa Indonesia) 🇮🇩
|
||||
- `it` - Italian (Italiano) 🇮🇹
|
||||
- `jp` - Japanese (日本語) 🇯🇵
|
||||
- `ko` - Korean (한국어) 🇰🇷
|
||||
- `nl` - Dutch (Nederlands) 🇳🇱
|
||||
- `no` - Norwegian (Norsk) 🇳🇴
|
||||
- `pl` - Polish (Polski) 🇵🇱
|
||||
- `pt` - Portuguese (Português) 🇵🇹
|
||||
- `ro` - Romanian (Română) 🇷🇴
|
||||
- `ru` - Russian (Русский) 🇷🇺
|
||||
- `sl` - Slovenian (Slovenščina) 🇸🇮
|
||||
- `sv` - Swedish (Svenska) 🇸🇪
|
||||
- `tr` - Turkish (Türkçe) 🇹🇷
|
||||
- `ua` - Ukrainian (Українська) 🇺🇦
|
||||
- `vi` - Vietnamese (Tiếng Việt) 🇻🇳
|
||||
- `zh` - Chinese (中文) 🇨🇳
|
||||
|
||||
## Examples
|
||||
|
||||
### Using NPM Scripts (Recommended)
|
||||
|
||||
```bash
|
||||
# Quick dry run of all languages
|
||||
npm run translations:dry-run
|
||||
|
||||
# Detailed analysis with all missing translations
|
||||
npm run translations:check
|
||||
|
||||
# Export missing translations to JSON file
|
||||
npm run translations:export
|
||||
|
||||
# Update all languages (requires OPENAI_API_KEY)
|
||||
npm run translations:sync-all
|
||||
```
|
||||
|
||||
### Full Synchronization
|
||||
```bash
|
||||
# Update all languages with missing translations
|
||||
./sync-translations.js --all
|
||||
```
|
||||
|
||||
### Targeted Updates
|
||||
```bash
|
||||
# Update only Japanese and Italian
|
||||
./sync-translations.js --lang=jp,it
|
||||
|
||||
# Preview changes for German
|
||||
./sync-translations.js --lang=de --dry-run
|
||||
|
||||
# Detailed analysis of missing Spanish translations
|
||||
./sync-translations.js --lang=es --dry-run --verbose
|
||||
|
||||
# Export all missing translations to review
|
||||
./sync-translations.js --all --dry-run --output review.json
|
||||
```
|
||||
|
||||
### Sample Output
|
||||
```
|
||||
🌍 Translation Synchronization Tool
|
||||
|
||||
📁 Locales directory: /path/to/public/locales
|
||||
🏴 Base language: en
|
||||
🔄 Processing all available languages: es, de, el, jp, ua, it
|
||||
|
||||
📊 Analyzing es (Spanish)...
|
||||
✅ es: All translations are up to date!
|
||||
|
||||
📊 Analyzing de (German)...
|
||||
📝 Found 5 missing translation(s) for de
|
||||
📦 Processing 1 batch(es) of translations...
|
||||
Batch 1/1 (5 items)...
|
||||
✅ de: Applied 5/5 translations
|
||||
|
||||
🎉 Completed! Successfully updated 2/6 languages
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Batch Size
|
||||
Modify `BATCH_SIZE` in the script to change how many translations are sent per API request (default: 50).
|
||||
|
||||
### API Model
|
||||
The script uses GPT-4 by default. You can change this in the `requestTranslations` function.
|
||||
|
||||
### Rate Limiting
|
||||
The script includes a 1-second delay between API requests to respect rate limits.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **API Key Not Set**
|
||||
```
|
||||
❌ Error: OPENAI_API_KEY environment variable is required
|
||||
```
|
||||
Solution: Set the environment variable with your OpenAI API key.
|
||||
|
||||
2. **Invalid Language Code**
|
||||
```
|
||||
❌ Invalid language codes: xx
|
||||
```
|
||||
Solution: Use valid language codes from the supported list.
|
||||
|
||||
3. **API Rate Limits**
|
||||
If you hit rate limits, the script will show an error. Wait a moment and try again.
|
||||
|
||||
### File Structure
|
||||
|
||||
The script expects the following directory structure:
|
||||
```
|
||||
public/
|
||||
├── locales/
|
||||
│ ├── en/
|
||||
│ │ ├── translation.json
|
||||
│ │ └── quotes.json
|
||||
│ ├── es/
|
||||
│ │ ├── translation.json
|
||||
│ │ └── quotes.json
|
||||
│ ├── de/
|
||||
│ │ ├── translation.json
|
||||
│ │ └── quotes.json
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
To add support for new languages:
|
||||
|
||||
1. Add the language code and name to `LANGUAGE_NAMES` object
|
||||
2. Create the language directory in `public/locales/`
|
||||
3. Run the script to generate initial translations
|
||||
|
||||
## Security
|
||||
|
||||
- API keys are passed securely through environment variables
|
||||
- No sensitive data is logged or stored
|
||||
- Translation requests use HTTPS
|
||||
|
|
@ -1,684 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { program } = require('commander');
|
||||
|
||||
// Configuration
|
||||
const LOCALES_DIR = path.join(__dirname, '../public/locales');
|
||||
const BASE_LANGUAGE = 'en';
|
||||
const BATCH_SIZE = 20;
|
||||
const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
|
||||
// Base translation files to check
|
||||
const BASE_TRANSLATION_FILES = ['translation.json', 'quotes.json'];
|
||||
|
||||
// Language mappings for OpenAI
|
||||
const LANGUAGE_NAMES = {
|
||||
es: 'Spanish',
|
||||
de: 'German',
|
||||
el: 'Greek',
|
||||
jp: 'Japanese',
|
||||
ua: 'Ukrainian',
|
||||
it: 'Italian',
|
||||
fr: 'French',
|
||||
ru: 'Russian',
|
||||
tr: 'Turkish',
|
||||
ko: 'Korean',
|
||||
vi: 'Vietnamese',
|
||||
ar: 'Arabic',
|
||||
nl: 'Dutch',
|
||||
ro: 'Romanian',
|
||||
zh: 'Mandarin Chinese',
|
||||
pt: 'Portuguese',
|
||||
id: 'Indonesian',
|
||||
no: 'Norwegian',
|
||||
fi: 'Finnish',
|
||||
da: 'Danish',
|
||||
sv: 'Swedish',
|
||||
pl: 'Polish',
|
||||
bg: 'Bulgarian',
|
||||
sl: 'Slovenian',
|
||||
};
|
||||
|
||||
// Get OpenAI API key from environment
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
||||
|
||||
/**
|
||||
* Load and parse a JSON translation file
|
||||
*/
|
||||
function loadTranslationFile(language, filename = 'translation.json') {
|
||||
const filePath = path.join(LOCALES_DIR, language, filename);
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`⚠️ Warning: Could not load ${filePath}: ${error.message}`
|
||||
);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save translation file with proper formatting
|
||||
*/
|
||||
function saveTranslationFile(language, data, filename = 'translation.json') {
|
||||
const filePath = path.join(LOCALES_DIR, language, filename);
|
||||
const dirPath = path.dirname(filePath);
|
||||
|
||||
// Ensure directory exists
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
const jsonString = JSON.stringify(data, null, 2);
|
||||
fs.writeFileSync(filePath, jsonString + '\n', 'utf8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available language codes
|
||||
*/
|
||||
function getAvailableLanguages() {
|
||||
try {
|
||||
return fs.readdirSync(LOCALES_DIR).filter((dir) => {
|
||||
const stat = fs.statSync(path.join(LOCALES_DIR, dir));
|
||||
return stat.isDirectory() && dir !== BASE_LANGUAGE;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`❌ Error reading locales directory: ${error.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available translation files for the base language
|
||||
*/
|
||||
function getAvailableTranslationFiles() {
|
||||
const availableFiles = [];
|
||||
|
||||
for (const filename of BASE_TRANSLATION_FILES) {
|
||||
const filePath = path.join(LOCALES_DIR, BASE_LANGUAGE, filename);
|
||||
if (fs.existsSync(filePath)) {
|
||||
availableFiles.push(filename);
|
||||
}
|
||||
}
|
||||
|
||||
return availableFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively compare objects and find missing keys
|
||||
*/
|
||||
function findMissingKeys(baseObj, targetObj, currentPath = '') {
|
||||
const missing = [];
|
||||
|
||||
for (const [key, value] of Object.entries(baseObj)) {
|
||||
const fullPath = currentPath ? `${currentPath}.${key}` : key;
|
||||
|
||||
if (!(key in targetObj)) {
|
||||
// If the missing key is an object, recursively add all its nested keys
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
missing.push(...findMissingKeys(value, {}, fullPath));
|
||||
} else {
|
||||
missing.push({
|
||||
path: fullPath,
|
||||
value: value,
|
||||
isNested: false,
|
||||
});
|
||||
}
|
||||
} else if (
|
||||
typeof value === 'object' &&
|
||||
value !== null &&
|
||||
typeof targetObj[key] === 'object' &&
|
||||
targetObj[key] !== null
|
||||
) {
|
||||
// Recursively check nested objects
|
||||
missing.push(...findMissingKeys(value, targetObj[key], fullPath));
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set nested object property using dot notation
|
||||
*/
|
||||
function setNestedProperty(obj, path, value) {
|
||||
const keys = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < keys.length - 1; i++) {
|
||||
const key = keys[i];
|
||||
if (!(key in current) || typeof current[key] !== 'object') {
|
||||
current[key] = {};
|
||||
}
|
||||
current = current[key];
|
||||
}
|
||||
|
||||
current[keys[keys.length - 1]] = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create translation batches for OpenAI API
|
||||
*/
|
||||
function createTranslationBatches(missingKeys, batchSize = BATCH_SIZE) {
|
||||
const batches = [];
|
||||
|
||||
for (let i = 0; i < missingKeys.length; i += batchSize) {
|
||||
const batch = missingKeys.slice(i, i + batchSize);
|
||||
batches.push(batch);
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Request translations from OpenAI API
|
||||
*/
|
||||
async function requestTranslations(batch, targetLanguage) {
|
||||
const languageName = LANGUAGE_NAMES[targetLanguage] || targetLanguage;
|
||||
|
||||
// Prepare the translation request
|
||||
const translationItems = batch.map((item) => ({
|
||||
key: item.path,
|
||||
english:
|
||||
typeof item.value === 'string'
|
||||
? item.value
|
||||
: JSON.stringify(item.value),
|
||||
}));
|
||||
|
||||
const prompt = `You are a professional translator. Translate the following English text to ${languageName}.
|
||||
|
||||
IMPORTANT INSTRUCTIONS:
|
||||
1. Maintain the exact same structure and formatting
|
||||
2. Preserve all placeholders like {{variable}}, {{count}}, etc.
|
||||
3. Keep HTML tags intact if present
|
||||
4. For technical terms, use appropriate ${languageName} equivalents
|
||||
5. Maintain the tone and context appropriate for a task management application
|
||||
6. If the English value is an array, return an array in the translation (NOT an object with numbered keys)
|
||||
7. If the English value is an object, return an object in the translation
|
||||
8. Return ONLY a JSON object with the translations
|
||||
|
||||
Translate these English texts:
|
||||
${JSON.stringify(translationItems, null, 2)}
|
||||
|
||||
Return format:
|
||||
{
|
||||
"translations": [
|
||||
{
|
||||
"key": "path.to.key",
|
||||
"translation": "translated text in ${languageName}"
|
||||
}
|
||||
]
|
||||
}`;
|
||||
|
||||
// Debug: Log request details for troubleshooting
|
||||
console.log(` 🔍 Debug: Translating ${batch.length} items to ${languageName}`);
|
||||
console.log(` 📊 Prompt length: ${prompt.length} characters`);
|
||||
|
||||
// Check payload size
|
||||
const requestPayload = {
|
||||
model: 'gpt-4o-mini',
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You are a professional translator specializing in software localization. Always return valid JSON in the exact format requested.`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
temperature: 0.3,
|
||||
max_tokens: 2000,
|
||||
};
|
||||
const payloadSize = JSON.stringify(requestPayload).length;
|
||||
console.log(` 📦 Request payload size: ${payloadSize} bytes`);
|
||||
|
||||
try {
|
||||
const response = await fetch(OPENAI_API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${OPENAI_API_KEY}`,
|
||||
},
|
||||
body: JSON.stringify(requestPayload),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let errorDetails = '';
|
||||
try {
|
||||
const errorBody = await response.json();
|
||||
errorDetails = ` - ${JSON.stringify(errorBody)}`;
|
||||
} catch (e) {
|
||||
// If response body isn't JSON, try to get text
|
||||
try {
|
||||
errorDetails = ` - ${await response.text()}`;
|
||||
} catch (e2) {
|
||||
errorDetails = ' - Unable to read error response';
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`OpenAI API error: ${response.status} ${response.statusText}${errorDetails}`
|
||||
);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const content = data.choices[0].message.content;
|
||||
|
||||
// Parse the JSON response
|
||||
const translationResponse = JSON.parse(content);
|
||||
return translationResponse.translations || [];
|
||||
} catch (error) {
|
||||
console.error(`❌ Error requesting translations: ${error.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single language
|
||||
*/
|
||||
async function processLanguage(language) {
|
||||
console.log(
|
||||
`\n📊 Analyzing ${language} (${LANGUAGE_NAMES[language] || language})...`
|
||||
);
|
||||
|
||||
let totalMissingKeys = 0;
|
||||
let totalAppliedCount = 0;
|
||||
let processedFiles = 0;
|
||||
|
||||
// Get available translation files from base language
|
||||
const availableFiles = getAvailableTranslationFiles();
|
||||
|
||||
// Process each translation file
|
||||
for (const filename of availableFiles) {
|
||||
console.log(`\n 📄 Processing ${filename}...`);
|
||||
|
||||
// Load base and target translation files
|
||||
const baseTranslations = loadTranslationFile(BASE_LANGUAGE, filename);
|
||||
const targetTranslations = loadTranslationFile(language, filename);
|
||||
|
||||
if (Object.keys(baseTranslations).length === 0) {
|
||||
console.warn(
|
||||
`⚠️ Warning: Could not load base ${filename} for ${BASE_LANGUAGE}, skipping...`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find missing keys
|
||||
const missingKeys = findMissingKeys(baseTranslations, targetTranslations);
|
||||
|
||||
if (missingKeys.length === 0) {
|
||||
console.log(` ✅ ${filename}: All translations are up to date!`);
|
||||
processedFiles++;
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
` 📝 Found ${missingKeys.length} missing translation(s) in ${filename}`
|
||||
);
|
||||
totalMissingKeys += missingKeys.length;
|
||||
|
||||
// Create batches for API requests
|
||||
const batches = createTranslationBatches(missingKeys);
|
||||
console.log(` 📦 Processing ${batches.length} batch(es) of translations...`);
|
||||
|
||||
let allTranslations = [];
|
||||
|
||||
// Process each batch
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
const batch = batches[i];
|
||||
console.log(
|
||||
` Batch ${i + 1}/${batches.length} (${batch.length} items)...`
|
||||
);
|
||||
|
||||
try {
|
||||
const translations = await requestTranslations(batch, language);
|
||||
allTranslations.push(...translations);
|
||||
|
||||
// Add a small delay between requests to be respectful to the API
|
||||
if (i < batches.length - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
` ❌ Error processing batch ${i + 1}: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (allTranslations.length === 0) {
|
||||
console.error(` ❌ No translations received for ${filename}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply translations to the target object
|
||||
const updatedTranslations = { ...targetTranslations };
|
||||
let appliedCount = 0;
|
||||
|
||||
for (const translation of allTranslations) {
|
||||
try {
|
||||
// Try to parse as JSON if it looks like an object
|
||||
let translatedValue = translation.translation;
|
||||
if (
|
||||
translatedValue.startsWith('{') ||
|
||||
translatedValue.startsWith('[')
|
||||
) {
|
||||
try {
|
||||
translatedValue = JSON.parse(translatedValue);
|
||||
|
||||
// Check if this should be an array but was translated as a numbered object
|
||||
if (typeof translatedValue === 'object' && translatedValue !== null && !Array.isArray(translatedValue)) {
|
||||
const keys = Object.keys(translatedValue);
|
||||
const isNumberedObject = keys.every((key, index) => key === index.toString());
|
||||
|
||||
if (isNumberedObject && keys.length > 0) {
|
||||
// Convert numbered object back to array
|
||||
translatedValue = keys.map(key => translatedValue[key]);
|
||||
console.log(` 🔧 Fixed numbered object to array for ${translation.key}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Keep as string if JSON parsing fails
|
||||
}
|
||||
}
|
||||
|
||||
setNestedProperty(
|
||||
updatedTranslations,
|
||||
translation.key,
|
||||
translatedValue
|
||||
);
|
||||
appliedCount++;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
` ⚠️ Warning: Could not apply translation for ${translation.key}: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Save the updated translations
|
||||
saveTranslationFile(language, updatedTranslations, filename);
|
||||
|
||||
console.log(
|
||||
` ✅ ${filename}: Applied ${appliedCount}/${missingKeys.length} translations`
|
||||
);
|
||||
|
||||
totalAppliedCount += appliedCount;
|
||||
processedFiles++;
|
||||
}
|
||||
|
||||
if (processedFiles === 0) {
|
||||
console.error(`❌ ${language}: No files could be processed`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (totalMissingKeys === 0) {
|
||||
console.log(`✅ ${language}: All translation files are up to date!`);
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`✅ ${language}: Applied ${totalAppliedCount}/${totalMissingKeys} total translations across ${processedFiles} files`
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main execution function
|
||||
*/
|
||||
async function main() {
|
||||
program
|
||||
.name('sync-translations')
|
||||
.description(
|
||||
'Synchronize translation files with English base and fill missing translations using OpenAI'
|
||||
)
|
||||
.option('--all', 'Update all available languages')
|
||||
.option(
|
||||
'--lang <languages>',
|
||||
'Comma-separated list of language codes to update (e.g., jp,el,de)'
|
||||
)
|
||||
.option(
|
||||
'--dry-run',
|
||||
'Show what would be updated without making changes'
|
||||
)
|
||||
.option(
|
||||
'--verbose',
|
||||
'Show detailed missing translations in dry run mode'
|
||||
)
|
||||
.option('--output <file>', 'Save missing translations to a JSON file')
|
||||
.parse();
|
||||
|
||||
const options = program.opts();
|
||||
|
||||
console.log('🌍 Translation Synchronization Tool\n');
|
||||
console.log(`📁 Locales directory: ${LOCALES_DIR}`);
|
||||
console.log(`🏴 Base language: ${BASE_LANGUAGE}`);
|
||||
|
||||
// Show available translation files
|
||||
const availableFiles = getAvailableTranslationFiles();
|
||||
console.log(`📄 Available translation files: ${availableFiles.join(', ')}`);
|
||||
|
||||
// Determine which languages to process
|
||||
let languagesToProcess = [];
|
||||
|
||||
if (options.all) {
|
||||
languagesToProcess = getAvailableLanguages();
|
||||
console.log(
|
||||
`🔄 Processing all available languages: ${languagesToProcess.join(', ')}`
|
||||
);
|
||||
} else if (options.lang) {
|
||||
languagesToProcess = options.lang.split(',').map((lang) => lang.trim());
|
||||
console.log(
|
||||
`🎯 Processing specified languages: ${languagesToProcess.join(', ')}`
|
||||
);
|
||||
} else {
|
||||
console.log('❓ No languages specified. Use --all or --lang=<codes>');
|
||||
console.log(
|
||||
'\nAvailable languages:',
|
||||
getAvailableLanguages().join(', ')
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (languagesToProcess.length === 0) {
|
||||
console.log('❌ No valid languages to process');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Validate that all specified languages exist
|
||||
const availableLanguages = getAvailableLanguages();
|
||||
const invalidLanguages = languagesToProcess.filter(
|
||||
(lang) => !availableLanguages.includes(lang)
|
||||
);
|
||||
|
||||
if (invalidLanguages.length > 0) {
|
||||
console.error(
|
||||
`❌ Invalid language codes: ${invalidLanguages.join(', ')}`
|
||||
);
|
||||
console.error(`Available languages: ${availableLanguages.join(', ')}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
console.log('\n🧪 DRY RUN MODE - No files will be modified\n');
|
||||
}
|
||||
|
||||
// Process each language
|
||||
let successCount = 0;
|
||||
const allMissingTranslations = {};
|
||||
|
||||
for (const language of languagesToProcess) {
|
||||
try {
|
||||
if (options.dryRun) {
|
||||
// In dry run, just analyze without making API calls
|
||||
console.log(
|
||||
`\n📊 Analyzing ${language} (${LANGUAGE_NAMES[language] || language})...`
|
||||
);
|
||||
|
||||
let totalMissingKeys = 0;
|
||||
let allMissingForLanguage = [];
|
||||
|
||||
// Get available translation files from base language
|
||||
const availableFiles = getAvailableTranslationFiles();
|
||||
|
||||
// Process each translation file
|
||||
for (const filename of availableFiles) {
|
||||
console.log(`\n 📄 Checking ${filename}...`);
|
||||
|
||||
const baseTranslations = loadTranslationFile(BASE_LANGUAGE, filename);
|
||||
const targetTranslations = loadTranslationFile(language, filename);
|
||||
|
||||
if (Object.keys(baseTranslations).length === 0) {
|
||||
console.warn(
|
||||
` ⚠️ Warning: Could not load base ${filename} for ${BASE_LANGUAGE}, skipping...`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const missingKeys = findMissingKeys(
|
||||
baseTranslations,
|
||||
targetTranslations
|
||||
);
|
||||
|
||||
if (missingKeys.length === 0) {
|
||||
console.log(` ✅ ${filename}: All translations are up to date!`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
` 📝 Found ${missingKeys.length} missing translation(s) in ${filename}`
|
||||
);
|
||||
|
||||
totalMissingKeys += missingKeys.length;
|
||||
|
||||
// Add filename prefix to paths for clarity
|
||||
const prefixedMissingKeys = missingKeys.map(item => ({
|
||||
...item,
|
||||
path: `${filename}:${item.path}`,
|
||||
filename: filename
|
||||
}));
|
||||
|
||||
allMissingForLanguage.push(...prefixedMissingKeys);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\n📊 ${language}: Would update ${totalMissingKeys} missing translation(s) across ${availableFiles.length} files`
|
||||
);
|
||||
|
||||
// Store missing translations for potential output
|
||||
if (totalMissingKeys > 0) {
|
||||
allMissingTranslations[language] = {
|
||||
languageName: LANGUAGE_NAMES[language] || language,
|
||||
missingCount: totalMissingKeys,
|
||||
missing: allMissingForLanguage.map((item) => ({
|
||||
path: item.path,
|
||||
englishValue: item.value,
|
||||
isNested: item.isNested,
|
||||
filename: item.filename,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
if (totalMissingKeys > 0) {
|
||||
if (options.verbose) {
|
||||
console.log(
|
||||
`\n🔍 Detailed missing translations for ${language}:`
|
||||
);
|
||||
console.log('─'.repeat(60));
|
||||
|
||||
allMissingForLanguage.forEach((item, index) => {
|
||||
const displayValue =
|
||||
typeof item.value === 'string'
|
||||
? item.value.length > 100
|
||||
? item.value.substring(0, 97) + '...'
|
||||
: item.value
|
||||
: JSON.stringify(item.value);
|
||||
|
||||
console.log(
|
||||
`${(index + 1).toString().padStart(3)}. ${item.path}`
|
||||
);
|
||||
console.log(` 📝 EN: "${displayValue}"`);
|
||||
console.log(
|
||||
` 🎯 ${LANGUAGE_NAMES[language] || language}: [MISSING]`
|
||||
);
|
||||
|
||||
if (index < allMissingForLanguage.length - 1) {
|
||||
console.log('');
|
||||
}
|
||||
});
|
||||
console.log('─'.repeat(60));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Check API key for actual processing
|
||||
if (!OPENAI_API_KEY) {
|
||||
console.error('❌ Error: OPENAI_API_KEY environment variable is required for actual translation updates');
|
||||
console.error('Please set it with: export OPENAI_API_KEY="your-api-key"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const success = await processLanguage(language);
|
||||
if (success) successCount++;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${language}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
console.log(
|
||||
`\n🎉 Completed! Successfully updated ${successCount}/${languagesToProcess.length} languages`
|
||||
);
|
||||
} else {
|
||||
// Show summary for dry run
|
||||
const totalMissing = Object.values(allMissingTranslations).reduce((sum, lang) => sum + lang.missingCount, 0);
|
||||
if (totalMissing > 0) {
|
||||
console.log(`\n📋 Summary: ${totalMissing} total missing translations across ${Object.keys(allMissingTranslations).length} languages`);
|
||||
}
|
||||
|
||||
// Handle output file for dry run
|
||||
if (options.output && Object.keys(allMissingTranslations).length > 0) {
|
||||
const outputData = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseLanguage: BASE_LANGUAGE,
|
||||
languages: allMissingTranslations,
|
||||
summary: {
|
||||
totalLanguages: Object.keys(allMissingTranslations).length,
|
||||
totalMissingTranslations: totalMissing,
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
fs.writeFileSync(
|
||||
options.output,
|
||||
JSON.stringify(outputData, null, 2)
|
||||
);
|
||||
console.log(`💾 Missing translations saved to: ${options.output}`);
|
||||
} catch (error) {
|
||||
console.error(`❌ Error saving output file: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle uncaught errors
|
||||
process.on('unhandledRejection', (error) => {
|
||||
console.error('❌ Unhandled error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Run the script
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error('❌ Script failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
loadTranslationFile,
|
||||
findMissingKeys,
|
||||
createTranslationBatches,
|
||||
processLanguage,
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue