translate_to_tamil.py
translate_to_tamil.py
#!/usr/bin/env python3
"""Translate the English book text in this repository into Tamil.
Adapted from ``translate_repo.py`` (the Chinese->English pass). Differences:
* Direction - English -> Tamil (தமிழ்).
* Non-destructive - the English sources are never modified. Each source
``foo.md`` is written to a sibling ``foo.ta.md``.
* Scoped by default - only the book Markdown (``book/*.md``) is translated
unless ``--include`` globs are given. Generated ``*.ta.md`` files are always
excluded so re-runs never translate a translation.
Everything else mirrors the original: structure-preserving (code, identifiers,
Markdown/LaTeX/JSON syntax, URLs, paths and whitespace are copied verbatim),
resumable (a JSON state file records the source hash of each finished file), and
concurrent (a thread pool issues API calls in parallel). The API key is read
from the environment and is never written to disk.
Usage
-----
export DEEPSEEK_API_KEY=sk-...
python translate_to_tamil.py --list # show what would be translated
python translate_to_tamil.py --limit 1 # translate one file first
python translate_to_tamil.py # translate the whole book
Run ``python translate_to_tamil.py --help`` for all options.
"""
from __future__ import annotations
import argparse
import concurrent.futures as cf
import fnmatch
import hashlib
import json
import os
import re
import subprocess
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING: # import only for type hints; not required for --list/--dry-run
from openai import OpenAI
# --------------------------------------------------------------------------- #
# Constants
# --------------------------------------------------------------------------- #
# Presence of at least one ASCII letter marks a fragment as worth translating.
# Fragments that are pure whitespace/numbers/symbols are copied verbatim.
LETTER_RE = re.compile(r"[A-Za-z]")
# Marks already-generated Tamil outputs like ``chapter1.ta.md`` so we never
# translate a translation.
TAMIL_OUT_RE = re.compile(r"\.ta\.[^.]+$")
# File extensions we never translate (binary / data / vendored artefacts).
SKIP_EXTS = {
".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a", ".mp3", ".wav", ".flac",
".ogg", ".aac", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp",
".tiff", ".psd",
".pdf", ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar",
".woff", ".woff2", ".ttf", ".otf", ".eot",
".onnx", ".parquet", ".tiktoken", ".bin", ".pt", ".pth", ".safetensors",
".npy", ".npz", ".h5", ".hdf5", ".pkl", ".pickle", ".model", ".arrow",
".pyc", ".pyo", ".so", ".dylib", ".dll", ".class", ".o", ".a", ".lock",
".min.js", ".min.css", ".map",
}
SKIP_NAMES = {
"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock",
"Cargo.lock", "uv.lock", "composer.lock", "Gemfile.lock",
}
SKIP_DIRS = {
".git", "node_modules", "__pycache__", ".venv", "venv", "env", ".mypy_cache",
".pytest_cache", ".ruff_cache", "dist", "build", ".next", ".turbo", ".cache",
".idea", ".vscode", "site-packages",
}
TYPE_HINTS = {
".md": "Markdown documentation",
".mdx": "MDX documentation (Markdown + JSX)",
".py": "Python source code",
".ts": "TypeScript source code",
".tsx": "TypeScript React (TSX) source code",
".js": "JavaScript source code",
".jsx": "JavaScript React (JSX) source code",
".svg": "SVG vector image (translate ONLY the visible text inside <text>/<tspan>/<title> and human-facing attributes)",
".json": "JSON data (translate ONLY human-readable string values; keep object keys and identifier-like values unchanged)",
".jsonl": "JSON Lines data (translate ONLY human-readable string values; keep keys unchanged)",
".yaml": "YAML config (translate ONLY comments and human-readable string values; keep keys unchanged)",
".yml": "YAML config (translate ONLY comments and human-readable string values; keep keys unchanged)",
".toml": "TOML config (translate ONLY comments and human-readable string values; keep keys unchanged)",
".ini": "INI config (translate ONLY comments and human-readable values; keep keys unchanged)",
".html": "HTML (translate ONLY visible text and human-facing attributes such as title/alt/placeholder)",
".htm": "HTML (translate ONLY visible text and human-facing attributes such as title/alt/placeholder)",
".xml": "XML (translate ONLY visible text content and human-facing attribute values)",
".sql": "SQL (translate ONLY comments and human-facing string literals; keep identifiers)",
".sh": "Shell script (translate ONLY comments and human-facing echoed strings)",
".tex": "LaTeX (translate prose; keep commands, math, labels and references)",
".lua": "Lua source code",
".css": "CSS stylesheet (translate ONLY comments and content strings)",
".less": "LESS stylesheet (translate ONLY comments and content strings)",
".txt": "plain text",
}
SYSTEM_PROMPT = """\
You are a professional book-localization engine. You translate English technical \
prose into clear, natural, fluent Tamil (தமிழ்) for a professional \
software-engineering book about AI agents.
You are given the contents (or a fragment) of a source file. The file type is: {filetype}.
TRANSLATE
- All English natural-language text: prose, headings, list items, table cells,
block quotes, figure/image captions and alt text, footnote bodies, and
human-facing string literals or comments inside code blocks.
- Produce fluent, idiomatic Tamil (meaning-for-meaning, not word-for-word),
written for a technical reader in a precise, professional tone.
- CRITICAL — NEVER transliterate English technical terms, concept keywords, or
proper nouns into Tamil script. Keep them VERBATIM in Latin (English) script,
exactly as written in the source. Transliteration (spelling an English word
using Tamil letters) is forbidden. This applies to, among others: agent,
model, context, tool/tools, prompt, token, embedding, attention, transformer,
workflow, harness, pipeline, fine-tuning, reinforcement learning, and all
abbreviations / product / company names (LLM, RAG, API, KV Cache, SFT, RL,
ReAct, GPT, Claude, Cursor, OpenAI, ...). Translate ONLY the genuine Tamil
grammatical/connective words around them.
Worked example: "Model as Agent" MUST become "Model ஆக Agent" — translate
only "as" → "ஆக" and keep "Model" and "Agent" in English. It is WRONG to
write "மாடல் ஆக ஏஜென்ட்" or "மாடல்" or "ஏஜென்ட்".
- Do not invent obscure Tamil neologisms for technical concepts; if the natural
Tamil word is uncommon or ambiguous, keep the English term instead.
- Keep terminology consistent throughout the file.
DO NOT CHANGE (copy verbatim, byte for byte)
- Code structure, syntax, indentation and ALL whitespace. Never reflow or reindent.
- Programming identifiers: variable/function/class/module names, keywords, decorators.
- Import paths, file paths, URLs, environment-variable names, CLI flags, hashes, IDs.
- Markdown/HTML/XML/JSON/YAML structure and syntax; translate ONLY the human text.
Keep heading markers (#), list markers, table pipes, emphasis markers, and the
link/image syntax intact (translate the visible text or caption, but keep the
URL/path). Keep footnote reference labels such as [^name] unchanged (translate
only the note body). Keep Markdown attribute blocks like {{.unnumbered}} as-is.
- Code fences and the code inside them, EXCEPT translate comments and
human-facing strings within that code. Keep all code identifiers in English.
- LaTeX and math ($...$, $$...$$), numbers, dates, and content already written in
a non-English script.
OUTPUT RULES (critical)
- Output ONLY the translated file content. No preamble, no explanation, no notes.
- Do NOT wrap the whole output in a Markdown code fence.
- Preserve the exact leading and trailing whitespace, including the final newline.
- The output must be structurally identical to the input, with English prose replaced by Tamil.\
"""
# --------------------------------------------------------------------------- #
# Helpers
# --------------------------------------------------------------------------- #
def sha256(text: str) -> str:
return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()
def is_secret_file(name: str) -> bool:
if name == ".env":
return True
if name.startswith(".env.") and not name.endswith(
(".example", ".template", ".sample")
):
return True
return False
def has_source_text(text: str) -> bool:
"""True if the fragment contains English text worth translating."""
return LETTER_RE.search(text) is not None
def type_hint(path: Path) -> str:
return TYPE_HINTS.get(path.suffix.lower(), "plain text / config")
def tamil_out_path(path: Path) -> Path:
"""``book/afterword.md`` -> ``book/afterword.ta.md``."""
return path.parent / f"{path.stem}.ta{path.suffix}"
def read_text(path: Path) -> Optional[str]:
try:
data = path.read_bytes()
except OSError:
return None
if b"\x00" in data:
return None
try:
return data.decode("utf-8")
except UnicodeDecodeError:
return None
def split_budget(text: str, budget: int) -> list[str]:
"""Split text into fragments <= ``budget`` chars on line boundaries."""
lines = text.splitlines(keepends=True)
parts: list[str] = []
cur: list[str] = []
cur_len = 0
for ln in lines:
if cur and cur_len + len(ln) > budget:
parts.append("".join(cur))
cur, cur_len = [], 0
cur.append(ln)
cur_len += len(ln)
if cur:
parts.append("".join(cur))
return parts or [text]
# A fenced code block opens with >=3 backticks or tildes (optionally indented)
# and closes with a line of only that fence character (>= the opening length).
FENCE_OPEN_RE = re.compile(r"^[ \t]*(`{3,}|~{3,})")
def split_prose_code(text: str) -> list[tuple[str, str]]:
"""Split markdown into ('prose'|'code', segment) parts on fenced code blocks.
Concatenating the segments reproduces ``text`` exactly. Fenced code blocks
(the fence lines and everything between them) are isolated as 'code' so the
caller can pass them through untranslated -- translating inside code fences
is what corrupts fence structure (the model reflows fences, bolds lines,
merges markers), so we never send code to the model.
"""
lines = text.splitlines(keepends=True)
out: list[tuple[str, str]] = []
prose: list[str] = []
i, n = 0, len(lines)
while i < n:
m = FENCE_OPEN_RE.match(lines[i])
if m:
fence = m.group(1)
fchar, flen = fence[0], len(fence)
if prose:
out.append(("prose", "".join(prose)))
prose = []
code = [lines[i]]
i += 1
while i < n:
code.append(lines[i])
s = lines[i].strip()
if s and all(c == fchar for c in s) and len(s) >= flen:
i += 1
break
i += 1
out.append(("code", "".join(code)))
else:
prose.append(lines[i])
i += 1
if prose:
out.append(("prose", "".join(prose)))
return out
def strip_wrapping_fence(src: str, out: str) -> str:
"""Remove a Markdown code fence the model may have wrapped the output in."""
if src.lstrip().startswith("```"):
return out
stripped = out.strip()
if stripped.startswith("```") and stripped.endswith("```") and len(stripped) > 6:
nl = stripped.find("\n")
if nl != -1:
inner = stripped[nl + 1:]
last = inner.rfind("```")
if last != -1:
return inner[:last].rstrip("\n")
return out
def match_globs(rel: str, globs: list[str]) -> bool:
return any(fnmatch.fnmatch(rel, g) for g in globs)
# --------------------------------------------------------------------------- #
# Translator
# --------------------------------------------------------------------------- #
@dataclass
class Translator:
client: OpenAI
model: str
temperature: float = 0.2
max_output_tokens: int = 8192
max_chars: int = 3000
max_retries: int = 5
min_chunk: int = 400
def _call(self, system: str, user: str) -> tuple[str, str]:
last_err: Optional[Exception] = None
for attempt in range(self.max_retries):
try:
resp = self.client.chat.completions.create(
model=self.model,
temperature=self.temperature,
max_tokens=self.max_output_tokens,
messages=[
{"role": "system", "content": system},
{"role": "user", "content": user},
],
)
choice = resp.choices[0]
content = choice.message.content or ""
if not content.strip():
raise RuntimeError("empty response from model")
return content, (choice.finish_reason or "stop")
except Exception as exc: # noqa: BLE001 - surface after retries
last_err = exc
time.sleep(min(2 ** attempt, 30))
raise RuntimeError(f"API call failed after {self.max_retries} retries: {last_err}")
def _fragment(self, text: str, filetype: str, depth: int = 0) -> str:
if not has_source_text(text):
return text
system = SYSTEM_PROMPT.format(filetype=filetype)
content, finish = self._call(system, text)
if finish == "length" and depth < 6 and len(text) > self.min_chunk:
parts = split_budget(text, max(len(text) // 2, self.min_chunk))
if len(parts) > 1:
return "".join(self._fragment(p, filetype, depth + 1) for p in parts)
mid = len(text) // 2
return self._fragment(text[:mid], filetype, depth + 1) + self._fragment(
text[mid:], filetype, depth + 1
)
return strip_wrapping_fence(text, content)
def translate(self, text: str, filetype: str) -> tuple[str, int]:
"""Translate a file, passing fenced code blocks through verbatim.
Only prose segments are sent to the model; fenced code blocks (fences +
contents) are copied unchanged, so the Tamil output keeps byte-identical
code-fence structure to the English source. Returns (new_text, chunks).
The model often drops the leading/trailing newlines of a fragment. If
left uncorrected, the trailing newline before a code block is lost and
the block's opening ``` glues onto the previous prose line, breaking the
fence. So for every translated fragment we restore the EXACT leading and
trailing newline count of its source, guaranteeing fences and block
boundaries always start on their own line.
"""
out_parts: list[str] = []
chunks = 0
for kind, seg in split_prose_code(text):
if kind == "code" or not has_source_text(seg):
out_parts.append(seg) # code fences / blank gaps: verbatim
continue
parts = [seg] if len(seg) <= self.max_chars else split_budget(seg, self.max_chars)
for p in parts:
if not has_source_text(p):
out_parts.append(p)
continue
tp = self._fragment(p, filetype)
lead = len(p) - len(p.lstrip("\n"))
trail = len(p) - len(p.rstrip("\n"))
out_parts.append(("\n" * lead) + tp.strip("\n") + ("\n" * trail))
chunks += 1
out = "".join(out_parts)
if text.endswith("\n") and not out.endswith("\n"):
out += "\n"
elif not text.endswith("\n") and out.endswith("\n"):
out = out.rstrip("\n")
return out, chunks
# --------------------------------------------------------------------------- #
# State
# --------------------------------------------------------------------------- #
class State:
def __init__(self, path: Path):
self.path = path
self.lock = threading.Lock()
self._dirty = 0
self.data: dict = {"version": 1, "files": {}}
if path.exists():
try:
self.data = json.loads(path.read_text("utf-8"))
self.data.setdefault("files", {})
except (OSError, json.JSONDecodeError):
pass
def source_hash(self, rel: str) -> Optional[str]:
entry = self.data["files"].get(rel)
return entry.get("source_sha") if entry else None
def record(self, rel: str, **fields) -> None:
with self.lock:
self.data["files"][rel] = {"ts": int(time.time()), **fields}
self._dirty += 1
if self._dirty >= 8:
self._flush_locked()
def flush(self) -> None:
with self.lock:
self._flush_locked()
def _flush_locked(self) -> None:
self._dirty = 0
tmp = self.path.with_suffix(self.path.suffix + ".tmp")
tmp.write_text(json.dumps(self.data, ensure_ascii=False, indent=2), "utf-8")
tmp.replace(self.path)
# --------------------------------------------------------------------------- #
# File discovery
# --------------------------------------------------------------------------- #
def git_files(root: Path) -> Optional[list[Path]]:
try:
res = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
)
except (OSError, subprocess.CalledProcessError):
return None
return [root / n.decode("utf-8") for n in res.stdout.split(b"\x00") if n]
def walk_files(root: Path) -> list[Path]:
out: list[Path] = []
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
for f in filenames:
out.append(Path(dirpath) / f)
return out
def is_candidate(path: Path, self_path: Path, state_path: Path) -> bool:
if not path.is_file():
return False
if path in (self_path, state_path):
return False
name = path.name
if name in SKIP_NAMES or is_secret_file(name):
return False
if TAMIL_OUT_RE.search(name): # never translate a translation
return False
if path.suffix.lower() in SKIP_EXTS or name.endswith((".min.js", ".min.css")):
return False
if any(part in SKIP_DIRS or part.startswith(".venv") for part in path.parts):
return False
return True
# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def build_client(args) -> "OpenAI":
try:
from openai import OpenAI
except ImportError: # pragma: no cover
sys.exit("The 'openai' package is required to translate. "
"Install with: pip install openai")
api_key = (
args.api_key
or os.environ.get("TRANSLATE_API_KEY")
or os.environ.get("DEEPSEEK_API_KEY")
or os.environ.get("OPENAI_API_KEY")
)
if not api_key:
sys.exit(
"No API key found. Set DEEPSEEK_API_KEY (or TRANSLATE_API_KEY / "
"OPENAI_API_KEY) in the environment, or pass --api-key."
)
base_url = (
args.base_url
or os.environ.get("TRANSLATE_BASE_URL")
or os.environ.get("DEEPSEEK_BASE_URL")
or os.environ.get("OPENAI_BASE_URL")
or "https://api.deepseek.com"
)
return OpenAI(api_key=api_key, base_url=base_url, timeout=180.0, max_retries=0)
def parse_args(argv=None):
p = argparse.ArgumentParser(
description="Translate the English book text into Tamil (writes *.ta.md).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p.add_argument("--root", type=Path, default=Path(__file__).resolve().parent,
help="Repository root to translate.")
p.add_argument("--model", default=os.environ.get("TRANSLATE_MODEL")
or os.environ.get("DEEPSEEK_MODEL")
or os.environ.get("OPENAI_MODEL") or "deepseek-chat",
help="Chat model name.")
p.add_argument("--base-url", default=None, help="OpenAI-compatible base URL.")
p.add_argument("--api-key", default=None,
help="API key (prefer the environment variable instead).")
p.add_argument("--workers", type=int, default=6, help="Parallel API workers.")
p.add_argument("--temperature", type=float, default=0.2)
p.add_argument("--max-chars", type=int, default=3000,
help="Max characters per API request before splitting.")
p.add_argument("--max-output-tokens", type=int, default=8192)
p.add_argument("--max-file-bytes", type=int, default=2_000_000,
help="Skip files larger than this.")
p.add_argument("--include", action="append", default=[],
help="Only translate paths matching this glob (repeatable). "
"Default: book/*.md")
p.add_argument("--exclude", action="append", default=[],
help="Skip paths matching this glob (repeatable).")
p.add_argument("--limit", type=int, default=0, help="Translate at most N files.")
p.add_argument("--all-files", action="store_true",
help="Walk the filesystem instead of using 'git ls-files'.")
p.add_argument("--force", action="store_true",
help="Re-translate even files already recorded as done.")
p.add_argument("--state-file", type=Path, default=None,
help="Path to the resume-state JSON file.")
p.add_argument("--list", action="store_true",
help="List candidate files and exit (no API calls).")
p.add_argument("--dry-run", action="store_true", help="Alias for --list.")
return p.parse_args(argv)
def collect_candidates(args, self_path: Path, state_path: Path):
root: Path = args.root.resolve()
include = args.include or ["book/*.md"]
exclude = list(args.exclude) + ["*.ta.md"]
raw = None if args.all_files else git_files(root)
if raw is None:
raw = walk_files(root)
candidates = []
for path in raw:
path = path.resolve()
if not is_candidate(path, self_path, state_path):
continue
rel = os.path.relpath(path, root)
if not match_globs(rel, include):
continue
if match_globs(rel, exclude):
continue
try:
if path.stat().st_size > args.max_file_bytes:
continue
except OSError:
continue
text = read_text(path)
if text is None or not has_source_text(text):
continue
candidates.append((path, rel, text))
return root, candidates
def main(argv=None) -> int:
args = parse_args(argv)
self_path = Path(__file__).resolve()
root = args.root.resolve()
state_path = (args.state_file or (root / ".translate_tamil_state.json")).resolve()
print(f"Scanning {root} ...", flush=True)
root, candidates = collect_candidates(args, self_path, state_path)
state = State(state_path)
if not args.force:
pending = []
for path, rel, text in candidates:
out_path = tamil_out_path(path)
if state.source_hash(rel) == sha256(text) and out_path.exists():
continue # already translated by us and unchanged
pending.append((path, rel, text))
else:
pending = candidates
total_bytes = sum(len(t.encode("utf-8")) for _, _, t in pending)
print(f"Candidate source files: {len(candidates)}")
print(f"Pending (not yet done): {len(pending)} "
f"(~{total_bytes/1_000_000:.2f} MB of text)")
if args.limit:
pending = pending[: args.limit]
print(f"Limited to first {len(pending)} file(s).")
if args.list or args.dry_run:
for path, rel, _ in pending:
print(f" {rel} -> {os.path.relpath(tamil_out_path(path), root)}")
print(f"\n[list] {len(pending)} file(s) would be translated. No API calls made.")
return 0
if not pending:
print("Nothing to do - everything is already translated.")
return 0
client = build_client(args)
translator = Translator(
client=client, model=args.model, temperature=args.temperature,
max_output_tokens=args.max_output_tokens, max_chars=args.max_chars,
)
print(f"Model: {args.model} Workers: {args.workers}\n")
counter = {"done": 0, "failed": 0}
clock = threading.Lock()
n = len(pending)
def work(item):
path, rel, text = item
out_path = tamil_out_path(path)
t0 = time.time()
try:
new_text, chunks = translator.translate(text, type_hint(path))
except Exception as exc: # noqa: BLE001
state.record(rel, status="error", error=str(exc)[:300])
with clock:
counter["failed"] += 1
i = counter["done"] + counter["failed"]
print(f"[{i:>4}/{n}] FAIL {rel}\n {exc}", flush=True)
return
out_path.write_text(new_text, "utf-8")
state.record(rel, status="done", out=os.path.relpath(out_path, root),
source_sha=sha256(text), result_sha=sha256(new_text))
with clock:
counter["done"] += 1
i = counter["done"] + counter["failed"]
print(f"[{i:>4}/{n}] OK {rel} "
f"(chunks={chunks}, {time.time()-t0:.1f}s)", flush=True)
try:
with cf.ThreadPoolExecutor(max_workers=args.workers) as pool:
list(pool.map(work, pending))
except KeyboardInterrupt:
print("\nInterrupted - progress saved to state file; re-run to resume.")
finally:
state.flush()
print(f"\nDone. translated={counter['done']} "
f"failed={counter['failed']} state={state_path.name}")
return 1 if counter["failed"] else 0
if __name__ == "__main__":
raise SystemExit(main())