Translate interface strings

"""
Reuse the translations you already paid for: a translation memory.

Rung N0. No model, no service, no key. A string already in the memory gets its
translation back, and the cheapest translation is the one you do not order
twice.

Three answers, and the difference between them matters more than the code.
An exact match ships: the same string, give or take its spacing, invisible
format characters and Unicode composition. A close enough string is an
approximate match — a word added, a capital or an accent changed — and it is
a draft: it comes back with its score and a review flag, never as a finished
translation, because "Polish" and "polish" are not the same word. Anything
else is new text, which this rung has nothing to say about — see the breaking
point in the test.

Interpolation variables are checked apart from the score. A translation whose
variables do not match the source is a broken interface, however high it
scores, so it is flagged even on an exact hit. An ICU plural or select message
is checked on its argument name and type, not on the branches inside it.

A string longer than MAX_FUZZY_CHARACTERS gets exact matches only: scoring two
long texts costs of the order of the product of their lengths, for every entry
of the memory.
"""

from __future__ import annotations

import re
import unicodedata
from difflib import SequenceMatcher

# The variable forms an interface uses: {{count}} (i18next), {count}, {}, the
# head of an ICU argument such as {count, plural, ...}, %s, %d, %(count)s, the
# numbered %1$s of Android, and the %@, %1$@ and %ld of iOS.
PLACEHOLDER = re.compile(
    r"\{\{\s*[A-Za-z0-9_.]+\s*\}\}|\{[A-Za-z0-9_]*\}|\{\s*[A-Za-z0-9_]+\s*,\s*[A-Za-z]+"
    r"|%(?:\([A-Za-z0-9_]+\)|\d+\$)?l{0,2}[sd@]"
)

MAX_FUZZY_CHARACTERS = 500


def placeholders(text: str) -> list[str]:
    """The interpolation variables of a string, sorted so order does not count."""
    return sorted(PLACEHOLDER.findall(text))


def exact_key(text: str) -> str:
    """Fold only what cannot change the words: composition, format characters, spacing."""
    composed = unicodedata.normalize("NFC", text)
    visible = "".join(c for c in composed if unicodedata.category(c) != "Cf")
    return " ".join(visible.split())


def normalise(text: str) -> str:
    """Fold case, accents and spacing for the score. A match on this alone is reviewed."""
    stripped = unicodedata.normalize("NFKD", exact_key(text))
    without_accents = "".join(c for c in stripped if not unicodedata.category(c).startswith("M"))
    return " ".join(without_accents.lower().split())


def index(memory: dict[str, str]) -> dict[str, tuple[str, str]]:
    """
    The memory keyed by what an exact match compares.

    Build it once for a whole file and pass it to every `lookup`: most of an
    interface does not change from one version to the next, so most strings
    are found here, and scoring is what costs.
    """
    return {exact_key(known): (known, target) for known, target in memory.items()}


def lookup(source: str, memory: dict[str, str], threshold: float = 0.75, exact=None) -> dict:
    """
    Look a string up in a memory of `source: target` pairs already translated.

    Returns the status (`exact`, `fuzzy` or `none`), the target when there is
    one, the score, and whether a human has to look at it.

    The exact match is settled first, by lookup in a dictionary, and only a
    string that is not in the memory is scored against every entry of it.
    Scoring a string that was already there is the expensive way to learn
    nothing: `exact` is that dictionary, built here when the caller does not
    hand one in.
    """
    key, folded = exact_key(source), normalise(source)
    found = (index(memory) if exact is None else exact).get(key)
    if found is not None:
        return _decide("exact", source, found[0], found[1], 1.0)

    best_source, best_target, best_score = None, None, 0.0
    for known, target in memory.items():
        candidate = normalise(known)
        if max(len(folded), len(candidate)) > MAX_FUZZY_CHARACTERS:
            continue
        # autojunk=False so a long string scores exactly like the JavaScript
        # version of this snippet, which has no such heuristic.
        score = SequenceMatcher(None, folded, candidate, autojunk=False).ratio()
        if score > best_score:
            best_source, best_target, best_score = known, target, score
    if best_score >= threshold:
        return _decide("fuzzy", source, best_source, best_target, best_score)
    return {"status": "none", "target": None, "score": best_score,
            "matched": None, "review": False, "warnings": []}


def _decide(status: str, source: str, matched: str, target: str, score: float) -> dict:
    """Assemble the answer, and never let a fuzzy hit pass as a finished one."""
    warnings = []
    if placeholders(target) != placeholders(source):
        warnings.append("interpolation variables differ from the source string")
    return {"status": status, "target": target, "score": score, "matched": matched,
            "review": status == "fuzzy" or bool(warnings), "warnings": warnings}

/**
 * Reuse the translations you already paid for: a translation memory.
 *
 * Rung N0. No model, no service, no key. A string already in the memory gets
 * its translation back, and the cheapest translation is the one you do not
 * order twice.
 *
 * Three answers, and the difference between them matters more than the code.
 * An exact match ships: the same string, give or take its spacing, invisible
 * format characters and Unicode composition. A close enough string is an
 * approximate match — a word added, a capital or an accent changed — and it is
 * a draft: it comes back with its score and a review flag, never as a finished
 * translation, because "Polish" and "polish" are not the same word. Anything
 * else is new text, which this rung has nothing to say about — see the
 * breaking point in the test.
 *
 * Interpolation variables are checked apart from the score. A translation
 * whose variables do not match the source is a broken interface, however high
 * it scores, so it is flagged even on an exact hit. An ICU plural or select
 * message is checked on its argument name and type, not on the branches inside
 * it.
 *
 * A string longer than MAX_FUZZY_CHARACTERS gets exact matches only: scoring
 * two long texts costs of the order of the product of their lengths, for every
 * entry of the memory.
 */

// The variable forms an interface uses: {{count}} (i18next), {count}, {}, the
// head of an ICU argument such as {count, plural, ...}, %s, %d, %(count)s, the
// numbered %1$s of Android, and the %@, %1$@ and %ld of iOS.
const PLACEHOLDER =
  /\{\{\s*[A-Za-z0-9_.]+\s*\}\}|\{[A-Za-z0-9_]*\}|\{\s*[A-Za-z0-9_]+\s*,\s*[A-Za-z]+|%(?:\([A-Za-z0-9_]+\)|\d+\$)?l{0,2}[sd@]/g;

// Python's str.split() also splits on these; JavaScript's \s does not.
const SPACES = /[\s\x1c-\x1f\x85]+/;

export const MAX_FUZZY_CHARACTERS = 500;

/** The interpolation variables of a string, sorted so order does not count. */
export function placeholders(text) {
  return [...text.matchAll(PLACEHOLDER)].map((m) => m[0]).sort();
}

/** Fold only what cannot change the words: composition, format characters, spacing. */
export function exactKey(text) {
  return text.normalize('NFC').replace(/\p{Cf}/gu, '').split(SPACES).filter(Boolean).join(' ');
}

/** Fold case, accents and spacing for the score. A match on this alone is reviewed. */
export function normalise(text) {
  return exactKey(text).normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase().split(SPACES).filter(Boolean).join(' ');
}

/**
 * The memory keyed by what an exact match compares.
 *
 * Build it once for a whole file and pass it to every `lookup`: most of an
 * interface does not change from one version to the next, so most strings are
 * found here, and scoring is what costs.
 */
export function index(memory) {
  return new Map(Object.entries(memory).map(([known, target]) => [exactKey(known), [known, target]]));
}

/**
 * Look a string up in a memory of `source: target` pairs already translated.
 *
 * Returns the status (`exact`, `fuzzy` or `none`), the target when there is
 * one, the score, and whether a human has to look at it.
 *
 * The exact match is settled first, by lookup in a map, and only a string that
 * is not in the memory is scored against every entry of it. Scoring a string
 * that was already there is the expensive way to learn nothing: `exact` is that
 * map, built here when the caller does not hand one in.
 */
export function lookup(source, memory, threshold = 0.75, exact = null) {
  const key = exactKey(source);
  const folded = normalise(source);
  const found = (exact ?? index(memory)).get(key);
  if (found) return decide('exact', source, found[0], found[1], 1);

  let best = { source: null, target: null, score: 0 };
  for (const [known, target] of Object.entries(memory)) {
    const candidate = normalise(known);
    // Lengths in code points, as in Python.
    if (Math.max([...folded].length, [...candidate].length) > MAX_FUZZY_CHARACTERS) continue;
    const score = ratio(folded, candidate);
    if (score > best.score) best = { source: known, target, score };
  }
  if (best.score >= threshold) return decide('fuzzy', source, best.source, best.target, best.score);
  return { status: 'none', target: null, score: best.score, matched: null, review: false, warnings: [] };
}

/** Assemble the answer, and never let a fuzzy hit pass as a finished one. */
function decide(status, source, matched, target, score) {
  const warnings = [];
  if (String(placeholders(target)) !== String(placeholders(source))) {
    warnings.push('interpolation variables differ from the source string');
  }
  return {
    status, target, score, matched,
    review: status === 'fuzzy' || warnings.length > 0,
    warnings,
  };
}

/**
 * Ratcliff-Obershelp similarity, the algorithm behind Python's
 * difflib.SequenceMatcher.ratio: twice the number of matching characters over
 * the total length. Ported here so both versions of this snippet return the
 * very same score on the very same pair.
 */
export function ratio(a, b) {
  const left = [...a];
  const right = [...b];
  const total = left.length + right.length;
  if (total === 0) return 1;
  return (2 * matchedCount(left, right, 0, left.length, 0, right.length)) / total;
}

/** Longest common block, then the same search left and right of it. */
function matchedCount(a, b, alo, ahi, blo, bhi) {
  let bestI = alo, bestJ = blo, bestSize = 0;
  let lengths = new Map();
  for (let i = alo; i < ahi; i += 1) {
    const next = new Map();
    for (let j = blo; j < bhi; j += 1) {
      if (a[i] !== b[j]) continue;
      const size = (lengths.get(j - 1) ?? 0) + 1;
      next.set(j, size);
      // Strictly greater: the earliest block wins a tie, as in difflib.
      if (size > bestSize) [bestI, bestJ, bestSize] = [i - size + 1, j - size + 1, size];
    }
    lengths = next;
  }
  if (bestSize === 0) return 0;
  return bestSize
    + matchedCount(a, b, alo, bestI, blo, bestJ)
    + matchedCount(a, b, bestI + bestSize, ahi, bestJ + bestSize, bhi);
}

"""
Translate by asking a general-purpose model, with the interface context.

Rung N3. What this rung buys over a translation model is the context: a
translator model gets a string and nothing else, while a general-purpose model
can be told that `Save` is the label of a button and not the verb in a
sentence, that the interface is addressed to a customer rather than an
administrator, and that there is no room for a full sentence.

What it costs is everything around the call: a key, a provider that answers
prose when JSON was asked for, retries, a cap on the input, and the same
variable check as the rung below, because being asked to keep `{count}`
verbatim is not the same as doing it.
"""

from __future__ import annotations

import json
import re

PROMPT = (
    "Translate the user interface string below into {language}.\n"
    "Where it appears in the interface: {context}\n"
    "Keep these interpolation variables exactly as written: {variables}\n"
    "Keep the length of an interface label, not of a sentence.\n"
    'Answer with JSON only: {{"translation": "..."}}\n\n'
    "String:\n{source}"
)

# An interface string that no longer fits on one screen is not an interface
# string. Refusing it here, and a context as long, is a cost control.
MAX_CHARACTERS = 2000

# The variable forms an interface uses: {{count}} (i18next), {count}, {}, the
# head of an ICU argument such as {count, plural, ...}, %s, %d, %(count)s, the
# numbered %1$s of Android, and the %@, %1$@ and %ld of iOS.
PLACEHOLDER = re.compile(
    r"\{\{\s*[A-Za-z0-9_.]+\s*\}\}|\{[A-Za-z0-9_]*\}|\{\s*[A-Za-z0-9_]+\s*,\s*[A-Za-z]+"
    r"|%(?:\([A-Za-z0-9_]+\)|\d+\$)?l{0,2}[sd@]"
)

# The provider named here is an example, not a recommendation: the reasoning
# holds for any general-purpose model API, and the client is swappable. Pass
# any object with a `complete(prompt=..., temperature=...)` method.
MODEL = "gpt-4.1-mini"  # an example id: check the parameters your model accepts


class ProviderClient:
    """The one call this snippet makes, on top of the provider's SDK."""

    def __init__(self, sdk=None, model: str = MODEL):
        if sdk is None:  # pragma: no cover - needs a key and a network
            from openai import OpenAI

            sdk = OpenAI()
        self.sdk, self.model = sdk, model

    def complete(self, *, prompt: str, temperature: float) -> str:
        response = self.sdk.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            temperature=temperature,
        )
        return response.choices[0].message.content


class TranslationUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def placeholders(text: str) -> list[str]:
    """The interpolation variables, sorted so a moved one still matches."""
    return sorted(PLACEHOLDER.findall(text))


def translate(source: str, language: str, *, context: str = "",
              client=None, attempts: int = 3) -> dict:
    """
    Translate one interface string, with what the model needs to know about it.

    `client` is injected so this function can be tested without a network
    call. In production it defaults to a real provider client.
    """
    if len(source) > MAX_CHARACTERS or len(context) > MAX_CHARACTERS:
        raise ValueError(f"string or context longer than {MAX_CHARACTERS} characters")
    if not source.strip():  # nothing to translate is not worth a call
        return {"target": source, "review": False, "warnings": []}
    client = client or ProviderClient()

    variables = placeholders(source)
    prompt = PROMPT.format(
        language=language,
        context=context or "not given",
        variables=" ".join(variables) or "none",
        source=source,
    )
    target = _ask(client, prompt, attempts)

    warnings = []
    if placeholders(target) != variables:
        warnings.append("the model did not keep the interpolation variables")
    if any("," in variable for variable in variables):
        warnings.append("ICU message: check its branches by hand")
    return {"target": target, "review": bool(warnings), "warnings": warnings}


def _ask(client, prompt: str, attempts: int) -> str:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # The lowest temperature: the SDK documents lower values as more
            # focused and deterministic.
            answer = client.complete(prompt=prompt, temperature=0)
            if not isinstance(answer, str):  # a refusal comes back as no content
                last_error = ValueError("the model answered no text")
                continue
            text = answer.strip()
            # A JSON answer wrapped whole in one code fence is read; nothing else is.
            if text.startswith("```") and text.endswith("```") and text.count("```") == 2:
                text = text[3:-3].removeprefix("json")
            parsed = json.loads(text)
            target = parsed.get("translation") if isinstance(parsed, dict) else None
            if isinstance(target, str) and target.strip():
                return target.strip()
            last_error = ValueError("the model answered without a translation")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise TranslationUnavailable(str(last_error))

/**
 * Translate by asking a general-purpose model, with the interface context.
 *
 * Rung N3. What this rung buys over a translation model is the context: a
 * translator model gets a string and nothing else, while a general-purpose
 * model can be told that `Save` is the label of a button and not the verb in
 * a sentence, that the interface is addressed to a customer rather than an
 * administrator, and that there is no room for a full sentence.
 *
 * What it costs is everything around the call: a key, a provider that answers
 * prose when JSON was asked for, retries, a cap on the input, and the same
 * variable check as the rung below, because being asked to keep `{count}`
 * verbatim is not the same as doing it.
 */

const PROMPT = [
  'Translate the user interface string below into {language}.',
  'Where it appears in the interface: {context}',
  'Keep these interpolation variables exactly as written: {variables}',
  'Keep the length of an interface label, not of a sentence.',
  'Answer with JSON only: {"translation": "..."}',
  '',
  'String:',
  '{source}',
].join('\n');

// An interface string that no longer fits on one screen is not an interface
// string. Refusing it here, and a context as long, is a cost control.
export const MAX_CHARACTERS = 2000;

// The variable forms an interface uses: {{count}} (i18next), {count}, {}, the
// head of an ICU argument such as {count, plural, ...}, %s, %d, %(count)s, the
// numbered %1$s of Android, and the %@, %1$@ and %ld of iOS.
const PLACEHOLDER =
  /\{\{\s*[A-Za-z0-9_.]+\s*\}\}|\{[A-Za-z0-9_]*\}|\{\s*[A-Za-z0-9_]+\s*,\s*[A-Za-z]+|%(?:\([A-Za-z0-9_]+\)|\d+\$)?l{0,2}[sd@]/g;

// The provider named here is an example, not a recommendation: the reasoning
// holds for any general-purpose model API, and the client is swappable. Pass
// any object with a `complete({ prompt, temperature })` method.
export const MODEL = 'gpt-4.1-mini'; // an example id: check the parameters your model accepts

export async function providerClient(sdk, model = MODEL) {
  if (!sdk) {
    const { OpenAI } = await import('openai');
    sdk = new OpenAI();
  }
  return {
    async complete({ prompt, temperature }) {
      const response = await sdk.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        temperature,
      });
      return response.choices[0].message.content;
    },
  };
}

export class TranslationUnavailable extends Error {}

/** The interpolation variables, sorted so a moved one still matches. */
export function placeholders(text) {
  return [...text.matchAll(PLACEHOLDER)].map((m) => m[0]).sort();
}

/**
 * Translate one interface string, with what the model needs to know about it.
 *
 * @param {string} source
 * @param {string} language
 * @param {object} options
 * @param {string} [options.context] where the string appears in the interface
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {number} [options.attempts]
 */
export async function translate(source, language, { context = '', client, attempts = 3 } = {}) {
  // Characters are counted as code points, as in Python: an emoji is one.
  if ([...source].length > MAX_CHARACTERS || [...context].length > MAX_CHARACTERS) {
    throw new RangeError(`string or context longer than ${MAX_CHARACTERS} characters`);
  }
  // Nothing to translate is not worth a call.
  if (source.trim() === '') return { target: source, review: false, warnings: [] };
  client ??= await providerClient();

  const variables = placeholders(source);
  const prompt = fill(PROMPT, {
    language,
    context: context || 'not given',
    variables: variables.join(' ') || 'none',
    source,
  });
  const target = await ask(client, prompt, attempts);

  const warnings = [];
  if (String(placeholders(target)) !== String(variables)) {
    warnings.push('the model did not keep the interpolation variables');
  }
  if (variables.some((variable) => variable.includes(','))) {
    warnings.push('ICU message: check its branches by hand');
  }
  return { target, review: warnings.length > 0, warnings };
}

/**
 * Fill the prompt fields. The replacer is a function on purpose: a string
 * replacement would give a `$` inside an interface string, such as the one a
 * numbered variable carries, a meaning it does not have.
 */
function fill(template, fields) {
  return Object.entries(fields).reduce(
    (text, [key, value]) => text.replace(`{${key}}`, () => value),
    template,
  );
}

async function ask(client, prompt, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt,
        // The lowest temperature: the SDK documents lower values as more
        // focused and deterministic.
        temperature: 0,
      });
      if (typeof answer !== 'string') {
        // A refusal comes back as no content.
        lastError = new Error('the model answered no text');
        continue;
      }
      let text = answer.trim();
      // A JSON answer wrapped whole in one code fence is read; nothing else is.
      if (text.startsWith('```') && text.endsWith('```') && text.split('```').length === 3) {
        text = text.slice(3, -3).replace(/^json/, '');
      }
      const parsed = JSON.parse(text);
      const target = parsed && typeof parsed === 'object' ? parsed.translation : null;
      if (typeof target === 'string' && target.trim()) return target.trim();
      lastError = new Error('the model answered without a translation');
    } catch (error) {
      lastError = error;
    }
  }
  throw new TranslationUnavailable(String(lastError));
}