Translate interface strings

"""
Reuse the translations you already paid for: a translation memory.

Rung N0. No model, no service, no key. An interface string rarely changes
deeply: a word is added, a capital is fixed, a variable moves. When that
happens, last year's translation is still nearly right, and the cheapest
translation is the one you do not order twice.

Three answers, and the difference between them matters more than the code.
An exact match ships. An approximate match is a draft: it comes back with its
score and a review flag, never as a finished translation. Anything else is new
text, which this rung has nothing to say about — see the breaking point in the
test. Handing back an approximation as a certainty is the one behaviour that
would make this whole approach dishonest.

Interpolation variables are checked apart from the score. A translation whose
variables do not match the source is a broken interface, however high it
scores, so it is flagged even on an exact hit.
"""

from __future__ import annotations

import re
import unicodedata
from difflib import SequenceMatcher

# The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
# and the numbered variant of %s that Android and iOS string files carry.
PLACEHOLDER = re.compile(r"\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]")


def placeholders(text: str) -> list[str]:
    """The interpolation variables of a string, sorted so order does not count."""
    return sorted(PLACEHOLDER.findall(text))


def normalise(text: str) -> str:
    """Fold case, accents and spacing, which are not what makes a string new."""
    stripped = unicodedata.normalize("NFKD", text)
    without_accents = "".join(c for c in stripped if not unicodedata.combining(c))
    return " ".join(without_accents.lower().split())


def lookup(source: str, memory: dict[str, str], threshold: float = 0.75) -> dict:
    """
    Look a string up in a memory of `source: target` pairs already translated.

    Returns the status (`exact`, `fuzzy` or `none`), the target when there is
    one, the score, and whether a human has to look at it.
    """
    key = normalise(source)
    best_source, best_target, best_score = None, None, 0.0
    for known, target in memory.items():
        candidate = normalise(known)
        if candidate == key:
            # Exact after normalisation: a fixed capital or a stray double
            # space is not a new string to send to a translator.
            return _decide("exact", source, known, target, 1.0)
        # autojunk=False so a long string scores exactly like the JavaScript
        # version of this snippet, which has no such heuristic.
        score = SequenceMatcher(None, key, candidate, autojunk=False).ratio()
        if score > best_score:
            best_source, best_target, best_score = known, target, score
    if best_score >= threshold:
        return _decide("fuzzy", source, best_source, best_target, best_score)
    return {"status": "none", "target": None, "score": best_score,
            "matched": None, "review": False, "warnings": []}


def _decide(status: str, source: str, matched: str, target: str, score: float) -> dict:
    """Assemble the answer, and never let a fuzzy hit pass as a finished one."""
    warnings = []
    if placeholders(target) != placeholders(source):
        warnings.append("interpolation variables differ from the source string")
    return {"status": status, "target": target, "score": score, "matched": matched,
            "review": status == "fuzzy" or bool(warnings), "warnings": warnings}

/**
 * Reuse the translations you already paid for: a translation memory.
 *
 * Rung N0. No model, no service, no key. An interface string rarely changes
 * deeply: a word is added, a capital is fixed, a variable moves. When that
 * happens, last year's translation is still nearly right, and the cheapest
 * translation is the one you do not order twice.
 *
 * Three answers, and the difference between them matters more than the code.
 * An exact match ships. An approximate match is a draft: it comes back with
 * its score and a review flag, never as a finished translation. Anything else
 * is new text, which this rung has nothing to say about — see the breaking
 * point in the test. Handing back an approximation as a certainty is the one
 * behaviour that would make this whole approach dishonest.
 *
 * Interpolation variables are checked apart from the score. A translation
 * whose variables do not match the source is a broken interface, however high
 * it scores, so it is flagged even on an exact hit.
 */

// The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
// and the numbered variant of %s that Android and iOS string files carry.
const PLACEHOLDER = /\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]/g;

/** The interpolation variables of a string, sorted so order does not count. */
export function placeholders(text) {
  return [...text.matchAll(PLACEHOLDER)].map((m) => m[0]).sort();
}

/** Fold case, accents and spacing, which are not what makes a string new. */
export function normalise(text) {
  return text.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase().split(/\s+/).filter(Boolean).join(' ');
}

/**
 * Look a string up in a memory of `source: target` pairs already translated.
 *
 * Returns the status (`exact`, `fuzzy` or `none`), the target when there is
 * one, the score, and whether a human has to look at it.
 */
export function lookup(source, memory, threshold = 0.75) {
  const key = normalise(source);
  let best = { source: null, target: null, score: 0 };
  for (const [known, target] of Object.entries(memory)) {
    const candidate = normalise(known);
    if (candidate === key) {
      // Exact after normalisation: a fixed capital or a stray double space is
      // not a new string to send to a translator.
      return decide('exact', source, known, target, 1);
    }
    const score = ratio(key, candidate);
    if (score > best.score) best = { source: known, target, score };
  }
  if (best.score >= threshold) return decide('fuzzy', source, best.source, best.target, best.score);
  return { status: 'none', target: null, score: best.score, matched: null, review: false, warnings: [] };
}

/** Assemble the answer, and never let a fuzzy hit pass as a finished one. */
function decide(status, source, matched, target, score) {
  const warnings = [];
  if (String(placeholders(target)) !== String(placeholders(source))) {
    warnings.push('interpolation variables differ from the source string');
  }
  return {
    status, target, score, matched,
    review: status === 'fuzzy' || warnings.length > 0,
    warnings,
  };
}

/**
 * Ratcliff-Obershelp similarity, the algorithm behind Python's
 * difflib.SequenceMatcher.ratio: twice the number of matching characters over
 * the total length. Ported here so both versions of this snippet return the
 * very same score on the very same pair.
 */
export function ratio(a, b) {
  const left = [...a];
  const right = [...b];
  const total = left.length + right.length;
  if (total === 0) return 1;
  return (2 * matchedCount(left, right, 0, left.length, 0, right.length)) / total;
}

/** Longest common block, then the same search left and right of it. */
function matchedCount(a, b, alo, ahi, blo, bhi) {
  let bestI = alo, bestJ = blo, bestSize = 0;
  let lengths = new Map();
  for (let i = alo; i < ahi; i += 1) {
    const next = new Map();
    for (let j = blo; j < bhi; j += 1) {
      if (a[i] !== b[j]) continue;
      const size = (lengths.get(j - 1) ?? 0) + 1;
      next.set(j, size);
      // Strictly greater: the earliest block wins a tie, as in difflib.
      if (size > bestSize) [bestI, bestJ, bestSize] = [i - size + 1, j - size + 1, size];
    }
    lengths = next;
  }
  if (bestSize === 0) return 0;
  return bestSize
    + matchedCount(a, b, alo, bestI, blo, bestJ)
    + matchedCount(a, b, bestI + bestSize, ahi, bestJ + bestSize, bhi);
}

"""
Translate with a self-hosted neural model, one pair of languages at a time.

Rung N2. This is the rung that actually translates: unlike the memory of N0,
it has an answer for a string nobody has ever written before. The price is a
model file per language pair to ship, keep in sync and hold in a warm process,
and an output nobody can explain.

Most of the code below is not about translating. It is about the interpolation
variables, and that is the honest picture of this rung. A translation model
sees `{count} items selected` as text, so it happily translates the word
inside the braces, drops it, or repeats it. The interface then prints a brace
where a number should be, and the bug reaches production because the string
looked fine to everyone who does not read that language.

So the variables are hidden behind neutral markers before the model sees the
string, put back afterwards, and counted. Moving a marker is allowed — word
order is the model's job. Losing or inventing one is reported, and the caller
gets a flagged draft instead of a broken interface.
"""

from __future__ import annotations

import re
from types import SimpleNamespace

MODEL_NAME = "Helsinki-NLP/opus-mt-en-fr"

# The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
# and the numbered variant of %s that Android and iOS string files carry.
PLACEHOLDER = re.compile(r"\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]")

# The stand-in the model sees instead of a variable. Deliberately not a word.
MARK = "⟦{}⟧"


class TranslationUnavailable(Exception):
    """The model failed every attempt, or returned nothing usable."""


def load_translator(name: str = MODEL_NAME):
    """The real model: weights on disk, loaded once, run locally."""
    from transformers import pipeline  # pragma: no cover - needs the weights

    pipe = pipeline("translation", model=name)
    return SimpleNamespace(generate=lambda text: pipe(text)[0]["translation_text"])


def placeholders(text: str) -> list[str]:
    """The interpolation variables, in the order they appear."""
    return PLACEHOLDER.findall(text)


def translate(source: str, model=None, *, attempts: int = 2) -> dict:
    """
    Translate one interface string, and check what came back.

    `model` is injected so this can be tested without loading the weights.
    Left alone, it is the real one above.
    """
    model = load_translator() if model is None else model
    if not source.strip():
        return {"target": source, "review": False, "warnings": []}

    variables = placeholders(source)
    masked = source
    for index, variable in enumerate(variables):
        masked = masked.replace(variable, MARK.format(index), 1)

    target = _generate(model, masked, attempts).strip()
    for index, variable in enumerate(variables):
        target = target.replace(MARK.format(index), variable)

    warnings = []
    found = placeholders(target)
    if sorted(found) != sorted(variables):
        warnings.append(
            "variables differ from the source: expected "
            + (" ".join(variables) or "none") + ", got " + (" ".join(found) or "none")
        )
    return {"target": target, "review": bool(warnings), "warnings": warnings}


def _generate(model, text: str, attempts: int) -> str:
    """A local model still fails: out of memory, a worker that died, a batch."""
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            output = model.generate(text)
        except Exception as error:  # noqa: BLE001 - any model failure is retried
            last_error = error
            continue
        if output and output.strip():
            return output
        last_error = ValueError("the model returned an empty translation")
    raise TranslationUnavailable(str(last_error))

/**
 * Translate with a self-hosted neural model, one pair of languages at a time.
 *
 * Rung N2. This is the rung that actually translates: unlike the memory of
 * N0, it has an answer for a string nobody has ever written before. The price
 * is a model file per language pair to ship, keep in sync and hold in a warm
 * process, and an output nobody can explain.
 *
 * Most of the code below is not about translating. It is about the
 * interpolation variables, and that is the honest picture of this rung. A
 * translation model sees `{count} items selected` as text, so it happily
 * translates the word inside the braces, drops it, or repeats it. The
 * interface then prints a brace where a number should be, and the bug reaches
 * production because the string looked fine to everyone who does not read
 * that language.
 *
 * So the variables are hidden behind neutral markers before the model sees
 * the string, put back afterwards, and counted. Moving a marker is allowed —
 * word order is the model's job. Losing or inventing one is reported, and the
 * caller gets a flagged draft instead of a broken interface.
 */

export const MODEL_NAME = 'Xenova/opus-mt-en-fr';

// The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
// and the numbered variant of %s that Android and iOS string files carry.
const PLACEHOLDER = /\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]/g;

// The stand-in the model sees instead of a variable. Deliberately not a word.
const mark = (index) => `⟦${index}⟧`;

export class TranslationUnavailable extends Error {}

/** The real model: weights on disk, loaded once, run locally. */
export async function loadTranslator(name = MODEL_NAME) {
  const { pipeline } = await import('@xenova/transformers');
  const pipe = await pipeline('translation', name);
  return { generate: async (text) => (await pipe(text))[0].translation_text };
}

/** The interpolation variables, in the order they appear. */
export function placeholders(text) {
  return [...text.matchAll(PLACEHOLDER)].map((m) => m[0]);
}

/**
 * Translate one interface string, and check what came back.
 *
 * @param {string} source
 * @param {object} options
 * @param {{generate: Function}} [options.model] injected so this can be
 *   tested without loading the weights; defaults to the real one above
 * @param {number} [options.attempts]
 */
export async function translate(source, { model, attempts = 2 } = {}) {
  const translator = model ?? (await loadTranslator());
  if (source.trim() === '') return { target: source, review: false, warnings: [] };

  const variables = placeholders(source);
  let masked = source;
  variables.forEach((variable, index) => {
    masked = masked.replace(variable, mark(index));
  });

  let target = (await generate(translator, masked, attempts)).trim();
  variables.forEach((variable, index) => {
    target = target.split(mark(index)).join(variable);
  });

  const warnings = [];
  const found = placeholders(target);
  if (String([...found].sort()) !== String([...variables].sort())) {
    warnings.push(
      `variables differ from the source: expected ${variables.join(' ') || 'none'}` +
        `, got ${found.join(' ') || 'none'}`,
    );
  }
  return { target, review: warnings.length > 0, warnings };
}

/** A local model still fails: out of memory, a worker that died, a batch. */
async function generate(model, text, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    let output;
    try {
      output = await model.generate(text);
    } catch (error) {
      lastError = error;
      continue;
    }
    if (output && output.trim()) return output;
    lastError = new Error('the model returned an empty translation');
  }
  throw new TranslationUnavailable(String(lastError));
}

"""
Translate by asking a general-purpose model, with the interface context.

Rung N3. What this rung buys over a translation model is the context: a
translator model gets a string and nothing else, while a general-purpose model
can be told that `Save` is the label of a button and not the verb in a
sentence, that the interface is addressed to a customer rather than an
administrator, and that there is no room for a full sentence. That is exactly
the information a translation team asks for and rarely gets.

What it costs is everything around the call: a key, a provider that answers
prose when JSON was asked for, retries, a cap on the input, and the same
variable check as the rung below, because being asked to keep `{count}`
verbatim is not the same as doing it.
"""

from __future__ import annotations

import json
import re

PROMPT = (
    "Translate the user interface string below into {language}.\n"
    "Where it appears in the interface: {context}\n"
    "Keep these interpolation variables exactly as written: {variables}\n"
    "Keep the length of an interface label, not of a sentence.\n"
    'Answer with JSON only: {{"translation": "..."}}\n\n'
    "String:\n{source}"
)

# An interface string that no longer fits on one screen is not an interface
# string. Refusing it here is a cost control, not an optimisation.
MAX_CHARACTERS = 2000

# The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
# and the numbered variant of %s that Android and iOS string files carry.
PLACEHOLDER = re.compile(r"\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]")


class TranslationUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def placeholders(text: str) -> list[str]:
    """The interpolation variables, sorted so a moved one still matches."""
    return sorted(PLACEHOLDER.findall(text))


def translate(source: str, language: str, *, context: str = "",
              client=None, attempts: int = 3) -> dict:
    """
    Translate one interface string, with what the model needs to know about it.

    `client` is injected so this function can be tested without a network
    call. In production it defaults to a real provider client.
    """
    if client is None:  # pragma: no cover - needs a key and a network
        from openai import OpenAI

        client = OpenAI()
    if len(source) > MAX_CHARACTERS:
        raise ValueError(f"string longer than {MAX_CHARACTERS} characters")

    variables = placeholders(source)
    prompt = PROMPT.format(
        language=language,
        context=context or "not given",
        variables=" ".join(variables) or "none",
        source=source,
    )
    target = _ask(client, prompt, attempts)

    warnings = []
    if placeholders(target) != variables:
        warnings.append("the model did not keep the interpolation variables")
    return {"target": target, "review": bool(warnings), "warnings": warnings}


def _ask(client, prompt: str, attempts: int) -> str:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero: two identical strings must not come back
            # translated two different ways in the same interface.
            answer = client.complete(prompt=prompt, temperature=0)
            parsed = json.loads(answer)
            target = parsed.get("translation") if isinstance(parsed, dict) else None
            if isinstance(target, str) and target.strip():
                return target.strip()
            last_error = ValueError("the model answered without a translation")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise TranslationUnavailable(str(last_error))

/**
 * Translate by asking a general-purpose model, with the interface context.
 *
 * Rung N3. What this rung buys over a translation model is the context: a
 * translator model gets a string and nothing else, while a general-purpose
 * model can be told that `Save` is the label of a button and not the verb in
 * a sentence, that the interface is addressed to a customer rather than an
 * administrator, and that there is no room for a full sentence. That is
 * exactly the information a translation team asks for and rarely gets.
 *
 * What it costs is everything around the call: a key, a provider that answers
 * prose when JSON was asked for, retries, a cap on the input, and the same
 * variable check as the rung below, because being asked to keep `{count}`
 * verbatim is not the same as doing it.
 */

const PROMPT = [
  'Translate the user interface string below into {language}.',
  'Where it appears in the interface: {context}',
  'Keep these interpolation variables exactly as written: {variables}',
  'Keep the length of an interface label, not of a sentence.',
  'Answer with JSON only: {"translation": "..."}',
  '',
  'String:',
  '{source}',
].join('\n');

// An interface string that no longer fits on one screen is not an interface
// string. Refusing it here is a cost control, not an optimisation.
export const MAX_CHARACTERS = 2000;

// The variable forms an interface uses: {count}, {}, %s, %d, %(count)s,
// and the numbered variant of %s that Android and iOS string files carry.
const PLACEHOLDER = /\{[A-Za-z0-9_]*\}|%(?:\([A-Za-z0-9_]+\)|\d+\$)?[sd]/g;

export class TranslationUnavailable extends Error {}

/** The interpolation variables, sorted so a moved one still matches. */
export function placeholders(text) {
  return [...text.matchAll(PLACEHOLDER)].map((m) => m[0]).sort();
}

/**
 * Translate one interface string, with what the model needs to know about it.
 *
 * @param {string} source
 * @param {string} language
 * @param {object} options
 * @param {string} [options.context] where the string appears in the interface
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {number} [options.attempts]
 */
export async function translate(source, language, { context = '', client, attempts = 3 } = {}) {
  let provider = client;
  if (!provider) {
    // Needs a key and a network, so it is never reached in the tests.
    const { OpenAI } = await import('openai');
    provider = new OpenAI();
  }
  if (source.length > MAX_CHARACTERS) {
    throw new RangeError(`string longer than ${MAX_CHARACTERS} characters`);
  }

  const variables = placeholders(source);
  const prompt = fill(PROMPT, {
    language,
    context: context || 'not given',
    variables: variables.join(' ') || 'none',
    source,
  });
  const target = await ask(provider, prompt, attempts);

  const warnings = [];
  if (String(placeholders(target)) !== String(variables)) {
    warnings.push('the model did not keep the interpolation variables');
  }
  return { target, review: warnings.length > 0, warnings };
}

/**
 * Fill the prompt fields. The replacer is a function on purpose: a string
 * replacement would give a `$` inside an interface string, such as the one a
 * numbered variable carries, a meaning it does not have.
 */
function fill(template, fields) {
  return Object.entries(fields).reduce(
    (text, [key, value]) => text.replace(`{${key}}`, () => value),
    template,
  );
}

async function ask(client, prompt, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt,
        // Temperature zero: two identical strings must not come back
        // translated two different ways in the same interface.
        temperature: 0,
      });
      const parsed = JSON.parse(answer);
      const target = parsed && typeof parsed === 'object' ? parsed.translation : null;
      if (typeof target === 'string' && target.trim()) return target.trim();
      lastError = new Error('the model answered without a translation');
    } catch (error) {
      lastError = error;
    }
  }
  throw new TranslationUnavailable(String(lastError));
}