Detect the language of a text

"""
Detect the language of a text with a naive Bayes classifier on character
n-grams.

Rung N1. Features close to those of N0, n-grams of one to three characters
cut at word boundaries, but weighed instead of ranked. Each n-gram of the text votes for every language, in proportion to
how often that language uses it, and the votes are multiplied together.

What this buys over the rank distance of N0 is a number the caller can act
on. N0 answers "French"; this answers "French, and here is how far ahead
of Spanish it is". A detector that can abstain is worth more than one that
is right slightly more often.

Training takes a paragraph per language. The model is a table of counts.
"""

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline


def train(samples: dict[str, str]):
    """
    Fit on one sample of text per language.

    `char_wb` cuts n-grams inside word boundaries, so an n-gram carries the
    information that it opens or closes a word, exactly as the padding of N0
    did. Smoothing is light: an n-gram this language never used should count
    against it, without ruling it out on a single character.
    """
    model = make_pipeline(
        CountVectorizer(analyzer="char_wb", ngram_range=(1, 3), lowercase=True),
        MultinomialNB(alpha=0.1),
    )
    model.fit(list(samples.values()), list(samples.keys()))
    return model


def probabilities(model, text: str) -> dict[str, float]:
    """
    How the text splits between the known languages.

    Read these as an ordering, not as a measure of truth: they always sum to
    one, over the languages the model was trained on and no others.
    """
    scores = model.predict_proba([text])[0]
    return {str(name): float(score) for name, score in zip(model.classes_, scores)}


def detect(model, text: str, minimum: float = 0.0) -> str | None:
    """
    The most likely language, or None when the model is not sure enough.

    `minimum` is yours to set. Raise it when a wrong language costs more than
    no answer, for instance when the answer picks the queue a message is
    routed to. Leave it at zero to always get a name, as N0 does.
    """
    best, score = max(probabilities(model, text).items(), key=lambda item: item[1])
    return best if score >= minimum else None

/**
 * Detect the language of a text with a naive Bayes classifier on character
 * n-grams.
 *
 * Rung N1. Features close to those of N0, n-grams of one to three characters
 * cut at word boundaries, but weighed instead of ranked. Each n-gram of the text votes for every language, in proportion
 * to how often that language uses it, and the votes are multiplied together.
 *
 * What this buys over the rank distance of N0 is a number the caller can act
 * on. N0 answers "French"; this answers "French, and here is how far ahead of
 * Spanish it is". A detector that can abstain is worth more than one that is
 * right slightly more often.
 *
 * Written out in full rather than pulled from a library, because multinomial
 * naive Bayes is nothing but counting, and fits in this file. That is the
 * argument of this rung: the classical tool is small enough to read.
 */

// Smoothing. An n-gram a language never used should count against it, without
// ruling the language out on the strength of a single character.
const ALPHA = 0.1;

/**
 * The n-grams of one to three characters of every word, padded with spaces so
 * that an n-gram carries the information that it opens or closes a word.
 */
export function ngrams(text) {
  const grams = [];
  for (const word of text.toLowerCase().split(/\s+/).filter(Boolean)) {
    const padded = ` ${word} `;
    for (let n = 1; n <= 3; n += 1) {
      for (let i = 0; i + n <= padded.length; i += 1) grams.push(padded.slice(i, i + n));
    }
  }
  return grams;
}

/** Fit on one sample of text per language: `{ fr: "…", en: "…" }`. */
export function train(samples) {
  const vocabulary = new Set();
  const counts = new Map();
  for (const [language, sample] of Object.entries(samples)) {
    const perLanguage = new Map();
    for (const gram of ngrams(sample)) {
      perLanguage.set(gram, (perLanguage.get(gram) ?? 0) + 1);
      vocabulary.add(gram);
    }
    counts.set(language, perLanguage);
  }
  return { counts, vocabulary };
}

/**
 * How the text splits between the known languages.
 *
 * Read these as an ordering, not as a measure of truth: they always sum to
 * one, over the languages the model was trained on and no others.
 */
export function probabilities(model, text) {
  const { counts, vocabulary } = model;
  const grams = ngrams(text).filter((gram) => vocabulary.has(gram));
  const logScores = [...counts].map(([language, perLanguage]) => {
    const total = [...perLanguage.values()].reduce((a, b) => a + b, 0);
    const denominator = total + ALPHA * vocabulary.size;
    // Sum of logs rather than a product of probabilities: multiplying a few
    // thousand small numbers underflows to zero.
    let score = 0;
    for (const gram of grams) score += Math.log(((perLanguage.get(gram) ?? 0) + ALPHA) / denominator);
    return [language, score];
  });
  const highest = Math.max(...logScores.map(([, score]) => score));
  const weights = logScores.map(([language, score]) => [language, Math.exp(score - highest)]);
  const sum = weights.reduce((a, [, weight]) => a + weight, 0);
  return Object.fromEntries(weights.map(([language, weight]) => [language, weight / sum]));
}

/**
 * The most likely language, or null when the model is not sure enough.
 *
 * `minimum` is yours to set. Raise it when a wrong language costs more than no
 * answer, for instance when the answer picks the queue a message is routed to.
 * Leave it at zero to always get a name, as N0 does.
 */
export function detect(model, text, minimum = 0) {
  const scores = Object.entries(probabilities(model, text));
  const [best, score] = scores.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))[0];
  return score >= minimum ? best : null;
}

"""
Detect the language with a dedicated identifier, self-hosted.

Rung N2. N0 and N1 only know the languages you handed them a sample of. CLD3,
Google's Compact Language Detector, is a small neural network covering a
hundred-odd languages, shipped with its model inside the package: nothing to
sample, nothing to train, no key, no download, and no text leaving your
machine. `gcld3` in Python, `cld3-asm` in JavaScript, the same model
underneath.

What you own on this rung is not the model. It is three things.

The mapping from its codes to yours: CLD3 answers in BCP-47 style, and tells
Latin-script Hindi from Devanagari Hindi with "hi-Latn". A caller expecting
"hi" gets a code it has never seen.

The refusal, and the byte floor it rests on. Below `min_bytes` the library
answers "und" instead of guessing, and that floor — 140 bytes by default — is
the thing N0 and N1 do not have: it is the two-word message, the subject line,
the search box, refused by the library itself. Above the floor, `is_reliable`
and `probability` are two more handles, and this snippet turns all three into
None.

And the loading. The model is read when the identifier is built, and the
identifier holds memory the garbage collector does not manage: build one, keep
it, do not build one per message.
"""

from __future__ import annotations

# The library's own defaults, read from its header: below 140 bytes it returns
# "und", and it predicts on the first 700 bytes of what it is given. Lower the
# floor and you get an answer on a two-word message; you also get the guess
# this whole entry is about.
MIN_BYTES, MAX_BYTES = 140, 700

# Below this probability the answer is not used. CLD3 also carries its own
# `is_reliable`, computed from the text it saw; both have to agree.
THRESHOLD = 0.7


class IdentificationUnavailable(Exception):
    """The identifier could not be loaded, or answered something unusable."""


class Cld3Identifier:
    """The real identifier, and the whole surface of the library this needs."""

    def __init__(self, min_bytes: int = MIN_BYTES, max_bytes: int = MAX_BYTES, module=None) -> None:
        if module is None:
            import gcld3  # a local install, model included, no network and no key

            module = gcld3
        self._identifier = module.NNetLanguageIdentifier(
            min_num_bytes=min_bytes, max_num_bytes=max_bytes
        )

    def find(self, text: str) -> dict:
        """One answer: the code, how sure the model is, and its own verdict on that."""
        found = self._identifier.FindLanguage(text=text)
        return {
            "language": found.language,
            "probability": float(found.probability),
            "is_reliable": bool(found.is_reliable),
        }


_LOADED: Cld3Identifier | None = None


def loaded_identifier() -> Cld3Identifier:
    """The default identifier, built on first use and kept."""
    global _LOADED
    if _LOADED is None:
        _LOADED = Cld3Identifier()
    return _LOADED


def detect(text: str, identifier=None, threshold: float = THRESHOLD) -> str | None:
    """
    Return the language code, or None when nothing can be said.

    None on four counts, and they are not the same thing said four times: an
    empty text, the code CLD3 uses for "undetermined" — which is also what it
    answers on a text below the byte floor —, its own `is_reliable` set to
    false, and a probability below the threshold. Returning a language in any
    of those cases would hand the caller a guess wearing the clothes of a
    fact, which is the failure this whole entry is about.

    `identifier` is injected so this can be tested without installing the
    library. In production it defaults to the real one above.
    """
    if not text.strip():
        return None
    identifier = identifier or loaded_identifier()
    try:
        found = identifier.find(text)
    except Exception as error:  # noqa: BLE001 - a library failure is not the caller's fault
        raise IdentificationUnavailable(str(error)) from error
    if not isinstance(found, dict) or not isinstance(found.get("language"), str):
        raise IdentificationUnavailable("the identifier returned no language")
    if found["language"] == "und" or not found.get("is_reliable"):
        return None
    return found["language"] if float(found.get("probability", 0.0)) >= threshold else None

/**
 * Detect the language with a dedicated identifier, self-hosted.
 *
 * Rung N2. N0 and N1 only know the languages you handed them a sample of.
 * CLD3, Google's Compact Language Detector, is a small neural network covering
 * a hundred-odd languages, shipped with its model inside the package: nothing
 * to sample, nothing to train, no key, no download, and no text leaving your
 * machine. `cld3-asm` in JavaScript, `gcld3` in Python, the same model
 * underneath.
 *
 * What you own on this rung is not the model. It is three things.
 *
 * The mapping from its codes to yours: CLD3 answers in BCP-47 style, and tells
 * Latin-script Hindi from Devanagari Hindi with "hi-Latn". A caller expecting
 * "hi" gets a code it has never seen.
 *
 * The refusal, and the byte floor it rests on. Below `minBytes` the library
 * answers "und" instead of guessing, and that floor — 140 bytes by default —
 * is the thing N0 and N1 do not have: it is the two-word message, the subject
 * line, the search box, refused by the library itself. Above the floor,
 * `is_reliable` and `probability` are two more handles, and this snippet turns
 * all three into null.
 *
 * And the loading. The WebAssembly module is compiled when the identifier is
 * built, and the identifier holds memory the garbage collector does not manage
 * — the library asks you to `dispose()` it. Build one, keep it, do not build
 * one per message.
 */

// The library's own defaults, read from its header: below 140 bytes it returns
// "und", and it predicts on the first 700 bytes of what it is given. Lower the
// floor and you get an answer on a two-word message; you also get the guess
// this whole entry is about.
export const MIN_BYTES = 140;
export const MAX_BYTES = 700;

// Below this probability the answer is not used. CLD3 also carries its own
// `is_reliable`, computed from the text it saw; both have to agree.
export const THRESHOLD = 0.7;

export class IdentificationUnavailable extends Error {}

/** The real identifier, and the whole surface of the library this needs. */
export async function cld3Identifier({ minBytes = MIN_BYTES, maxBytes = MAX_BYTES, load } = {}) {
  // A local install, model included, no network and no key.
  const { loadModule } = await (load ? load() : import('cld3-asm'));
  const factory = await loadModule();
  const identifier = factory.create(minBytes, maxBytes);
  return {
    /** One answer: the code, how sure the model is, and its own verdict on that. */
    find(text) {
      const found = identifier.findLanguage(text);
      return {
        language: found.language,
        probability: Number(found.probability),
        is_reliable: Boolean(found.is_reliable),
      };
    },
  };
}

let loaded;

/** The default identifier, built on first use and kept. */
export function loadedIdentifier(options) {
  loaded ??= cld3Identifier(options).catch((error) => {
    loaded = undefined; // a failed load is not a cached answer
    throw error;
  });
  return loaded;
}

/**
 * Return the language code, or null when nothing can be said.
 *
 * Null on four counts, and they are not the same thing said four times: an
 * empty text, the code CLD3 uses for "undetermined" — which is also what it
 * answers on a text below the byte floor —, its own `is_reliable` set to
 * false, and a probability below the threshold. Returning a language in any of
 * those cases would hand the caller a guess wearing the clothes of a fact,
 * which is the failure this whole entry is about.
 *
 * `identifier` is injected so this can be tested without installing the
 * library. In production it defaults to the real one above.
 */
export async function detect(text, { identifier, threshold = THRESHOLD } = {}) {
  if (!text.trim()) return null;
  const engine = identifier ?? (await loadedIdentifier());
  let found;
  try {
    found = await engine.find(text);
  } catch (error) {
    throw new IdentificationUnavailable(String(error));
  }
  if (!found || typeof found.language !== 'string') {
    throw new IdentificationUnavailable('the identifier returned no language');
  }
  if (found.language === 'und' || !found.is_reliable) return null;
  return Number(found.probability) >= threshold ? found.language : null;
}

"""
Detect the language of a text by asking a general-purpose model.

Rung N3. This is the option people reach for first. It is here so you can see
what it costs, not because this entry recommends it.

Note what the code has to do that N0 did not: send only an excerpt, retry on
failure, parse an answer that is only probably valid JSON, normalise a code the
model may write in capitals, with a region or with stray spaces, and refuse an
answer that is outside the list it was given. That plumbing is the real cost of
this rung, and it is the part your tests have to cover, because the model
itself is not testable.

The one thing this rung genuinely adds is that it needs no sample of the
language. The one thing it cannot do is tell you it is wrong.
"""

from __future__ import annotations

import json
from collections.abc import Collection

# The provider named here is an example, not a recommendation: the reasoning
# holds for any general-purpose model API, and the client is swappable. Pass
# any object with a `complete(prompt=..., temperature=...)` method.
MODEL = "gpt-4.1-mini"  # an example id: check the parameters your model accepts

PROMPT = (
    "Identify the language of the text below.\n"
    "Answer with JSON only: an object with the key `language`, whose value is\n"
    "a two-letter ISO 639-1 code chosen from this list:\n"
    "{languages}, or `und` if the text is in none of them.\n\n"
    "Text:\n{excerpt}"
)

# Only the first characters are sent, a few sentences: N0 and N1 already name
# the language of a single sentence, and the model bills every token past it.
# That excerpt is the whole cost control — there is no cap on the input, because
# a cap would refuse a long email that costs exactly the same as a short one.
EXCERPT_CHARACTERS = 600


class DetectionUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


class ProviderClient:
    """The one call this snippet makes, on top of the provider's SDK."""

    def __init__(self, sdk=None, model: str = MODEL):
        if sdk is None:  # pragma: no cover - needs a key and a network
            from openai import OpenAI

            sdk = OpenAI()
        self.sdk, self.model = sdk, model

    def complete(self, *, prompt: str, temperature: float) -> str:
        response = self.sdk.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            temperature=temperature,
        )
        return response.choices[0].message.content


def detect(text: str, languages: Collection[str], client=None, *, attempts: int = 3) -> str | None:
    """
    Return the code of the detected language, or None when the text is blank
    or the model says it is in none of the languages it was offered.

    `client` is injected so this function can be tested without a network
    call. In production it defaults to a real provider client.
    """
    # A blank text costs nothing to refuse and cannot say anything. A long one
    # is not refused: only its first characters are ever sent.
    if not text.strip():
        return None

    client = client or ProviderClient()
    answer = _ask(client, text[:EXCERPT_CHARACTERS], sorted(languages), attempts)

    # "fr", "FR", "fr-CA" and the locale form "fr_CA" are all read as "fr". A
    # language name such as "French" is not a code, and is refused below.
    code = str(answer.get("language", "")).strip().lower().replace("_", "-").split("-")[0]
    if code == "und":
        return None
    if code not in languages:
        raise DetectionUnavailable(f"the model answered a language outside the list: {code!r}")
    return code


def _ask(client, excerpt: str, languages: list[str], attempts: int) -> dict:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            answer = client.complete(
                prompt=PROMPT.format(languages=", ".join(languages), excerpt=excerpt),
                # Temperature zero, because a routing decision that changes
                # between two identical calls cannot be reviewed.
                temperature=0,
            )
            # No content at all (a refusal) is as unusable as prose.
            parsed = json.loads(answer) if isinstance(answer, str) else None
            if isinstance(parsed, dict):
                return parsed
            last_error = ValueError("the model answered something that is not an object")
        except (TypeError, NameError, AttributeError):
            # A programming error in this file is not a provider failure:
            # retrying it would pay three bills for one bug.
            raise
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise DetectionUnavailable(str(last_error))

/**
 * Detect the language of a text by asking a general-purpose model.
 *
 * Rung N3. This is the option people reach for first. It is here so you can
 * see what it costs, not because this entry recommends it.
 *
 * Note what the code has to do that N0 did not: send only an excerpt, retry on
 * failure, parse an answer that is only probably valid JSON, normalise a code
 * the model may write in capitals, with a region or with stray spaces, and
 * refuse an answer that is outside the list it was given. That plumbing is the
 * real cost of this rung, and it is the part your tests have to cover, because
 * the model itself is not testable.
 *
 * The one thing this rung genuinely adds is that it needs no sample of the
 * language. The one thing it cannot do is tell you it is wrong.
 */

// The provider named here is an example, not a recommendation: the reasoning
// holds for any general-purpose model API, and the client is swappable. Pass
// any object with a `complete({ prompt, temperature })` method.
export const MODEL = 'gpt-4.1-mini'; // an example id: check the parameters your model accepts

// Only the first characters are sent, a few sentences: N0 and N1 already name
// the language of a single sentence, and the model bills every token past it.
// That excerpt is the whole cost control — there is no cap on the input,
// because a cap would refuse a long email that costs exactly the same as a
// short one.
export const EXCERPT_CHARACTERS = 600;

export class DetectionUnavailable extends Error {}

export async function providerClient(sdk, model = MODEL) {
  if (!sdk) {
    const { OpenAI } = await import('openai');
    sdk = new OpenAI();
  }
  return {
    async complete({ prompt, temperature }) {
      const response = await sdk.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        temperature,
      });
      return response.choices[0].message.content;
    },
  };
}

/** The exact request sent to the model. Exported so a test can read it. */
export function buildPrompt(languages, excerpt) {
  return [
    'Identify the language of the text below.',
    'Answer with JSON only: an object with the key `language`, whose value is',
    'a two-letter ISO 639-1 code chosen from this list:',
    `${[...languages].sort().join(', ')}, or \`und\` if the text is in none of them.`,
    '',
    'Text:',
    excerpt,
  ].join('\n');
}

/**
 * The code of the detected language, or null when the text is blank or the
 * model says it is in none of the languages it was offered.
 *
 * @param {string} text
 * @param {string[]} languages the codes the model may choose from
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {number} [options.attempts]
 */
export async function detect(text, languages, { client, attempts = 3 } = {}) {
  // Counted in characters, as Python counts them: `length` would count an
  // emoji twice, and `slice` could cut one in half.
  const characters = [...text];

  // A blank text costs nothing to refuse and cannot say anything. A long one is
  // not refused: only its first characters are ever sent.
  if (!text.trim()) return null;

  client ??= await providerClient();
  const excerpt = characters.slice(0, EXCERPT_CHARACTERS).join('');
  const answer = await ask(client, excerpt, languages, attempts);

  // "fr", "FR", "fr-CA" and the locale form "fr_CA" are all read as "fr". A
  // language name such as "French" is not a code, and is refused below.
  const code = String(answer.language ?? '').trim().toLowerCase().replace('_', '-').split('-')[0];
  if (code === 'und') return null;
  if (![...languages].includes(code)) {
    throw new DetectionUnavailable(`the model answered a language outside the list: ${code}`);
  }
  return code;
}

async function ask(client, excerpt, languages, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt: buildPrompt(languages, excerpt),
        // Temperature zero, because a routing decision that changes between
        // two identical calls cannot be reviewed.
        temperature: 0,
      });
      // No content at all (a refusal) is as unusable as prose.
      const parsed = typeof answer === 'string' ? JSON.parse(answer) : null;
      if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed;
      lastError = new Error('the model answered something that is not an object');
    } catch (error) {
      // Only what the provider or the decoder can raise. A TypeError from this
      // file is a bug, and retrying it would pay three bills for it.
      if (error instanceof TypeError || error instanceof ReferenceError) throw error;
      lastError = error;
    }
  }
  throw new DetectionUnavailable(String(lastError));
}