Étiqueter des articles par thème

"""
Étiqueter des articles depuis un vocabulaire contrôlé, par correspondance
après retrait des terminaisons courantes.

Niveau N0. Déterministe, bibliothèque standard seule, et vérifiable : chaque
étiquette se remonte jusqu'au terme qui l'a produite, ce que demande un
rédacteur la première fois qu'une étiquette paraît fausse.

Deux choses font que ça marche.

D'abord, l'article et le vocabulaire passent par exactement la même chaîne. Un
racineur appliqué d'un seul côté échouera allègrement à faire correspondre un
mot avec lui-même.

Ensuite, l'étiquetage est multi-étiquette par construction. Un article traite
plusieurs thèmes, ou aucun ; une fonction qui rend un thème par article répond
à une question que personne n'a posée.
"""

import re
import unicodedata

# Endings stripped, longest of any two that overlap first. This is a
# plural-and-suffix stripper, not a lemmatiser: "embauchons" does not become
# "embaucher", "fiscaux" does not become "fiscal", and "poste" and "post" fold
# onto the same stem. Both sides of the comparison get the same treatment.
SUFFIXES = ("ements", "ement", "ations", "ation", "es", "s", "x", "e")

WORD = re.compile(r"[^\W_]+")


def normalise(text: str) -> str:
    """Lowercase and drop accents, so "Fiscalité" and "FISCALITE" meet."""
    # NFD does not split the ligatures, and a soft hyphen or a zero-width
    # space (format characters) would cut a word in two: both are handled here.
    text = text.lower().replace("œ", "oe").replace("æ", "ae")
    decomposed = unicodedata.normalize("NFD", text)
    return "".join(
        c for c in decomposed if not unicodedata.combining(c) and unicodedata.category(c) != "Cf"
    )


def strip_ending(word: str) -> str:
    """Strip one ending, and only when a stem of three letters is left."""
    for suffix in SUFFIXES:
        if word.endswith(suffix) and len(word) - len(suffix) >= 3:
            return word[: -len(suffix)]
    return word


def stems(text: str) -> str:
    """
    The text as a run of stems, padded with spaces at both ends.

    The padding is what lets a multi-word term be found with a plain substring
    search: "impot" can then never match inside "impotent".
    """
    return " " + " ".join(strip_ending(w) for w in WORD.findall(normalise(text))) + " "


def tag(article: str, vocabulary: dict[str, list[str]], min_terms: int = 1) -> list[str]:
    """
    Every topic whose terms appear in the article, best supported first.

    `vocabulary` maps a topic name to the terms that stand for it, single or
    multi-word. Keeping it a parameter is the point of this rung: the taxonomy
    belongs to whoever edits the articles, not to the code.

    `min_terms` is how many distinct terms a topic needs before it is claimed.
    Raise it when a single passing mention is not enough to file an article.
    """
    if min_terms < 1:
        raise ValueError("min_terms must be at least 1: a topic needs a term")
    haystack = stems(article)
    hits = {}
    for topic, terms in vocabulary.items():
        remaining, found = haystack, 0
        # Longest first, and a term found is removed from the text: "impot"
        # inside "credit d impot" is the same mention, not a second term.
        for term in sorted({stems(term) for term in terms}, key=len, reverse=True):
            if term.strip() and term in remaining:
                found += 1
                remaining = remaining.replace(term, " ")
        if found >= min_terms:
            hits[topic] = found
    # Best supported first, then by name once lowercased and stripped of
    # accents, the same order in both languages: a tagging run has to be
    # replayable, and a set has no order to replay.
    return sorted(hits, key=lambda topic: (-hits[topic], normalise(topic), topic))

/**
 * Étiqueter des articles depuis un vocabulaire contrôlé, par correspondance
 * après retrait des terminaisons courantes.
 *
 * Niveau N0. Déterministe, bibliothèque standard seule, et vérifiable : chaque
 * étiquette se remonte jusqu'au terme qui l'a produite, ce que demande un
 * rédacteur la première fois qu'une étiquette paraît fausse.
 *
 * Deux choses font que ça marche.
 *
 * D'abord, l'article et le vocabulaire passent par exactement la même chaîne. Un
 * racineur appliqué d'un seul côté échouera allègrement à faire correspondre un
 * mot avec lui-même.
 *
 * Ensuite, l'étiquetage est multi-étiquette par construction. Un article traite
 * plusieurs thèmes, ou aucun ; une fonction qui rend un thème par article répond
 * à une question que personne n'a posée.
 */

// Endings stripped, longest of any two that overlap first. This is a
// plural-and-suffix stripper, not a lemmatiser: "embauchons" does not become
// "embaucher", "fiscaux" does not become "fiscal", and "poste" and "post" fold
// onto the same stem. Both sides of the comparison get the same treatment.
const SUFFIXES = ['ements', 'ement', 'ations', 'ation', 'es', 's', 'x', 'e'];

const WORD = /[\p{L}\p{N}]+/gu;

/** Lowercase and drop accents, so "Fiscalité" and "FISCALITE" meet. */
export function normalise(text) {
  // NFD does not split the ligatures, and a soft hyphen or a zero-width space
  // (format characters) would cut a word in two: both are handled here.
  const unfolded = text.toLowerCase().replaceAll('œ', 'oe').replaceAll('æ', 'ae');
  return unfolded.normalize('NFD').replace(/[\p{M}\p{Cf}]+/gu, '');
}

/** Strip one ending, and only when a stem of three letters is left. */
export function stripEnding(word) {
  for (const suffix of SUFFIXES) {
    if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
      return word.slice(0, -suffix.length);
    }
  }
  return word;
}

/**
 * The text as a run of stems, padded with spaces at both ends.
 *
 * The padding is what lets a multi-word term be found with a plain substring
 * search: "impot" can then never match inside "impotent".
 */
export function stems(text) {
  const found = normalise(text).match(WORD) ?? [];
  return ` ${found.map(stripEnding).join(' ')} `;
}

/**
 * Every topic whose terms appear in the article, best supported first.
 *
 * `vocabulary` maps a topic name to the terms that stand for it, single or
 * multi-word. Keeping it a parameter is the point of this rung: the taxonomy
 * belongs to whoever edits the articles, not to the code.
 *
 * `minTerms` is how many distinct terms a topic needs before it is claimed.
 * Raise it when a single passing mention is not enough to file an article.
 */
export function tag(article, vocabulary, minTerms = 1) {
  if (!(minTerms >= 1)) throw new RangeError('minTerms must be at least 1: a topic needs a term');
  const haystack = stems(article);
  const hits = new Map();
  for (const [topic, terms] of Object.entries(vocabulary)) {
    let remaining = haystack;
    let found = 0;
    // Longest first, and a term found is removed from the text: "impot"
    // inside "credit d impot" is the same mention, not a second term.
    for (const term of [...new Set(terms.map(stems))].sort((a, b) => b.length - a.length)) {
      if (term.trim() && remaining.includes(term)) {
        found += 1;
        remaining = remaining.replaceAll(term, ' ');
      }
    }
    if (found >= minTerms) hits.set(topic, found);
  }
  // Best supported first, then by name once lowercased and stripped of
  // accents, the same order in both languages: a tagging run has to be
  // replayable, and a set has no order to replay.
  const byName = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  return [...hits.keys()].sort(
    (a, b) => hits.get(b) - hits.get(a) || byName(normalise(a), normalise(b)) || byName(a, b),
  );
}

"""
Étiquetage sans exemple : comparer l'article aux étiquettes elles-mêmes.

Niveau N2. Le niveau N1 demandait un corpus étiqueté, et un thème nouveau
voulait dire tout réétiqueter. Ici un thème n'est qu'une courte description,
encodée comme n'importe quel autre texte : en ajouter un coûte une ligne, et
le premier article peut être étiqueté le jour même.

C'est tout l'attrait du niveau, et il est réel. Ce qu'il coûte est un fichier
de modèle à livrer et à tenir à jour, un processus chaud pour le garder en
mémoire, et un score qui est un cosinus plutôt qu'une probabilité : il peut
être négatif, les thèmes d'un même article ne somment pas à un, et un seul
seuil doit servir tous les thèmes.
"""

from __future__ import annotations

import re

import numpy as np

MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"

# The encoder reads 128 tokens and silently drops the rest (`max_seq_length`
# in the model's sentence_bert_config.json). With its tokenizer, 400 characters
# of French prose come to 110 tokens, so the article is encoded in passages of
# that size. Text the tokenizer cuts finer, code or a table, can still overflow.
PASSAGE_CHARACTERS = 400

SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\s*\n\s*")


def load_encoder(name: str = MODEL_NAME):
    """The real encoder: fetched once, then held in memory and run locally."""
    from sentence_transformers import SentenceTransformer  # pragma: no cover

    return SentenceTransformer(name)


def build_labeller(topics: dict[str, str], encoder=None) -> dict:
    """
    Encode the topic descriptions once, and keep the encoder for the articles.

    `topics` maps a topic name to the sentence that stands for it. Write it as
    a human would say it out loud: the model card describes an encoder of
    sentences and paragraphs.

    `encoder` is injected so this can be tested without loading a model. Left
    alone, it is the real one above.
    """
    encoder = load_encoder() if encoder is None else encoder
    names = list(topics)
    vectors = _encode(encoder, [topics[name] for name in names])
    return {"topics": names, "vectors": vectors, "encoder": encoder}


def passages(article: str, size: int = PASSAGE_CHARACTERS) -> list[str]:
    """Consecutive sentences joined into pieces of at most `size` characters."""
    pieces: list[str] = []
    for sentence in filter(None, (s.strip() for s in SENTENCE_END.split(article.strip()))):
        if pieces and len(pieces[-1]) + 1 + len(sentence) <= size:
            pieces[-1] = f"{pieces[-1]} {sentence}"
        else:
            pieces.append(sentence)  # a sentence longer than `size` stays whole
    return pieces


def score(labeller: dict, article: str) -> dict[str, float]:
    """
    The best cosine, over the passages of the article, with each topic
    description: a topic covered in one paragraph counts.
    """
    pieces, known = passages(article), labeller["vectors"]
    if not pieces or not labeller["topics"]:
        return {topic: 0.0 for topic in labeller["topics"]}
    vectors = _encode(labeller["encoder"], pieces)
    if vectors.shape[1] != known.shape[1]:
        raise ValueError("the encoder returned vectors of another width for the article")
    # One row per passage, one column per topic: a matrix of cosines.
    best = (vectors @ known.T).max(axis=0)
    return {topic: float(value) for topic, value in zip(labeller["topics"], best)}


def tag(labeller: dict, article: str, threshold: float = 0.3) -> list[str]:
    """
    Every topic whose description is close enough, best first.

    Multi-label falls out of the shape of the thing: each topic is compared
    with the article on its own, so several can pass, or none.

    The threshold is not a probability. It is a cosine, it has no calibrated
    meaning, and choosing it by its effect on errors takes articles whose
    topics are already known — a labelled corpus, the very thing this rung was
    supposed to save you.
    """
    scored = score(labeller, article)
    kept = [topic for topic, value in scored.items() if value >= threshold]
    return sorted(kept, key=lambda topic: (-scored[topic], topic))


def _encode(encoder, texts: list[str]) -> np.ndarray:
    """
    One row of length one per text, or an error: a wrong count, a ragged
    table or a non-number is not a score. Cosine similarity is a dot product
    once both sides have length one.
    """
    raw = encoder.encode(texts)
    if len(raw) != len(texts):
        raise ValueError("the encoder returned a vector count that does not match the texts")
    if not texts:
        return np.zeros((0, 0))
    vectors = np.asarray(raw, dtype=float)  # raises on rows of different widths
    if vectors.ndim != 2 or not np.isfinite(vectors).all():
        raise ValueError("the encoder returned something that is not a table of finite numbers")
    norms = np.linalg.norm(vectors, axis=1, keepdims=True)
    return vectors / np.where(norms == 0, 1.0, norms)

/**
 * Étiquetage sans exemple : comparer l'article aux étiquettes elles-mêmes.
 *
 * Niveau N2. Le niveau N1 demandait un corpus étiqueté, et un thème nouveau
 * voulait dire tout réétiqueter. Ici un thème n'est qu'une courte description,
 * encodée comme n'importe quel autre texte : en ajouter un coûte une ligne, et
 * le premier article peut être étiqueté le jour même.
 *
 * C'est tout l'attrait du niveau, et il est réel. Ce qu'il coûte est un fichier
 * de modèle à livrer et à tenir à jour, un processus chaud pour le garder en
 * mémoire, et un score qui est un cosinus plutôt qu'une probabilité : il peut
 * être négatif, les thèmes d'un même article ne somment pas à un, et un seul
 * seuil doit servir tous les thèmes.
 */

// The ONNX conversion of sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2.
export const MODEL_NAME = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';

// The Python encoder reads 128 tokens and silently drops the rest. With this
// model's tokenizer, 400 characters of French prose come to 110 tokens, so the
// article is encoded in passages of that size, in both languages. Text the
// tokenizer cuts finer, code or a table, can still overflow.
export const PASSAGE_CHARACTERS = 400;

const SENTENCE_END = /(?<=[.!?])\s+|\s*\n\s*/;

/** The real encoder: fetched once, then held in memory and run locally. */
export async function loadEncoder(name = MODEL_NAME) {
  const { pipeline } = await import('@huggingface/transformers');
  // Full-precision weights, the ones Python loads, rather than a quantised copy.
  const extract = await pipeline('feature-extraction', name, { dtype: 'fp32' });
  return { encode: async (texts) => (await extract(texts, { pooling: 'mean' })).tolist() };
}

/**
 * Encode the topic descriptions once, and keep the encoder for the articles.
 *
 * `topics` maps a topic name to the sentence that stands for it. Write it as a
 * human would say it out loud: the model card describes an encoder of
 * sentences and paragraphs.
 *
 * `encoder` is injected so this can be tested without loading a model. Left
 * alone, it is the real one above.
 */
export async function buildLabeller(topics, encoder) {
  const model = encoder ?? (await loadEncoder());
  const names = Object.keys(topics);
  const vectors = await encode(model, names.map((name) => topics[name]));
  return { topics: names, vectors, encoder: model };
}

/** Consecutive sentences joined into pieces of at most `size` characters. */
export function passages(article, size = PASSAGE_CHARACTERS) {
  const pieces = [];
  for (const sentence of article.trim().split(SENTENCE_END).map((s) => s.trim()).filter(Boolean)) {
    if (pieces.length && pieces.at(-1).length + 1 + sentence.length <= size) {
      pieces[pieces.length - 1] = `${pieces.at(-1)} ${sentence}`;
    } else {
      pieces.push(sentence); // a sentence longer than `size` stays whole
    }
  }
  return pieces;
}

/**
 * The best cosine, over the passages of the article, with each topic
 * description: a topic covered in one paragraph counts.
 */
export async function score(labeller, article) {
  const pieces = passages(article);
  const vectors = pieces.length ? await encode(labeller.encoder, pieces) : [];
  const width = labeller.vectors[0]?.length;
  if (width !== undefined && vectors.some((vector) => vector.length !== width)) {
    throw new Error('the encoder returned vectors of another width for the article');
  }
  const scored = {};
  labeller.topics.forEach((topic, index) => {
    scored[topic] = vectors.length ? Math.max(...vectors.map((v) => dot(labeller.vectors[index], v))) : 0;
  });
  return scored;
}

/**
 * Every topic whose description is close enough, best first.
 *
 * Multi-label falls out of the shape of the thing: each topic is compared with
 * the article on its own, so several can pass, or none.
 *
 * The threshold is not a probability. It is a cosine, it has no calibrated
 * meaning, and choosing it by its effect on errors takes articles whose topics
 * are already known — a labelled corpus, the very thing this rung was supposed
 * to save you.
 */
export async function tag(labeller, article, threshold = 0.3) {
  const scored = await score(labeller, article);
  return labeller.topics
    .filter((topic) => scored[topic] >= threshold)
    .sort((a, b) => scored[b] - scored[a] || (a < b ? -1 : a > b ? 1 : 0));
}

/** One unit vector per text, or an error: a wrong count or a non-number is not a score. */
async function encode(encoder, texts) {
  const vectors = (await encoder.encode(texts)).map((vector) => Array.from(vector, Number));
  if (vectors.length !== texts.length || new Set(vectors.map((v) => v.length)).size > 1) {
    throw new Error('the encoder returned the wrong number or shape of vectors');
  }
  if (!vectors.every((vector) => vector.every(Number.isFinite))) {
    throw new Error('the encoder returned a value that is not a finite number');
  }
  return vectors.map(unit);
}

/** Cosine similarity is a dot product once both sides have length one. */
function unit(vector) {
  let sum = 0;
  for (const v of vector) sum += v * v;
  const norm = Math.sqrt(sum);
  return norm ? vector.map((v) => v / norm) : [...vector];
}

function dot(a, b) {
  let total = 0;
  for (let i = 0; i < a.length; i += 1) total += a[i] * b[i];
  return total;
}

"""
Étiqueter des articles en interrogeant un modèle généraliste.

Niveau N3. Sur cette fiche, c'est le seul niveau qui ne demande ni liste de
termes, ni corpus étiqueté, ni fichier de modèle. Il est là pour qu'on voie ce
qu'il coûte, pas parce que cette fiche le recommande.

Voyez ce que le code doit faire et que le niveau N0 n'avait pas à faire :
borner la taille de l'entrée, réessayer en cas d'échec, analyser une réponse
qui n'est que probablement du JSON valide, et ne garder que les thèmes qui
existent dans votre taxonomie. Ce dernier point n'est pas facultatif — rien
n'empêche un modèle de répondre un nom absent de la liste — et c'est pourquoi
le vocabulaire reste un paramètre ici, exactement comme au niveau le plus bas.

Cette plomberie est le vrai coût de ce niveau, et c'est la part que vos tests
doivent couvrir, parce qu'un test contre un double ne dit rien de la façon
dont le modèle étiquette.
"""

from __future__ import annotations

import json
import unicodedata

PROMPT = (
    "Tag the article below with the topics it covers.\n"
    "Choose only from this list, and answer with the spellings given:\n"
    "{topics}\n"
    "An article may cover several topics, or none at all.\n"
    "Answer with JSON only: a list of topic names, empty if none apply.\n\n"
    "Article:\n{article}"
)

MAX_CHARACTERS = 12000

# The provider named here is an example, not a recommendation: the reasoning
# holds for any general-purpose model API, and the client is swappable. Pass
# any object with a `complete(prompt=..., temperature=...)` method.
MODEL = "gpt-4.1-mini"  # an example id: check the parameters your model accepts


class ProviderClient:
    """The one call this snippet makes, on top of the provider's SDK."""

    def __init__(self, sdk=None, model: str = MODEL):
        if sdk is None:  # pragma: no cover - needs a key and a network
            from openai import OpenAI

            sdk = OpenAI()
        self.sdk, self.model = sdk, model

    def complete(self, *, prompt: str, temperature: float) -> str:
        response = self.sdk.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            temperature=temperature,
        )
        return response.choices[0].message.content


class TaggingUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def tag(article: str, topics: list[str], client=None, *, attempts: int = 3) -> list[str]:
    """
    The topics of the article, in the order of the taxonomy.

    `client` is injected so this function can be tested without a network call.
    In production it defaults to a real provider client.
    """
    # A model charges by the token. Refusing oversized input is not an
    # optimisation, it is a cost control.
    if len(article) > MAX_CHARACTERS:
        raise ValueError(f"article longer than {MAX_CHARACTERS} characters")
    # No text or no topic: the answer is known, and asking would still be billed.
    if not article.strip() or not topics:
        return []
    return _ask(client or ProviderClient(), article, topics, attempts)


def _key(name: str) -> str:
    """The same name whatever its case, its spacing or its Unicode form."""
    return unicodedata.normalize("NFC", name).strip().casefold()


def _ask(client, article: str, topics: list[str], attempts: int) -> list[str]:
    prompt = PROMPT.format(topics="\n".join(f"- {topic}" for topic in topics), article=article)
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero, the low end of the range, which the provider
            # documents as more focused and deterministic. It does not promise
            # that two identical calls agree.
            answer = client.complete(prompt=prompt, temperature=0)
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
            continue
        try:
            return _decode(answer, topics)
        except ValueError as error:  # an unusable answer is asked for again
            last_error = error
    raise TaggingUnavailable(str(last_error))


def _decode(answer, topics: list[str]) -> list[str]:
    """
    Keep only what the taxonomy knows, in the taxonomy's own order. A model
    that answers "actualité juridique" must not create a topic in your
    database, and the order of its answer must not decide the order of yours.
    """
    if not isinstance(answer, str):  # the SDK types the content as optional
        raise ValueError("the model returned no text")
    # A JSON answer wrapped whole in one code fence is read; nothing else is:
    # a fence opened and never closed, or prose around it, is a failed answer.
    text = answer.strip()
    if text.startswith("```") and text.endswith("```") and text.count("```") == 2:
        text = text[3:-3].removeprefix("json")
    try:
        parsed = json.loads(text)
    except RecursionError as error:
        raise ValueError("the answer is nested too deep to read") from error
    # A list of anything but names, or of names none of which is on the list,
    # must not pass for the legitimate empty answer.
    if not isinstance(parsed, list) or not all(isinstance(name, str) for name in parsed):
        raise ValueError("the model answered something that is not a list of names")
    answered = {_key(name) for name in parsed}
    kept = [topic for topic in topics if _key(topic) in answered]
    if parsed and not kept:
        raise ValueError("the model answered only names that are not in the taxonomy")
    return kept

/**
 * Étiqueter des articles en interrogeant un modèle généraliste.
 *
 * Niveau N3. Sur cette fiche, c'est le seul niveau qui ne demande ni liste de
 * termes, ni corpus étiqueté, ni fichier de modèle. Il est là pour qu'on voie ce
 * qu'il coûte, pas parce que cette fiche le recommande.
 *
 * Voyez ce que le code doit faire et que le niveau N0 n'avait pas à faire :
 * borner la taille de l'entrée, réessayer en cas d'échec, analyser une réponse
 * qui n'est que probablement du JSON valide, et ne garder que les thèmes qui
 * existent dans votre taxonomie. Ce dernier point n'est pas facultatif — rien
 * n'empêche un modèle de répondre un nom absent de la liste — et c'est pourquoi
 * le vocabulaire reste un paramètre ici, exactement comme au niveau le plus bas.
 *
 * Cette plomberie est le vrai coût de ce niveau, et c'est la part que vos tests
 * doivent couvrir, parce qu'un test contre un double ne dit rien de la façon
 * dont le modèle étiquette.
 */

const PROMPT = [
  'Tag the article below with the topics it covers.',
  'Choose only from this list, and answer with the spellings given:',
  '{topics}',
  'An article may cover several topics, or none at all.',
  'Answer with JSON only: a list of topic names, empty if none apply.',
  '',
  'Article:',
].join('\n');

export const MAX_CHARACTERS = 12000;

// The provider named here is an example, not a recommendation: the reasoning
// holds for any general-purpose model API, and the client is swappable. Pass
// any object with a `complete({ prompt, temperature })` method.
export const MODEL = 'gpt-4.1-mini'; // an example id: check the parameters your model accepts

export async function providerClient(sdk, model = MODEL) {
  if (!sdk) {
    const { OpenAI } = await import('openai');
    sdk = new OpenAI();
  }
  return {
    async complete({ prompt, temperature }) {
      const response = await sdk.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        temperature,
      });
      return response.choices[0].message.content;
    },
  };
}

export class TaggingUnavailable extends Error {}

/**
 * The topics of the article, in the order of the taxonomy.
 *
 * @param {string} article
 * @param {string[]} topics the controlled vocabulary, as on rung N0
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {number} [options.attempts]
 */
export async function tag(article, topics, { client, attempts = 3 } = {}) {
  // A model charges by the token. Refusing oversized input is not an
  // optimisation, it is a cost control. Counted in code points, as Python
  // counts characters.
  if ([...article].length > MAX_CHARACTERS) {
    throw new RangeError(`article longer than ${MAX_CHARACTERS} characters`);
  }
  // No text or no topic: the answer is known, and asking would still be billed.
  if (!article.trim() || topics.length === 0) return [];
  return ask(client ?? (await providerClient()), article, topics, attempts);
}

/** The same name whatever its case, its spacing or its Unicode form. */
const key = (name) => name.normalize('NFC').trim().toLowerCase();

async function ask(client, article, topics, attempts) {
  const prompt = `${PROMPT.replace('{topics}', topics.map((t) => `- ${t}`).join('\n'))}\n${article}`;
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    let answer;
    try {
      // Temperature zero, the low end of the range, which the provider
      // documents as more focused and deterministic. It does not promise
      // that two identical calls agree.
      answer = await client.complete({ prompt, temperature: 0 });
    } catch (error) {
      lastError = error; // any provider failure is retried
      continue;
    }
    try {
      return decode(answer, topics);
    } catch (error) {
      lastError = error; // an unusable answer is asked for again
    }
  }
  throw new TaggingUnavailable(String(lastError));
}

/**
 * Keep only what the taxonomy knows, in the taxonomy's own order. A model that
 * answers "actualité juridique" must not create a topic in your database, and
 * the order of its answer must not decide the order of yours.
 */
function decode(answer, topics) {
  if (typeof answer !== 'string') throw new Error('the model returned no text');
  // A JSON answer wrapped whole in one code fence is read; nothing else is: a
  // fence opened and never closed, or prose around it, is a failed answer.
  const text = answer.trim();
  const fenced = text.startsWith('```') && text.endsWith('```') && text.split('```').length === 3;
  const parsed = JSON.parse(fenced ? text.slice(3, -3).replace(/^json/, '') : text);
  // A list of anything but names, or of names none of which is on the list,
  // must not pass for the legitimate empty answer.
  if (!Array.isArray(parsed) || !parsed.every((name) => typeof name === 'string')) {
    throw new Error('the model answered something that is not a list of names');
  }
  const answered = new Set(parsed.map(key));
  const kept = topics.filter((topic) => answered.has(key(topic)));
  if (parsed.length && !kept.length) {
    throw new Error('the model answered only names that are not in the taxonomy');
  }
  return kept;
}