Chercher dans vos propres documents

"""
Chercher dans vos propres documents avec un index inversé et un BM25 que
vous avez écrits.

Niveau N1. Non parce que l'index de N0 serait mauvais — N0 reste la
recommandation — mais parce que le code ci-dessous est un BM25 qu'on peut
lire, et que le lire une fois vous dit pourquoi un document s'est classé là
où il s'est classé. Il ne reproduit pas FTS5 à la décimale près : la règle
d'appariement, l'IDF, la longueur et le poids du titre diffèrent, et chacun
est une ligne que vous pouvez changer.

Trois décisions vous appartiennent ici, qui appartenaient au moteur avant.

Le découpage en jetons : ce qui compte pour un mot, quels accents sont
repliés, quels termes sont écartés. La règle d'appariement : celle-ci garde
tout document qui porte au moins un terme, là où FTS5 les exige tous. Et le
classement : k1 sature la répétition, b corrige la longueur du document, et
les poids de champ disent ce que vaut un titre. Changez-en une et l'ordre
change ; c'est tout l'intérêt d'en être propriétaire.
"""

from __future__ import annotations

import math
import unicodedata
from collections import Counter

FIELD_WEIGHTS = {"title": 3.0, "body": 1.0}


def tokenise(text: str) -> list[str]:
    """Lower case, strip accents, keep letters and digits."""
    decomposed = unicodedata.normalize("NFKD", text.lower())
    letters = "".join(c for c in decomposed if not unicodedata.combining(c))
    return "".join(c if c.isalnum() else " " for c in letters).split()


def build_index(documents: list[dict], weights: dict | None = None) -> dict:
    """Build the postings: for each term, the documents carrying it."""
    weights = weights or FIELD_WEIGHTS
    postings: dict[str, dict[str, float]] = {}
    lengths: dict[str, float] = {}
    for document in documents:
        counts: Counter[str] = Counter()
        for field, weight in weights.items():
            for term in tokenise(document.get(field) or ""):  # a NULL column is empty
                counts[term] += weight
        for term, frequency in counts.items():
            postings.setdefault(term, {})[document["id"]] = frequency
        lengths[document["id"]] = sum(counts.values())
    average = sum(lengths.values()) / len(lengths) if lengths else 0.0
    return {"postings": postings, "lengths": lengths, "average_length": average}


def search(index: dict, query: str, limit: int = 5, k1: float = 1.2, b: float = 0.75) -> list[dict]:
    """Return the best matches, best first, each with the score it was given.

    `terms` says what each query word contributed. A ranking nobody can explain
    is a ranking nobody can fix.
    """
    if limit < 0:
        raise ValueError("limit must be zero or more")
    total = len(index["lengths"])
    scores: dict[str, float] = {}
    contributions: dict[str, dict[str, float]] = {}
    # dict.fromkeys keeps the order and drops repeats: a word typed twice is
    # not twice as important.
    for term in dict.fromkeys(tokenise(query)):
        postings = index["postings"].get(term)
        if not postings:
            continue
        # The rarer the term, the more a match on it means.
        idf = math.log(1 + (total - len(postings) + 0.5) / (len(postings) + 0.5))
        for doc_id, frequency in postings.items():
            norm = 1 - b + b * index["lengths"][doc_id] / index["average_length"]
            share = idf * frequency * (k1 + 1) / (frequency + k1 * norm)
            scores[doc_id] = scores.get(doc_id, 0.0) + share
            contributions.setdefault(doc_id, {})[term] = round(share, 4)
    ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
    return [
        {"id": doc_id, "score": round(score, 4), "terms": contributions[doc_id]}
        for doc_id, score in ranked[:limit]
    ]

/**
 * Chercher dans vos propres documents avec un index inversé et un BM25 que
 * vous avez écrits.
 *
 * Niveau N1. Non parce que l'index de N0 serait mauvais — N0 reste la
 * recommandation — mais parce que le code ci-dessous est un BM25 qu'on peut
 * lire, et que le lire une fois vous dit pourquoi un document s'est classé là
 * où il s'est classé. Il ne reproduit pas FTS5 à la décimale près : la règle
 * d'appariement, l'IDF, la longueur et le poids du titre diffèrent, et chacun
 * est une ligne que vous pouvez changer.
 *
 * Trois décisions vous appartiennent ici, qui appartenaient au moteur avant.
 *
 * Le découpage en jetons : ce qui compte pour un mot, quels accents sont
 * repliés, quels termes sont écartés. La règle d'appariement : celle-ci garde
 * tout document qui porte au moins un terme, là où FTS5 les exige tous. Et le
 * classement : k1 sature la répétition, b corrige la longueur du document, et
 * les poids de champ disent ce que vaut un titre. Changez-en une et l'ordre
 * change ; c'est tout l'intérêt d'en être propriétaire.
 */

export const FIELD_WEIGHTS = { title: 3, body: 1 };

/** Lower case, strip accents, keep letters and digits. */
export function tokenise(text) {
  return String(text).toLowerCase().normalize('NFKD')
    .replace(/\p{Diacritic}/gu, '')
    .split(/[^\p{L}\p{N}]+/u)
    .filter(Boolean);
}

/** Build the postings: for each term, the documents carrying it. */
export function buildIndex(documents, weights = FIELD_WEIGHTS) {
  const postings = new Map(); // term -> Map(document id -> weighted frequency)
  const lengths = new Map();
  for (const document of documents) {
    const counts = new Map();
    for (const [field, weight] of Object.entries(weights)) {
      for (const term of tokenise(document[field] ?? '')) {
        counts.set(term, (counts.get(term) ?? 0) + weight);
      }
    }
    for (const [term, frequency] of counts) {
      if (!postings.has(term)) postings.set(term, new Map());
      postings.get(term).set(document.id, frequency);
    }
    lengths.set(document.id, [...counts.values()].reduce((sum, v) => sum + v, 0));
  }
  const total = [...lengths.values()].reduce((sum, v) => sum + v, 0);
  return { postings, lengths, averageLength: lengths.size ? total / lengths.size : 0 };
}

/**
 * Return the best matches, best first, each with the score it was given.
 *
 * `terms` says what each query word contributed. A ranking nobody can explain
 * is a ranking nobody can fix.
 */
export function search(index, query, { limit = 5, k1 = 1.2, b = 0.75 } = {}) {
  // A negative slice would silently drop the last results: refuse it.
  if (limit < 0) throw new RangeError('limit must be zero or more');
  const scores = new Map();
  const contributions = new Map();
  // A Set keeps the order and drops repeats: a word typed twice is not twice
  // as important.
  for (const term of new Set(tokenise(query))) {
    const postings = index.postings.get(term);
    if (!postings) continue;
    // The rarer the term, the more a match on it means.
    const idf = Math.log(1 + (index.lengths.size - postings.size + 0.5) / (postings.size + 0.5));
    for (const [id, frequency] of postings) {
      const norm = 1 - b + (b * index.lengths.get(id)) / index.averageLength;
      const share = (idf * frequency * (k1 + 1)) / (frequency + k1 * norm);
      scores.set(id, (scores.get(id) ?? 0) + share);
      if (!contributions.has(id)) contributions.set(id, {});
      contributions.get(id)[term] = Math.round(share * 1e4) / 1e4;
    }
  }
  return [...scores.entries()]
    .sort((a, b2) => b2[1] - a[1] || (a[0] < b2[0] ? -1 : 1))
    .slice(0, limit)
    .map(([id, score]) => ({ id, score: Math.round(score * 1e4) / 1e4, terms: contributions.get(id) }));
}

"""
Ajouter une jambe vectorielle à la recherche plein texte que vous avez déjà.

Niveau N2. N0 et N1 apparient des mots. Un lecteur qui demande « vacances »
là où le manuel dit « congés payés » n'obtient rien, et c'est ce trou que ce
niveau existe pour combler : un encodeur de phrases auto-hébergé, conçu pour
la recherche sémantique, projette chaque texte sur un vecteur dense, et les
textes se comparent sur ces vecteurs plutôt que sur les mots qu'ils
partagent.

Il le comble en plus, non à la place. La recherche par mots-clés trouve ce
qui partage les mots et se tait sinon ; la recherche vectorielle classe tous
les documents, toujours. Fusionner les deux classements garde le premier
résultat du plein texte devant toute page que les mots n'ont pas trouvée, et
emprunte la portée des vecteurs ; plus bas dans la liste, chaque jambe pèse
autant que l'autre.

Ce que ce niveau coûte vraiment n'est pas l'arithmétique ci-dessous. C'est le
déploiement : des poids à télécharger et à charger, un processus à tenir
chaud, et les vecteurs d'un document à recalculer quand il change — à côté
d'un index plein texte que la base entretient déjà pour rien.
"""

from __future__ import annotations

import functools
import math
import weakref

# A multilingual model: its card lists French among its languages, and the
# handbook in the tests is French.
MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"

# Per encoder, the unit vector of each document text seen in the last search:
# a document is encoded again only once its text has changed.
_document_vectors: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()


class EncodingFailed(RuntimeError):
    """The encoder could not be run, or returned something unusable."""


@functools.cache
def default_encoder():  # pragma: no cover - loads the weights
    """Loaded once per process, then kept warm."""
    from sentence_transformers import SentenceTransformer

    return SentenceTransformer(MODEL_NAME)


def unit(vector) -> list[float]:
    """Normalise once, so that a cosine is a dot product afterwards."""
    values = [float(v) for v in vector]
    norm = sum(v * v for v in values) ** 0.5
    return [v / norm for v in values] if norm else values


def vector_ranking(query: str, documents: list[dict], encoder) -> list[str]:
    """Rank every document by cosine similarity to the query."""
    texts = [f"{d['title']} {d['body']}" for d in documents]
    known = _document_vectors.get(encoder, {})
    missing = [text for text in dict.fromkeys(texts) if text not in known]
    try:
        # One batched call: the query travels with the documents not yet encoded.
        vectors = [unit(v) for v in encoder.encode(missing + [query])]
    except Exception as error:  # noqa: BLE001 - a model failure is not the caller's fault
        raise EncodingFailed(str(error)) from error
    if len(vectors) != len(missing) + 1:
        raise EncodingFailed(f"{len(vectors)} vectors returned for {len(missing) + 1} texts")
    *fresh, query_vector = vectors
    known = {**known, **dict(zip(missing, fresh))}
    kept = {text: known[text] for text in texts}
    size = len(query_vector)
    if not all(len(v) == size and all(map(math.isfinite, v)) for v in [*kept.values(), query_vector]):
        raise EncodingFailed("vectors of different sizes, or values that are not finite numbers")
    _document_vectors[encoder] = kept

    similarities = [
        (document["id"], sum(x * y for x, y in zip(kept[text], query_vector)))
        for document, text in zip(documents, texts)
    ]
    # Ties are broken on the identifier, so two runs give the same order.
    similarities.sort(key=lambda pair: (-pair[1], pair[0]))
    return [doc_id for doc_id, _ in similarities]


def hybrid_search(query, documents, keyword_ids, encoder=None, *, limit=5, k=60) -> list[dict]:
    """Fuse the ranking your full-text search returned with a vector ranking.

    `keyword_ids` is what N0 already gave you, best first.
    """
    if not documents:
        return []
    if encoder is None:
        encoder = default_encoder()

    # Reciprocal rank fusion: each list votes with 1/(k + rank). Nothing has to
    # be rescaled, because a BM25 score and a cosine are never comparable, and
    # k says how much being second is worth compared with being first.
    scores: dict[str, float] = {}
    for ranking in (list(keyword_ids), vector_ranking(query, documents, encoder)):
        for rank, doc_id in enumerate(ranking, start=1):
            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
    ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
    return [{"id": doc_id, "score": round(score, 6)} for doc_id, score in ranked[:limit]]

/**
 * Ajouter une jambe vectorielle à la recherche plein texte que vous avez déjà.
 *
 * Niveau N2. N0 et N1 apparient des mots. Un lecteur qui demande « vacances »
 * là où le manuel dit « congés payés » n'obtient rien, et c'est ce trou que ce
 * niveau existe pour combler : un encodeur de phrases auto-hébergé, conçu pour
 * la recherche sémantique, projette chaque texte sur un vecteur dense, et les
 * textes se comparent sur ces vecteurs plutôt que sur les mots qu'ils
 * partagent.
 *
 * Il le comble en plus, non à la place. La recherche par mots-clés trouve ce
 * qui partage les mots et se tait sinon ; la recherche vectorielle classe tous
 * les documents, toujours. Fusionner les deux classements garde le premier
 * résultat du plein texte devant toute page que les mots n'ont pas trouvée, et
 * emprunte la portée des vecteurs ; plus bas dans la liste, chaque jambe pèse
 * autant que l'autre.
 *
 * Ce que ce niveau coûte vraiment n'est pas l'arithmétique ci-dessous. C'est le
 * déploiement : des poids à télécharger et à charger, un processus à tenir
 * chaud, et les vecteurs d'un document à recalculer quand il change — à côté
 * d'un index plein texte que la base entretient déjà pour rien.
 */

// A multilingual model: its card lists French among its languages, and the
// handbook in the tests is French.
export const MODEL_NAME = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';

// Per encoder, the unit vector of each document text seen in the last search:
// a document is encoded again only once its text has changed.
const documentVectors = new WeakMap();

export class EncodingFailed extends Error {}

let defaultEncoder;

/** Loaded once per process, then kept warm. */
function loadDefaultEncoder() {
  defaultEncoder ??= (async () => {
    const { pipeline } = await import('@huggingface/transformers');
    const extract = await pipeline('feature-extraction', MODEL_NAME);
    return { encode: async (batch) => (await extract(batch, { pooling: 'mean' })).tolist() };
  })().catch((error) => {
    defaultEncoder = undefined; // a failed load is tried again on the next search
    throw error;
  });
  return defaultEncoder;
}

/** Normalise once, so that a cosine is a dot product afterwards. */
export function unit(vector) {
  const values = Array.from(vector, Number);
  const norm = Math.hypot(...values);
  return norm ? values.map((value) => value / norm) : values;
}

/** Rank every document by cosine similarity to the query. */
export async function vectorRanking(query, documents, encoder) {
  const texts = documents.map((document) => `${document.title} ${document.body}`);
  const known = documentVectors.get(encoder) ?? new Map();
  const missing = [...new Set(texts)].filter((text) => !known.has(text));
  let vectors;
  try {
    // One batched call: the query travels with the documents not yet encoded.
    vectors = (await encoder.encode([...missing, query])).map(unit);
  } catch (error) {
    throw new EncodingFailed(String(error));
  }
  if (vectors.length !== missing.length + 1) {
    throw new EncodingFailed(`${vectors.length} vectors returned for ${missing.length + 1} texts`);
  }
  const queryVector = vectors.pop();
  // A copy, not the served map: a vector that turns out to be unusable must not
  // stay behind in the cache, where it would make this page unfindable, and
  // every later search that touches it fail, until the process restarts.
  // Python writes into a new dictionary for the same reason.
  const fresh = new Map(known);
  missing.forEach((text, i) => fresh.set(text, vectors[i]));
  const kept = new Map(texts.map((text) => [text, fresh.get(text)]));
  const usable = (v) => v.length === queryVector.length && v.every(Number.isFinite);
  if (![...kept.values(), queryVector].every(usable)) {
    throw new EncodingFailed('vectors of different sizes, or values that are not finite numbers');
  }
  documentVectors.set(encoder, kept);

  const similarities = documents.map((document, i) => [
    document.id,
    kept.get(texts[i]).reduce((sum, value, d) => sum + value * queryVector[d], 0),
  ]);
  // Ties are broken on the identifier, so two runs give the same order.
  similarities.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
  return similarities.map(([id]) => id);
}

/**
 * Fuse the ranking your full-text search returned with a vector ranking.
 *
 * @param {string} query
 * @param {object[]} documents
 * @param {string[]} keywordIds what N0 already gave you, best first
 * @param {object} options
 * @param {{encode: Function}} [options.encoder] injected so this can be tested
 *   without downloading a model; defaults to a real local encoder
 */
export async function hybridSearch(query, documents, keywordIds, { encoder, limit = 5, k = 60 } = {}) {
  if (documents.length === 0) return [];
  encoder ??= await loadDefaultEncoder();

  // Reciprocal rank fusion: each list votes with 1/(k + rank). Nothing has to
  // be rescaled, because a BM25 score and a cosine are never comparable, and k
  // says how much being second is worth compared with being first.
  const scores = new Map();
  const rankings = [[...keywordIds], await vectorRanking(query, documents, encoder)];
  for (const ranking of rankings) {
    ranking.forEach((id, index) => {
      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + index + 1));
    });
  }
  return [...scores.entries()]
    .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
    .slice(0, limit)
    .map(([id, score]) => ({ id, score: Math.round(score * 1e6) / 1e6 }));
}

"""
Répondre à une question à partir de vos propres documents : génération
augmentée par récupération.

Niveau N3. Le modèle ne connaît de votre manuel que ce que la consigne lui
apporte. La récupération passe donc d'abord — N0, N1 ou N2 trouve les
passages — et le modèle écrit la phrase à partir d'eux.

Tout ce qui suit est de la plomberie, et la plomberie est là où sont les
bogues : combien de passages envoyer et quelle longueur chacun peut faire,
ce que le modèle a le droit de dire quand ils ne répondent pas, comment
décoder une réponse qui n'est que probablement du JSON, et quoi faire quand
il cite un passage que personne ne lui a envoyé.

Ce dernier contrôle mérite ses lignes, et il vaut de savoir exactement ce
qu'il achète : il attrape une source inventée. Il ne peut pas attraper une
phrase inventée accrochée à une source réelle : il compare des
identifiants, et ne lit jamais la phrase.
"""

from __future__ import annotations

import json

# The provider named here is an example, not a recommendation: the reasoning
# holds for any general-purpose model API, and the client is swappable. Pass
# any object with a `complete(prompt=..., temperature=...)` method.
MODEL = "gpt-4.1-mini"  # an example id: check the parameters your model accepts

PROMPT = (
    "Answer the question using only the passages below.\n"
    "If they do not contain the answer, answer exactly: {no_answer}\n"
    'Answer with JSON only: {{"answer": "...", "sources": ["id", ...]}},\n'
    "where every id is one of the passage ids you were given.\n\n"
    "Passages:\n{passages}\n\n"
    "Question: {question}"
)

MAX_PASSAGES = 4
MAX_CHARACTERS = 1500  # per passage
MAX_QUESTION = 1000  # characters
NO_ANSWER = "je ne sais pas"


class ProviderClient:
    """The one call this snippet makes, on top of the provider's SDK."""

    def __init__(self, sdk=None, model: str = MODEL):
        if sdk is None:  # pragma: no cover - needs a key and a network
            from openai import OpenAI

            sdk = OpenAI()
        self.sdk, self.model = sdk, model

    def complete(self, *, prompt: str, temperature: float) -> str:
        response = self.sdk.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            temperature=temperature,
        )
        return response.choices[0].message.content


class AnswerUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


class AnswerNotGrounded(Exception):
    """The answer cites a passage that was never sent, or cites nothing."""


def answer(question: str, passages: list[dict], client=None, *, attempts: int = 2,
           max_passages: int = MAX_PASSAGES) -> dict:
    """`passages` are the retrieved dicts with keys id and text, best first."""
    # The provider bills every character of the prompt: refuse before any call.
    if len(question) > MAX_QUESTION:
        raise ValueError(f"question longer than {MAX_QUESTION} characters")
    kept = passages[:max_passages]
    if not kept or not question.strip():
        # Nothing retrieved, or nothing asked. There is nothing to answer from,
        # and no reason to pay for a call that can only invent.
        return {"answer": NO_ANSWER, "sources": []}

    client = client or ProviderClient()
    block = "\n\n".join(f"[{p['id']}] {p['text'][:MAX_CHARACTERS]}" for p in kept)
    reply = _ask(client, PROMPT.format(no_answer=NO_ANSWER, passages=block, question=question),
                 attempts)

    text = reply["answer"].strip()
    sent = {str(p["id"]): p["id"] for p in kept}  # ids may be integers: compare as text
    unknown = [source for source in reply["sources"] if str(source) not in sent]
    if unknown:
        raise AnswerNotGrounded(f"the model cited {unknown}, which it was never sent")
    sources = [sent[str(source)] for source in reply["sources"]]
    if not text:
        raise AnswerUnavailable("the model answered without an answer")
    if text.lower().rstrip(" .!") == NO_ANSWER:  # "Je ne sais pas." is the same answer
        return {"answer": NO_ANSWER, "sources": sources}
    if not sources:
        raise AnswerNotGrounded("an answer that cites nothing cannot be checked")
    return {"answer": text, "sources": sources}


def _ask(client, prompt: str, attempts: int) -> dict:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero narrows the sampling. It does not make the call
            # deterministic: the provider documents chat completions as
            # non-deterministic by default, and the entry says so.
            parsed = _decode(client.complete(prompt=prompt, temperature=0))
            if isinstance(parsed, dict) and isinstance(parsed.get("answer"), str) \
                    and isinstance(parsed.get("sources"), list) \
                    and all(isinstance(s, (str, int)) and not isinstance(s, bool) for s in parsed["sources"]):
                return parsed
            last_error = ValueError("the model answered something that is not the object asked for")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise AnswerUnavailable(str(last_error))


def _decode(reply) -> object:
    """
    JSON, or JSON wrapped whole in one code fence. No text at all (a refusal)
    is unusable, and so is anything else around the object: prose before or
    after it, a second block, or a fence opened and never closed. Salvaging
    those would be guessing which part of a badly shaped answer to believe.
    """
    if not isinstance(reply, str):
        raise ValueError("the model returned no text")
    text = reply.strip()
    if text.startswith("```") and text.endswith("```") and text.count("```") == 2:
        text = text[3:-3].removeprefix("json")
    return json.loads(text)

/**
 * Répondre à une question à partir de vos propres documents : génération
 * augmentée par récupération.
 *
 * Niveau N3. Le modèle ne connaît de votre manuel que ce que la consigne lui
 * apporte. La récupération passe donc d'abord — N0, N1 ou N2 trouve les
 * passages — et le modèle écrit la phrase à partir d'eux.
 *
 * Tout ce qui suit est de la plomberie, et la plomberie est là où sont les
 * bogues : combien de passages envoyer et quelle longueur chacun peut faire,
 * ce que le modèle a le droit de dire quand ils ne répondent pas, comment
 * décoder une réponse qui n'est que probablement du JSON, et quoi faire quand
 * il cite un passage que personne ne lui a envoyé.
 *
 * Ce dernier contrôle mérite ses lignes, et il vaut de savoir exactement ce
 * qu'il achète : il attrape une source inventée. Il ne peut pas attraper une
 * phrase inventée accrochée à une source réelle : il compare des
 * identifiants, et ne lit jamais la phrase.
 */

// The provider named here is an example, not a recommendation: the reasoning
// holds for any general-purpose model API, and the client is swappable. Pass
// any object with a `complete({ prompt, temperature })` method.
export const MODEL = 'gpt-4.1-mini'; // an example id: check the parameters your model accepts

export async function providerClient(sdk, model = MODEL) {
  if (!sdk) {
    const { OpenAI } = await import('openai');
    sdk = new OpenAI();
  }
  return {
    async complete({ prompt, temperature }) {
      const response = await sdk.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        temperature,
      });
      return response.choices[0].message.content;
    },
  };
}

export const MAX_PASSAGES = 4;
export const MAX_CHARACTERS = 1500; // per passage
export const MAX_QUESTION = 1000; // characters
export const NO_ANSWER = 'je ne sais pas';

const PROMPT = (passages, question) => [
  'Answer the question using only the passages below.',
  `If they do not contain the answer, answer exactly: ${NO_ANSWER}`,
  'Answer with JSON only: {"answer": "...", "sources": ["id", ...]},',
  'where every id is one of the passage ids you were given.',
  '',
  'Passages:',
  passages,
  '',
  `Question: ${question}`,
].join('\n');

export class AnswerUnavailable extends Error {}

export class AnswerNotGrounded extends Error {}

/** The first `max` characters, counted in code points as Python does, so no emoji is cut in two. */
const head = (text, max) => [...text.slice(0, 2 * max)].slice(0, max).join('');

/**
 * @param {string} question
 * @param {{id: string|number, text: string}[]} passages retrieved, best first
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 */
export async function answer(question, passages, { client, attempts = 2, maxPassages = MAX_PASSAGES } = {}) {
  // The provider bills every character of the prompt: refuse before any call.
  if ([...question].length > MAX_QUESTION) {
    throw new RangeError(`question longer than ${MAX_QUESTION} characters`);
  }
  const kept = passages.slice(0, maxPassages);
  if (kept.length === 0 || !question.trim()) {
    // Nothing retrieved, or nothing asked. There is nothing to answer from,
    // and no reason to pay for a call that can only invent.
    return { answer: NO_ANSWER, sources: [] };
  }

  client ??= await providerClient();
  const block = kept.map((p) => `[${p.id}] ${head(p.text, MAX_CHARACTERS)}`).join('\n\n');
  const reply = await ask(client, PROMPT(block, question), attempts);

  const text = reply.answer.trim();
  const sent = new Map(kept.map((p) => [String(p.id), p.id])); // ids may be numbers: compare as text
  const unknown = reply.sources.filter((source) => !sent.has(String(source)));
  if (unknown.length > 0) {
    throw new AnswerNotGrounded(`the model cited ${unknown}, which it was never sent`);
  }
  const sources = reply.sources.map((source) => sent.get(String(source)));
  if (!text) throw new AnswerUnavailable('the model answered without an answer');
  if (text.toLowerCase().replace(/[ .!]+$/, '') === NO_ANSWER) { // "Je ne sais pas." is the same answer
    return { answer: NO_ANSWER, sources };
  }
  if (sources.length === 0) {
    throw new AnswerNotGrounded('an answer that cites nothing cannot be checked');
  }
  return { answer: text, sources };
}

async function ask(client, prompt, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      // Temperature zero narrows the sampling. It does not make the call
      // deterministic: the provider documents chat completions as
      // non-deterministic by default, and the entry says so.
      const parsed = decode(await client.complete({ prompt, temperature: 0 }));
      const isId = (s) => typeof s === 'string' || Number.isInteger(s);
      if (typeof parsed?.answer === 'string' && Array.isArray(parsed.sources) && parsed.sources.every(isId)) {
        return parsed;
      }
      lastError = new Error('the model answered something that is not the object asked for');
    } catch (error) {
      lastError = error;
    }
  }
  throw new AnswerUnavailable(String(lastError));
}

/**
 * JSON, or JSON wrapped whole in one code fence. No text at all (a refusal) is
 * unusable, and so is anything else around the object: prose before or after
 * it, a second block, or a fence opened and never closed. Salvaging those would
 * be guessing which part of a badly shaped answer to believe.
 */
function decode(reply) {
  if (typeof reply !== 'string') throw new Error('the model returned no text');
  const trimmed = reply.trim();
  const fences = trimmed.match(/```/g)?.length ?? 0;
  if (trimmed.startsWith('```') && trimmed.endsWith('```') && fences === 2) {
    const inner = trimmed.slice(3, -3);
    return JSON.parse(inner.startsWith('json') ? inner.slice(4) : inner);
  }
  return JSON.parse(trimmed);
}