Search in your own documents

"""
Search your own documents with the full-text index of the database you
already run.

Rung N0. SQLite ships FTS5: a virtual table holding an inverted index, and a
bm25() ranking function. Postgres has tsvector and ts_rank, MySQL has
FULLTEXT ... IN NATURAL LANGUAGE MODE. Whatever is under your application
already does this, and does it well.

Two things are worth knowing before you use it.

First, bm25() returns a negative number, the best match being the most
negative. Negate it, as below, and a bigger score means a better match again.

Second, MATCH takes a query language, not a string. A user typing a double
quote, AND or NEAR must never be handed to it raw: quoting each token turns
the query back into plain words, and turns a syntax error into a search.
"""

import sqlite3
import unicodedata

# A title match counts for more than a body match. bm25() takes one weight per
# column, in the order the columns were declared.
COLUMN_WEIGHTS = (0.0, 10.0, 1.0)  # doc_id, title, body

CREATE = (
    "CREATE VIRTUAL TABLE documents USING fts5("
    "doc_id UNINDEXED, title, body, tokenize='unicode61 remove_diacritics 2')"
)


def tokenise(text: str) -> list[str]:
    """Lower case, strip accents, keep letters and digits.

    The same folding as the tokenizer declared above, so what we look up is
    spelled the way the index stored it.
    """
    decomposed = unicodedata.normalize("NFKD", text.lower())
    letters = "".join(c for c in decomposed if not unicodedata.combining(c))
    return "".join(c if c.isalnum() else " " for c in letters).split()


def build_index(documents: list[dict]) -> sqlite3.Connection:
    """Documents are dicts with keys id, title and body.

    In memory here so the snippet runs alone; in production it is a table in
    the database you already back up, filled by a trigger or a nightly job.
    """
    connection = sqlite3.connect(":memory:")
    connection.execute(CREATE)
    connection.executemany(
        "INSERT INTO documents (doc_id, title, body) VALUES (?, ?, ?)",
        [(d["id"], d["title"], d["body"]) for d in documents],
    )
    return connection


def search(connection: sqlite3.Connection, query: str, limit: int = 5) -> list[dict]:
    """Return the best matches, best first, as dicts with keys id and score."""
    terms = tokenise(query)
    if not terms:
        # An empty MATCH is a syntax error, not an empty result set.
        return []
    # Quoted terms, separated by a space: FTS5 requires all of them to appear.
    match = " ".join(f'"{term}"' for term in terms)
    rows = connection.execute(
        "SELECT doc_id, -bm25(documents, ?, ?, ?) AS score FROM documents "
        "WHERE documents MATCH ? ORDER BY score DESC, doc_id LIMIT ?",
        (*COLUMN_WEIGHTS, match, limit),
    ).fetchall()
    return [{"id": doc_id, "score": round(score, 4)} for doc_id, score in rows]

/**
 * Search your own documents with the full-text index of the database you
 * already run.
 *
 * Rung N0. The Python version of this snippet is four SQL statements against
 * SQLite's FTS5: a virtual table holding an inverted index, and a bm25()
 * ranking function. Postgres has tsvector and ts_rank, MySQL has FULLTEXT ...
 * IN NATURAL LANGUAGE MODE. Whatever is under your application already does
 * this, and does it well: on this rung you write SQL, not an algorithm.
 *
 * Node 22 does ship `node:sqlite`, but it needs --experimental-sqlite and the
 * bundled build has no FTS5 module, so there is nothing to call from a plain
 * `node` process. This file therefore writes out what the FTS5 table does:
 * the same tokenizer (lower case, accents folded), the same implicit AND
 * between terms, the same BM25 with the same constants and column weights.
 * Read it as the documentation of the SQL, not as something to deploy.
 */

// SQLite's fts5 defaults. Its bm25() is negated so that ORDER BY works
// ascending; the scores below are the plain ones, bigger being better.
const K1 = 1.2;
const B = 0.75;

// A title match counts for more than a body match.
const COLUMN_WEIGHTS = { title: 10, body: 1 };

/** Lower case, strip accents, keep letters and digits. */
export function tokenise(text) {
  return String(text).toLowerCase().normalize('NFKD')
    .replace(/\p{Diacritic}/gu, '')
    .split(/[^\p{L}\p{N}]+/u)
    .filter(Boolean);
}

/** Documents are objects with keys id, title and body. */
export function buildIndex(documents) {
  const documentFrequency = new Map();
  const rows = documents.map((document) => {
    const counts = new Map(); // term -> frequency, weighted by column
    let length = 0;
    for (const [field, weight] of Object.entries(COLUMN_WEIGHTS)) {
      for (const term of tokenise(document[field] ?? '')) {
        counts.set(term, (counts.get(term) ?? 0) + weight);
        length += 1;
      }
    }
    for (const term of counts.keys()) {
      documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
    }
    return { id: document.id, counts, length };
  });
  const total = rows.reduce((sum, row) => sum + row.length, 0);
  return { rows, documentFrequency, averageLength: rows.length ? total / rows.length : 0 };
}

/** Return the best matches, best first, as objects with keys id and score. */
export function search(index, query, limit = 5) {
  const terms = tokenise(query);
  if (terms.length === 0) return [];

  const results = [];
  for (const row of index.rows) {
    // Implicit AND: a document missing one term of the query is not a result.
    if (!terms.every((term) => row.counts.has(term))) continue;
    let score = 0;
    for (const term of terms) {
      const hits = index.documentFrequency.get(term);
      // A term carried by more than half the documents separates nothing.
      // fts5 floors its weight rather than letting it go negative.
      const idf = Math.max(Math.log((index.rows.length - hits + 0.5) / (hits + 0.5)), 1e-6);
      const frequency = row.counts.get(term);
      const norm = 1 - B + (B * row.length) / index.averageLength;
      score += (idf * frequency * (K1 + 1)) / (frequency + K1 * norm);
    }
    results.push({ id: row.id, score });
  }
  results.sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : 1));
  return results.slice(0, limit).map(({ id, score }) => ({ id, score: Math.round(score * 1e4) / 1e4 }));
}

"""
Search your own documents with an inverted index and a BM25 you wrote.

Rung N1. Not because the index of N0 is bad — N0 is still the recommendation —
but because the forty lines below are what the database does, and reading them
once tells you why a document ranked where it did.

Three decisions are yours here, and they were the engine's before.

The tokenizer: what counts as a word, which accents are folded, which terms are
dropped. The matching rule: this one keeps any document carrying at least one
term, where FTS5 demands all of them. And the ranking: k1 saturates repetition,
b corrects for document length, and the field weights say how much a title is
worth. Change one and the order changes; that is the point of owning it.
"""

from __future__ import annotations

import math
import unicodedata
from collections import Counter

FIELD_WEIGHTS = {"title": 3.0, "body": 1.0}


def tokenise(text: str) -> list[str]:
    """Lower case, strip accents, keep letters and digits."""
    decomposed = unicodedata.normalize("NFKD", text.lower())
    letters = "".join(c for c in decomposed if not unicodedata.combining(c))
    return "".join(c if c.isalnum() else " " for c in letters).split()


def build_index(documents: list[dict], weights: dict | None = None) -> dict:
    """Build the postings: for each term, the documents carrying it."""
    weights = weights or FIELD_WEIGHTS
    postings: dict[str, dict[str, float]] = {}
    lengths: dict[str, float] = {}
    for document in documents:
        counts: Counter[str] = Counter()
        for field, weight in weights.items():
            for term in tokenise(document.get(field, "")):
                counts[term] += weight
        for term, frequency in counts.items():
            postings.setdefault(term, {})[document["id"]] = frequency
        lengths[document["id"]] = sum(counts.values())
    average = sum(lengths.values()) / len(lengths) if lengths else 0.0
    return {"postings": postings, "lengths": lengths, "average_length": average}


def search(index: dict, query: str, limit: int = 5, k1: float = 1.2, b: float = 0.75) -> list[dict]:
    """Return the best matches, best first, each with the score it was given.

    `terms` says what each query word contributed. A ranking nobody can explain
    is a ranking nobody can fix.
    """
    total = len(index["lengths"])
    scores: dict[str, float] = {}
    contributions: dict[str, dict[str, float]] = {}
    # dict.fromkeys keeps the order and drops repeats: a word typed twice is
    # not twice as important.
    for term in dict.fromkeys(tokenise(query)):
        postings = index["postings"].get(term)
        if not postings:
            continue
        # The rarer the term, the more a match on it means.
        idf = math.log(1 + (total - len(postings) + 0.5) / (len(postings) + 0.5))
        for doc_id, frequency in postings.items():
            norm = 1 - b + b * index["lengths"][doc_id] / index["average_length"]
            share = idf * frequency * (k1 + 1) / (frequency + k1 * norm)
            scores[doc_id] = scores.get(doc_id, 0.0) + share
            contributions.setdefault(doc_id, {})[term] = round(share, 4)
    ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
    return [
        {"id": doc_id, "score": round(score, 4), "terms": contributions[doc_id]}
        for doc_id, score in ranked[:limit]
    ]

/**
 * Search your own documents with an inverted index and a BM25 you wrote.
 *
 * Rung N1. Not because the index of N0 is bad — N0 is still the
 * recommendation — but because the forty lines below are what the database
 * does, and reading them once tells you why a document ranked where it did.
 *
 * Three decisions are yours here, and they were the engine's before.
 *
 * The tokenizer: what counts as a word, which accents are folded, which terms
 * are dropped. The matching rule: this one keeps any document carrying at
 * least one term, where FTS5 demands all of them. And the ranking: k1
 * saturates repetition, b corrects for document length, and the field weights
 * say how much a title is worth. Change one and the order changes; that is the
 * point of owning it.
 */

export const FIELD_WEIGHTS = { title: 3, body: 1 };

/** Lower case, strip accents, keep letters and digits. */
export function tokenise(text) {
  return String(text).toLowerCase().normalize('NFKD')
    .replace(/\p{Diacritic}/gu, '')
    .split(/[^\p{L}\p{N}]+/u)
    .filter(Boolean);
}

/** Build the postings: for each term, the documents carrying it. */
export function buildIndex(documents, weights = FIELD_WEIGHTS) {
  const postings = new Map(); // term -> Map(document id -> weighted frequency)
  const lengths = new Map();
  for (const document of documents) {
    const counts = new Map();
    for (const [field, weight] of Object.entries(weights)) {
      for (const term of tokenise(document[field] ?? '')) {
        counts.set(term, (counts.get(term) ?? 0) + weight);
      }
    }
    for (const [term, frequency] of counts) {
      if (!postings.has(term)) postings.set(term, new Map());
      postings.get(term).set(document.id, frequency);
    }
    lengths.set(document.id, [...counts.values()].reduce((sum, v) => sum + v, 0));
  }
  const total = [...lengths.values()].reduce((sum, v) => sum + v, 0);
  return { postings, lengths, averageLength: lengths.size ? total / lengths.size : 0 };
}

/**
 * Return the best matches, best first, each with the score it was given.
 *
 * `terms` says what each query word contributed. A ranking nobody can explain
 * is a ranking nobody can fix.
 */
export function search(index, query, { limit = 5, k1 = 1.2, b = 0.75 } = {}) {
  const scores = new Map();
  const contributions = new Map();
  // A Set keeps the order and drops repeats: a word typed twice is not twice
  // as important.
  for (const term of new Set(tokenise(query))) {
    const postings = index.postings.get(term);
    if (!postings) continue;
    // The rarer the term, the more a match on it means.
    const idf = Math.log(1 + (index.lengths.size - postings.size + 0.5) / (postings.size + 0.5));
    for (const [id, frequency] of postings) {
      const norm = 1 - b + (b * index.lengths.get(id)) / index.averageLength;
      const share = (idf * frequency * (k1 + 1)) / (frequency + k1 * norm);
      scores.set(id, (scores.get(id) ?? 0) + share);
      if (!contributions.has(id)) contributions.set(id, {});
      contributions.get(id)[term] = Math.round(share * 1e4) / 1e4;
    }
  }
  return [...scores.entries()]
    .sort((a, b2) => b2[1] - a[1] || (a[0] < b2[0] ? -1 : 1))
    .slice(0, limit)
    .map(([id, score]) => ({ id, score: Math.round(score * 1e4) / 1e4, terms: contributions.get(id) }));
}

"""
Add a vector leg to the full-text search you already have.

Rung N2. N0 and N1 match words. A reader who asks for "vacances" where the
handbook says "congés payés" gets nothing, and that gap is what this rung
exists to close: a self-hosted encoder maps text to vectors where two ways of
saying the same thing land close together.

It closes it in addition, not instead. Keyword search is exact when the words
match and silent when they do not; vector search always answers, and is vague.
Fusing the two rankings keeps the precision of the first and borrows the reach
of the second, which is why the entry calls this a complement.

What the rung really costs is not the arithmetic below. It is the deployment: a
few hundred megabytes of weights, a process kept warm, vectors recomputed
whenever a document changes, and a vector index as soon as they stop fitting in
a list — next to a full-text index the database already maintains for free.
"""

from __future__ import annotations

# A multilingual model, because a handbook is rarely written in English.
MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"


class EncodingFailed(RuntimeError):
    """The encoder could not be run, or returned something unusable."""


def unit(vector) -> list[float]:
    """Normalise once, so that a cosine is a dot product afterwards."""
    values = [float(v) for v in vector]
    norm = sum(v * v for v in values) ** 0.5
    return [v / norm for v in values] if norm else values


def vector_ranking(query: str, documents: list[dict], encoder) -> list[str]:
    """Rank every document by cosine similarity to the query."""
    texts = [f"{d['title']} {d['body']}" for d in documents]
    try:
        # One batched call: the query travels with the documents.
        vectors = encoder.encode(texts + [query])
    except Exception as error:  # noqa: BLE001 - a model failure is not the caller's fault
        raise EncodingFailed(str(error)) from error
    if len(vectors) != len(texts) + 1:
        raise EncodingFailed(f"{len(vectors)} vectors returned for {len(texts) + 1} texts")

    *document_vectors, query_vector = [unit(v) for v in vectors]
    similarities = [
        (document["id"], sum(x * y for x, y in zip(vector, query_vector)))
        for document, vector in zip(documents, document_vectors)
    ]
    # Ties are broken on the identifier, so two runs give the same order.
    similarities.sort(key=lambda pair: (-pair[1], pair[0]))
    return [doc_id for doc_id, _ in similarities]


def hybrid_search(query, documents, keyword_ids, encoder=None, *, limit=5, k=60) -> list[dict]:
    """Fuse the ranking your full-text search returned with a vector ranking.

    `keyword_ids` is what N0 already gave you, best first.
    """
    if not documents:
        return []
    if encoder is None:  # pragma: no cover - loads several hundred megabytes
        from sentence_transformers import SentenceTransformer

        encoder = SentenceTransformer(MODEL_NAME)

    # Reciprocal rank fusion: each list votes with 1/(k + rank). Nothing has to
    # be rescaled, because a BM25 score and a cosine are never comparable, and
    # k says how much being second is worth compared with being first.
    scores: dict[str, float] = {}
    for ranking in (list(keyword_ids), vector_ranking(query, documents, encoder)):
        for rank, doc_id in enumerate(ranking, start=1):
            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
    ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
    return [{"id": doc_id, "score": round(score, 6)} for doc_id, score in ranked[:limit]]

/**
 * Add a vector leg to the full-text search you already have.
 *
 * Rung N2. N0 and N1 match words. A reader who asks for "vacances" where the
 * handbook says "congés payés" gets nothing, and that gap is what this rung
 * exists to close: a self-hosted encoder maps text to vectors where two ways
 * of saying the same thing land close together.
 *
 * It closes it in addition, not instead. Keyword search is exact when the
 * words match and silent when they do not; vector search always answers, and
 * is vague. Fusing the two rankings keeps the precision of the first and
 * borrows the reach of the second, which is why the entry calls this a
 * complement.
 *
 * What the rung really costs is not the arithmetic below. It is the
 * deployment: a few hundred megabytes of weights, a process kept warm, vectors
 * recomputed whenever a document changes, and a vector index as soon as they
 * stop fitting in an array — next to a full-text index the database already
 * maintains for free.
 */

// A multilingual model, because a handbook is rarely written in English.
export const MODEL_NAME = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';

export class EncodingFailed extends Error {}

/** Normalise once, so that a cosine is a dot product afterwards. */
export function unit(vector) {
  const norm = Math.hypot(...vector);
  return norm ? vector.map((value) => value / norm) : [...vector];
}

/** Rank every document by cosine similarity to the query. */
export async function vectorRanking(query, documents, encoder) {
  const texts = documents.map((document) => `${document.title} ${document.body}`);
  let vectors;
  try {
    // One batched call: the query travels with the documents.
    vectors = await encoder.encode([...texts, query]);
  } catch (error) {
    throw new EncodingFailed(String(error));
  }
  if (vectors.length !== texts.length + 1) {
    throw new EncodingFailed(`${vectors.length} vectors returned for ${texts.length + 1} texts`);
  }

  const normalised = vectors.map(unit);
  const queryVector = normalised.at(-1);
  const similarities = documents.map((document, i) => [
    document.id,
    normalised[i].reduce((sum, value, d) => sum + value * queryVector[d], 0),
  ]);
  // Ties are broken on the identifier, so two runs give the same order.
  similarities.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
  return similarities.map(([id]) => id);
}

/**
 * Fuse the ranking your full-text search returned with a vector ranking.
 *
 * @param {string} query
 * @param {object[]} documents
 * @param {string[]} keywordIds what N0 already gave you, best first
 * @param {object} options
 * @param {{encode: Function}} [options.encoder] injected so this can be tested
 *   without downloading a model; defaults to a real local encoder
 */
export async function hybridSearch(query, documents, keywordIds, { encoder, limit = 5, k = 60 } = {}) {
  if (documents.length === 0) return [];
  if (!encoder) {
    // Downloads and loads the weights, so it is never reached in the tests.
    const { pipeline } = await import('@xenova/transformers');
    const extract = await pipeline('feature-extraction', MODEL_NAME);
    encoder = { encode: async (batch) => (await extract(batch, { pooling: 'mean' })).tolist() };
  }

  // Reciprocal rank fusion: each list votes with 1/(k + rank). Nothing has to
  // be rescaled, because a BM25 score and a cosine are never comparable, and k
  // says how much being second is worth compared with being first.
  const scores = new Map();
  const rankings = [[...keywordIds], await vectorRanking(query, documents, encoder)];
  for (const ranking of rankings) {
    ranking.forEach((id, index) => {
      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + index + 1));
    });
  }
  return [...scores.entries()]
    .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
    .slice(0, limit)
    .map(([id, score]) => ({ id, score: Math.round(score * 1e6) / 1e6 }));
}

"""
Answer a question from your own documents: retrieval-augmented generation.

Rung N3. The model has never read your handbook. Asked without it, it answers
from memory, confidently, about your company. So retrieval comes first — N0,
N1 or N2 finds the passages — and the model only writes the sentence.

Everything below is plumbing, and the plumbing is where the bugs are: how many
passages to send and how long each may be, what the model is allowed to say
when they do not answer, how to decode a reply that is only probably JSON, and
what to do when it cites a passage nobody sent.

That last check earns its lines, and it is worth knowing exactly what it buys:
it catches an invented source. It cannot catch an invented sentence hung on a
real one, and no amount of prompting turns it into a check that can.
"""

from __future__ import annotations

import json

PROMPT = (
    "Answer the question using only the passages below.\n"
    "If they do not contain the answer, answer exactly: {no_answer}\n"
    'Answer with JSON only: {{"answer": "...", "sources": ["id", ...]}},\n'
    "where every id is one of the passage ids you were given.\n\n"
    "Passages:\n{passages}\n\n"
    "Question: {question}"
)

MAX_PASSAGES = 4
MAX_CHARACTERS = 1500  # per passage
NO_ANSWER = "je ne sais pas"


class AnswerUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


class AnswerNotGrounded(Exception):
    """The answer cites a passage that was never sent, or cites nothing."""


def answer(question: str, passages: list[dict], client=None, *, attempts: int = 2,
           max_passages: int = MAX_PASSAGES) -> dict:
    """`passages` are the retrieved dicts with keys id and text, best first."""
    if client is None:  # pragma: no cover - needs a key and a network
        from openai import OpenAI

        client = OpenAI()

    kept = passages[:max_passages]
    if not kept:
        # Retrieval found nothing. There is nothing to answer from, and no
        # reason to pay for a call that can only invent.
        return {"answer": NO_ANSWER, "sources": []}

    block = "\n\n".join(f"[{p['id']}] {p['text'][:MAX_CHARACTERS]}" for p in kept)
    reply = _ask(client, PROMPT.format(no_answer=NO_ANSWER, passages=block, question=question),
                 attempts)

    text = str(reply.get("answer", "")).strip()
    sources = [str(source) for source in reply.get("sources", [])]
    unknown = [source for source in sources if source not in {p["id"] for p in kept}]
    if unknown:
        raise AnswerNotGrounded(f"the model cited {unknown}, which it was never sent")
    if text and text != NO_ANSWER and not sources:
        raise AnswerNotGrounded("an answer that cites nothing cannot be checked")
    if not text:
        raise AnswerUnavailable("the model answered without an answer")
    return {"answer": text, "sources": sources}


def _ask(client, prompt: str, attempts: int) -> dict:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero: two identical questions must give one answer,
            # or nobody can review what the thing told a customer.
            parsed = json.loads(client.complete(prompt=prompt, temperature=0))
            if isinstance(parsed, dict):
                return parsed
            last_error = ValueError("the model answered something that is not an object")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise AnswerUnavailable(str(last_error))

/**
 * Answer a question from your own documents: retrieval-augmented generation.
 *
 * Rung N3. The model has never read your handbook. Asked without it, it
 * answers from memory, confidently, about your company. So retrieval comes
 * first — N0, N1 or N2 finds the passages — and the model only writes the
 * sentence.
 *
 * Everything below is plumbing, and the plumbing is where the bugs are: how
 * many passages to send and how long each may be, what the model is allowed to
 * say when they do not answer, how to decode a reply that is only probably
 * JSON, and what to do when it cites a passage nobody sent.
 *
 * That last check earns its lines, and it is worth knowing exactly what it
 * buys: it catches an invented source. It cannot catch an invented sentence
 * hung on a real one, and no amount of prompting turns it into a check that
 * can.
 */

export const MAX_PASSAGES = 4;
export const MAX_CHARACTERS = 1500; // per passage
export const NO_ANSWER = 'je ne sais pas';

const PROMPT = (passages, question) => [
  'Answer the question using only the passages below.',
  `If they do not contain the answer, answer exactly: ${NO_ANSWER}`,
  'Answer with JSON only: {"answer": "...", "sources": ["id", ...]},',
  'where every id is one of the passage ids you were given.',
  '',
  'Passages:',
  passages,
  '',
  `Question: ${question}`,
].join('\n');

export class AnswerUnavailable extends Error {}

export class AnswerNotGrounded extends Error {}

/**
 * @param {string} question
 * @param {{id: string, text: string}[]} passages retrieved, best first
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 */
export async function answer(question, passages, { client, attempts = 2, maxPassages = MAX_PASSAGES } = {}) {
  if (!client) {
    // Needs a key and a network, so it is never reached in the tests.
    const { OpenAI } = await import('openai');
    client = new OpenAI();
  }

  const kept = passages.slice(0, maxPassages);
  if (kept.length === 0) {
    // Retrieval found nothing. There is nothing to answer from, and no reason
    // to pay for a call that can only invent.
    return { answer: NO_ANSWER, sources: [] };
  }

  const block = kept.map((p) => `[${p.id}] ${p.text.slice(0, MAX_CHARACTERS)}`).join('\n\n');
  const reply = await ask(client, PROMPT(block, question), attempts);

  const text = String(reply.answer ?? '').trim();
  const sources = (reply.sources ?? []).map(String);
  const known = new Set(kept.map((p) => p.id));
  const unknown = sources.filter((source) => !known.has(source));
  if (unknown.length > 0) {
    throw new AnswerNotGrounded(`the model cited ${unknown}, which it was never sent`);
  }
  if (text && text !== NO_ANSWER && sources.length === 0) {
    throw new AnswerNotGrounded('an answer that cites nothing cannot be checked');
  }
  if (!text) throw new AnswerUnavailable('the model answered without an answer');
  return { answer: text, sources };
}

async function ask(client, prompt, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      // Temperature zero: two identical questions must give one answer, or
      // nobody can review what the thing told a customer.
      const parsed = JSON.parse(await client.complete({ prompt, temperature: 0 }));
      if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed;
      lastError = new Error('the model answered something that is not an object');
    } catch (error) {
      lastError = error;
    }
  }
  throw new AnswerUnavailable(String(lastError));
}