Summarise a long document

"""
Summarise a long document by choosing its own best sentences.

Rung N0. Extractive: every sentence of the summary appears verbatim in the
source, because this code never writes a word, it only selects.

Two classical signals, and nothing else.

Term frequency. A word the document keeps coming back to is what the document
is about, so a sentence dense in such words carries more of the subject than a
sentence made of connectives. Dividing by the length of the sentence measures
density rather than volume, which stops a long sentence from winning by size.

Position. An author states the subject early. A small bonus that decays with
the rank of the sentence encodes that habit, without handing the summary to
the opening paragraph outright.
"""

import re

# A sentence ends at a full stop, question or exclamation mark followed by
# whitespace. Abbreviations will fool this; a real corpus needs a better
# splitter, and that is a separate problem from choosing sentences.
SENTENCE_END = re.compile(r"(?<=[.!?])\s+")

# Letters and digits only: `[^\W_]` is `\w` without the underscore, so
# accented words survive and punctuation does not.
WORD = re.compile(r"[^\W_]+")

# Words too common to say anything about the subject of a document.
STOPWORDS = frozenset(
    "a an and are as at be been but by for from had has have in into is it its "
    "of on or that the their there they this to was were which will with".split()
)

# How much the opening of the document is worth. Large enough to break a tie
# between two equally dense sentences, too small to win on its own.
LEAD_BONUS = 0.15


def split_sentences(text: str) -> list[str]:
    """Cut the document into sentences, dropping empty ones."""
    parts = SENTENCE_END.split(text.strip())
    return [part.strip() for part in parts if part.strip()]


def _term_weights(sentences: list[str]) -> dict[str, float]:
    """Count content words, then scale so the most frequent one weighs one."""
    counts: dict[str, int] = {}
    for sentence in sentences:
        for word in WORD.findall(sentence.lower()):
            if len(word) > 2 and word not in STOPWORDS:
                counts[word] = counts.get(word, 0) + 1
    if not counts:
        return {}
    most = max(counts.values())
    return {word: count / most for word, count in counts.items()}


def score_sentences(sentences: list[str]) -> list[float]:
    """Density in the document's own vocabulary, plus the position bonus."""
    weights = _term_weights(sentences)
    scores = []
    for index, sentence in enumerate(sentences):
        words = WORD.findall(sentence.lower())
        # Accumulated in a plain loop rather than with `sum`, which since
        # Python 3.12 compensates rounding error on floats. That is the better
        # answer, but it is not the answer JavaScript gives, and the two
        # versions of this snippet have to rank sentences identically.
        total = 0.0
        for word in words:
            total += weights.get(word, 0.0)
        density = total / len(words) if words else 0.0
        scores.append(density + LEAD_BONUS / (index + 1))
    return scores


def summarise(text: str, max_sentences: int = 3) -> str:
    """
    Return the best sentences, in the order the document puts them.

    Ordering the summary by score would read as a list of quotations. Keeping
    document order keeps the sequence the author chose, which is the only part
    of the argument an extractive summary can preserve.
    """
    sentences = split_sentences(text)
    scores = score_sentences(sentences)
    # Sorting is stable, so two identical scores keep their document order.
    ranked = sorted(range(len(sentences)), key=lambda i: -scores[i])
    chosen = sorted(ranked[:max_sentences])
    return " ".join(sentences[index] for index in chosen)

/**
 * Summarise a long document by choosing its own best sentences.
 *
 * Rung N0. Extractive: every sentence of the summary appears verbatim in the
 * source, because this code never writes a word, it only selects.
 *
 * Two classical signals, and nothing else.
 *
 * Term frequency. A word the document keeps coming back to is what the
 * document is about, so a sentence dense in such words carries more of the
 * subject than a sentence made of connectives. Dividing by the length of the
 * sentence measures density rather than volume, which stops a long sentence
 * from winning by size.
 *
 * Position. An author states the subject early. A small bonus that decays
 * with the rank of the sentence encodes that habit, without handing the
 * summary to the opening paragraph outright.
 */

// A sentence ends at a full stop, question or exclamation mark followed by
// whitespace. Abbreviations will fool this; a real corpus needs a better
// splitter, and that is a separate problem from choosing sentences.
const SENTENCE_END = /(?<=[.!?])\s+/;

// Letters and digits only, so accented words survive and punctuation does
// not. The Python counterpart writes the same class as `[^\W_]`.
const WORD = /[\p{L}\p{N}]+/gu;

// Words too common to say anything about the subject of a document.
const STOPWORDS = new Set(
  ('a an and are as at be been but by for from had has have in into is it its ' +
    'of on or that the their there they this to was were which will with').split(' '),
);

// How much the opening of the document is worth. Large enough to break a tie
// between two equally dense sentences, too small to win on its own.
const LEAD_BONUS = 0.15;

/** Cut the document into sentences, dropping empty ones. */
export function splitSentences(text) {
  return text
    .trim()
    .split(SENTENCE_END)
    .map((part) => part.trim())
    .filter(Boolean);
}

/** Count content words, then scale so the most frequent one weighs one. */
function termWeights(sentences) {
  const counts = new Map();
  for (const sentence of sentences) {
    for (const word of sentence.toLowerCase().match(WORD) ?? []) {
      if (word.length > 2 && !STOPWORDS.has(word)) {
        counts.set(word, (counts.get(word) ?? 0) + 1);
      }
    }
  }
  if (counts.size === 0) return new Map();
  const most = Math.max(...counts.values());
  return new Map([...counts].map(([word, count]) => [word, count / most]));
}

/** Density in the document's own vocabulary, plus the position bonus. */
export function scoreSentences(sentences) {
  const weights = termWeights(sentences);
  return sentences.map((sentence, index) => {
    const words = sentence.toLowerCase().match(WORD) ?? [];
    let total = 0;
    for (const word of words) total += weights.get(word) ?? 0;
    const density = words.length ? total / words.length : 0;
    return density + LEAD_BONUS / (index + 1);
  });
}

/**
 * Return the best sentences, in the order the document puts them.
 *
 * Ordering the summary by score would read as a list of quotations. Keeping
 * document order keeps the sequence the author chose, which is the only part
 * of the argument an extractive summary can preserve.
 */
export function summarise(text, maxSentences = 3) {
  const sentences = splitSentences(text);
  const scores = scoreSentences(sentences);
  // Sorting is stable, so two identical scores keep their document order.
  const ranked = sentences.map((_, index) => index).sort((a, b) => scores[b] - scores[a]);
  const chosen = ranked.slice(0, maxSentences).sort((a, b) => a - b);
  return chosen.map((index) => sentences[index]).join(' ');
}

"""
Score sentences with a classifier trained on surface features.

Rung N1. Same extractive shape as N0 — the summary is still made of the
document's own sentences — but the weights are learnt instead of guessed.

N0 fixes the trade between position and term density by hand, once, for every
document in the world. That constant is a guess. Here, a few dozen documents
whose summary sentences someone has ticked off decide it instead: if, in your
corpus, the wrap-up at the end matters more than the opening, the model will
find that out and N0 never will.

The features are deliberately surface-level. They describe where a sentence
sits and what it looks like, not what it means. That is the ceiling of this
rung, and the reason the entry does not stop here.
"""

import re

import numpy as np
from sklearn.linear_model import LogisticRegression

SENTENCE_END = re.compile(r"(?<=[.!?])\s+")
WORD = re.compile(r"[^\W_]+")

STOPWORDS = frozenset(
    "a an and are as at be been but by for from had has have in into is it its "
    "of on or that the their there they this to was were which will with".split()
)

# Words an author uses when about to state the point of what came before.
CUES = frozenset(
    "overall therefore total conclusion result summary altogether finally "
    "consequently".split()
)

# Sentences longer than this are already long; the feature saturates rather
# than letting one outlier stretch the scale for every other sentence.
LONG_SENTENCE = 25


def split_sentences(text: str) -> list[str]:
    """Cut the document into sentences, dropping empty ones."""
    parts = SENTENCE_END.split(text.strip())
    return [part.strip() for part in parts if part.strip()]


def sentence_features(sentences: list[str], index: int) -> list[float]:
    """
    Five things a reader notices before reading: where the sentence sits, how
    long it is, whether it carries a figure, whether it announces a conclusion,
    and how much of the opening it repeats.
    """
    sentence = sentences[index]
    words = WORD.findall(sentence.lower())
    opening = set(WORD.findall(sentences[0].lower()))
    content = [w for w in words if len(w) > 2 and w not in STOPWORDS]
    shared = sum(1 for word in content if word in opening)
    return [
        1.0 / (index + 1),
        min(len(words) / LONG_SENTENCE, 1.0),
        1.0 if any(c in "0123456789" for c in sentence) else 0.0,
        1.0 if any(word in CUES for word in words) else 0.0,
        shared / len(content) if content else 0.0,
    ]


def train(documents: list[list[str]], labels: list[list[int]]) -> LogisticRegression:
    """
    `documents` are lists of sentences; `labels[d][i]` is 1 when sentence `i`
    of document `d` belongs in the summary.

    Ticking sentences in a few dozen documents is an afternoon of work, and it
    is the entire training set. Nothing here needs a GPU or a corpus.
    """
    rows, targets = [], []
    for sentences, marks in zip(documents, labels):
        for index in range(len(sentences)):
            rows.append(sentence_features(sentences, index))
            targets.append(marks[index])
    return LogisticRegression(max_iter=2000).fit(np.array(rows), np.array(targets))


def summarise(model: LogisticRegression, text: str, max_sentences: int = 3) -> str:
    """Return the best-scored sentences, in the order the document puts them."""
    sentences = split_sentences(text)
    if not sentences:
        return ""
    rows = np.array([sentence_features(sentences, i) for i in range(len(sentences))])
    scores = model.predict_proba(rows)[:, 1]
    # Sorting is stable, so two identical scores keep their document order.
    ranked = sorted(range(len(sentences)), key=lambda i: -scores[i])
    chosen = sorted(ranked[:max_sentences])
    return " ".join(sentences[index] for index in chosen)

/**
 * Score sentences with a classifier trained on surface features.
 *
 * Rung N1. Same extractive shape as N0 — the summary is still made of the
 * document's own sentences — but the weights are learnt instead of guessed.
 *
 * N0 fixes the trade between position and term density by hand, once, for
 * every document in the world. That constant is a guess. Here, a few dozen
 * documents whose summary sentences someone has ticked off decide it instead:
 * if, in your corpus, the wrap-up at the end matters more than the opening,
 * the model will find that out and N0 never will.
 *
 * The features are deliberately surface-level. They describe where a sentence
 * sits and what it looks like, not what it means. That is the ceiling of this
 * rung, and the reason the entry does not stop here.
 *
 * Logistic regression is written out rather than pulled from a package,
 * because on five features it is a dozen lines. The Python version of this
 * snippet calls scikit-learn; the objective minimised below is the same one,
 * so both rank a document's sentences alike.
 */

const SENTENCE_END = /(?<=[.!?])\s+/;
const WORD = /[\p{L}\p{N}]+/gu;

const STOPWORDS = new Set(
  ('a an and are as at be been but by for from had has have in into is it its ' +
    'of on or that the their there they this to was were which will with').split(' '),
);

// Words an author uses when about to state the point of what came before.
const CUES = new Set(
  ('overall therefore total conclusion result summary altogether finally ' +
    'consequently').split(' '),
);

// Sentences longer than this are already long; the feature saturates rather
// than letting one outlier stretch the scale for every other sentence.
const LONG_SENTENCE = 25;

/** Cut the document into sentences, dropping empty ones. */
export function splitSentences(text) {
  return text
    .trim()
    .split(SENTENCE_END)
    .map((part) => part.trim())
    .filter(Boolean);
}

/**
 * Five things a reader notices before reading: where the sentence sits, how
 * long it is, whether it carries a figure, whether it announces a conclusion,
 * and how much of the opening it repeats.
 */
export function sentenceFeatures(sentences, index) {
  const sentence = sentences[index];
  const words = sentence.toLowerCase().match(WORD) ?? [];
  const opening = new Set(sentences[0].toLowerCase().match(WORD) ?? []);
  const content = words.filter((w) => w.length > 2 && !STOPWORDS.has(w));
  const shared = content.filter((word) => opening.has(word)).length;
  return [
    1 / (index + 1),
    Math.min(words.length / LONG_SENTENCE, 1),
    /[0-9]/.test(sentence) ? 1 : 0,
    words.some((word) => CUES.has(word)) ? 1 : 0,
    content.length ? shared / content.length : 0,
  ];
}

/**
 * `documents` are lists of sentences; `labels[d][i]` is 1 when sentence `i` of
 * document `d` belongs in the summary.
 *
 * Full-batch gradient descent on the mean log-loss, plus the same L2 penalty
 * scikit-learn applies by default. The bias is left unpenalised, as it is
 * there too. The objective is strictly convex, so enough steps land on the one
 * optimum whichever language walks towards it.
 */
export function train(documents, labels, { epochs = 4000, rate = 1, strength = 1 } = {}) {
  const rows = [];
  const targets = [];
  for (const [d, sentences] of documents.entries()) {
    for (let i = 0; i < sentences.length; i += 1) {
      rows.push(sentenceFeatures(sentences, i));
      targets.push(labels[d][i]);
    }
  }
  const width = rows[0].length;
  const weights = new Array(width).fill(0);
  let bias = 0;
  const penalty = 1 / (strength * rows.length);

  for (let epoch = 0; epoch < epochs; epoch += 1) {
    const gradient = new Array(width).fill(0);
    let biasGradient = 0;
    for (let i = 0; i < rows.length; i += 1) {
      let z = bias;
      for (let j = 0; j < width; j += 1) z += weights[j] * rows[i][j];
      const error = 1 / (1 + Math.exp(-z)) - targets[i];
      for (let j = 0; j < width; j += 1) gradient[j] += (error * rows[i][j]) / rows.length;
      biasGradient += error / rows.length;
    }
    for (let j = 0; j < width; j += 1) weights[j] -= rate * (gradient[j] + penalty * weights[j]);
    bias -= rate * biasGradient;
  }
  return { weights, bias };
}

/** Probability that a sentence belongs in the summary. */
function score(model, features) {
  let z = model.bias;
  for (let j = 0; j < features.length; j += 1) z += model.weights[j] * features[j];
  return 1 / (1 + Math.exp(-z));
}

/** Return the best-scored sentences, in the order the document puts them. */
export function summarise(model, text, maxSentences = 3) {
  const sentences = splitSentences(text);
  if (sentences.length === 0) return '';
  const scores = sentences.map((_, i) => score(model, sentenceFeatures(sentences, i)));
  // Sorting is stable, so two identical scores keep their document order.
  const ranked = sentences.map((_, index) => index).sort((a, b) => scores[b] - scores[a]);
  const chosen = ranked.slice(0, maxSentences).sort((a, b) => a - b);
  return chosen.map((index) => sentences[index]).join(' ');
}

"""
Summarise a long document with a self-hosted abstractive model.

Rung N2. The first rung that writes. N0 and N1 choose sentences and can only
ever return what the author already wrote; this one produces a sentence that
was not in the document, which is the only way to state a conclusion drawn
from two passages ten pages apart.

That gain has a price, and the price is in this file rather than in the model.
A sequence-to-sequence summariser reads a fixed-size window. A long document
does not fit, so it has to be cut at sentence boundaries, summarised piece by
piece, and the pieces summarised again. Every one of those calls can fail or
come back empty, and a summary that is silently half a document is worse than
no summary at all.

What this file cannot do, at any price: check that what the model wrote is
what the document said. Nothing in this plumbing can. See the test.
"""

from __future__ import annotations

import re

SENTENCE_END = re.compile(r"(?<=[.!?])\s+")

MODEL_NAME = "facebook/bart-large-cnn"

# What one pass of the model is allowed to read. Beyond its window the model
# truncates without saying so, and a summary of the first half of a chunk is
# indistinguishable from a summary of all of it.
CHUNK_CHARACTERS = 3000

# What the whole function is allowed to read. A self-hosted model costs
# machine time rather than money, but a document nobody meant to send is still
# better refused than churned through in silence.
MAX_CHARACTERS = 200_000


class SummaryUnavailable(Exception):
    """The model failed, or returned nothing usable."""


class LocalSummariser:
    """
    The real model, loaded from local weights. Loaded once and kept for the
    life of the process: it is the loading that is slow, not the summarising.
    """

    def __init__(self, name: str = MODEL_NAME) -> None:  # pragma: no cover - loads weights
        from transformers import pipeline

        self._pipeline = pipeline("summarization", model=name)

    def generate(self, text: str) -> str:  # pragma: no cover - loads weights
        return self._pipeline(text, truncation=True)[0]["summary_text"]


def chunk(text: str, size: int = CHUNK_CHARACTERS) -> list[str]:
    """
    Cut the document into pieces that fit the model's window, at sentence
    boundaries.

    A sentence longer than the window on its own is passed whole and the model
    will truncate it. Cutting mid-sentence to avoid that would hand the model
    half a clause, which is a worse thing to summarise.
    """
    pieces: list[str] = []
    current = ""
    for sentence in SENTENCE_END.split(text.strip()):
        sentence = sentence.strip()
        if not sentence:
            continue
        if current and len(current) + 1 + len(sentence) > size:
            pieces.append(current)
            current = sentence
        else:
            current = f"{current} {sentence}".strip()
    if current:
        pieces.append(current)
    return pieces


def summarise(text: str, model=None, *, attempts: int = 2) -> str:
    """
    `model` is injected so this function can be tested without loading weights.
    In production it defaults to the real model.
    """
    if model is None:  # pragma: no cover - loads weights
        model = LocalSummariser()

    if len(text) > MAX_CHARACTERS:
        raise ValueError(f"document longer than {MAX_CHARACTERS} characters")

    pieces = chunk(text)
    if not pieces:
        return ""

    notes = [_generate(model, piece, attempts) for piece in pieces]
    if len(notes) == 1:
        return notes[0]

    # Second pass: the model reads back its own notes. Concatenating them
    # instead would give a text as long as the number of chunks, which is not
    # a summary of the document but a summary of each of its parts.
    return _generate(model, " ".join(notes), attempts)


def _generate(model, text: str, attempts: int) -> str:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            written = (model.generate(text) or "").strip()
            if written:
                return written
            # An empty answer is a failure, not a summary. Returning it would
            # leave a hole in the middle of the document with no trace.
            last_error = ValueError("the model returned an empty summary")
        except Exception as error:  # noqa: BLE001 - any model failure is retried
            last_error = error
    raise SummaryUnavailable(str(last_error))

/**
 * Summarise a long document with a self-hosted abstractive model.
 *
 * Rung N2. The first rung that writes. N0 and N1 choose sentences and can only
 * ever return what the author already wrote; this one produces a sentence that
 * was not in the document, which is the only way to state a conclusion drawn
 * from two passages ten pages apart.
 *
 * That gain has a price, and the price is in this file rather than in the
 * model. A sequence-to-sequence summariser reads a fixed-size window. A long
 * document does not fit, so it has to be cut at sentence boundaries,
 * summarised piece by piece, and the pieces summarised again. Every one of
 * those calls can fail or come back empty, and a summary that is silently half
 * a document is worse than no summary at all.
 *
 * What this file cannot do, at any price: check that what the model wrote is
 * what the document said. Nothing in this plumbing can. See the test.
 */

const SENTENCE_END = /(?<=[.!?])\s+/;

export const MODEL_NAME = 'Xenova/distilbart-cnn-12-6';

// What one pass of the model is allowed to read. Beyond its window the model
// truncates without saying so, and a summary of the first half of a chunk is
// indistinguishable from a summary of all of it.
export const CHUNK_CHARACTERS = 3000;

// What the whole function is allowed to read. A self-hosted model costs
// machine time rather than money, but a document nobody meant to send is still
// better refused than churned through in silence.
export const MAX_CHARACTERS = 200_000;

export class SummaryUnavailable extends Error {}

/**
 * The real model, loaded from local weights. Loaded once and kept for the life
 * of the process: it is the loading that is slow, not the summarising.
 */
export class LocalSummariser {
  #pipeline;

  async load(name = MODEL_NAME) {
    // Loads weights, so this is never reached in the tests.
    const { pipeline } = await import('@xenova/transformers');
    this.#pipeline = await pipeline('summarization', name);
    return this;
  }

  async generate(text) {
    const [{ summary_text: written }] = await this.#pipeline(text, { truncation: true });
    return written;
  }
}

/**
 * Cut the document into pieces that fit the model's window, at sentence
 * boundaries.
 *
 * A sentence longer than the window on its own is passed whole and the model
 * will truncate it. Cutting mid-sentence to avoid that would hand the model
 * half a clause, which is a worse thing to summarise.
 */
export function chunk(text, size = CHUNK_CHARACTERS) {
  const pieces = [];
  let current = '';
  for (const raw of text.trim().split(SENTENCE_END)) {
    const sentence = raw.trim();
    if (!sentence) continue;
    if (current && current.length + 1 + sentence.length > size) {
      pieces.push(current);
      current = sentence;
    } else {
      current = `${current} ${sentence}`.trim();
    }
  }
  if (current) pieces.push(current);
  return pieces;
}

/**
 * @param {string} text
 * @param {object} options
 * @param {{generate: Function}} [options.model] injected so this can be tested
 *   without loading weights; defaults to the real model
 * @param {number} [options.attempts]
 */
export async function summarise(text, { model, attempts = 2 } = {}) {
  if (!model) {
    // Loads weights, so this is never reached in the tests.
    model = await new LocalSummariser().load();
  }

  if (text.length > MAX_CHARACTERS) {
    throw new RangeError(`document longer than ${MAX_CHARACTERS} characters`);
  }

  const pieces = chunk(text);
  if (pieces.length === 0) return '';

  const notes = [];
  for (const piece of pieces) notes.push(await generate(model, piece, attempts));
  if (notes.length === 1) return notes[0];

  // Second pass: the model reads back its own notes. Concatenating them
  // instead would give a text as long as the number of chunks, which is not a
  // summary of the document but a summary of each of its parts.
  return generate(model, notes.join(' '), attempts);
}

async function generate(model, text, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const written = ((await model.generate(text)) ?? '').trim();
      if (written) return written;
      // An empty answer is a failure, not a summary. Returning it would leave
      // a hole in the middle of the document with no trace.
      lastError = new Error('the model returned an empty summary');
    } catch (error) {
      lastError = error;
    }
  }
  throw new SummaryUnavailable(String(lastError));
}

"""
Summarise a long document by asking a general-purpose model.

Rung N3. This is the option people reach for first. It is here so you can see
what it costs, not because this entry recommends it.

What it buys over N2 is real: no weights to host, no machine to keep warm, and
an answer that follows an instruction — three sentences, or a list of points,
or both — without anyone fine-tuning anything.

What it costs is in this file. Cap the input, because the provider charges by
the token and a document nobody meant to send is money gone. Retry, because
the call goes over a network. Parse an answer that is only probably the JSON
you asked for. Refuse an answer of the wrong shape rather than passing half of
one to the caller. That plumbing is what your tests can cover.

What no test here can cover: whether the summary is true of the document. The
model will write a fluent, plausible sentence the document never supported, and
nothing below can tell that sentence from a good one. See the test.
"""

from __future__ import annotations

import json

PROMPT = (
    "Summarise the document below in at most {sentences} sentences.\n"
    "Use only what the document says, and add nothing to it.\n"
    "Answer with JSON only: an object with the key `summary`, a string, and\n"
    "the key `key_points`, a list of short strings.\n\n"
    "Document:\n{document}"
)

MAX_CHARACTERS = 40000


class SummaryUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def summarise(document: str, client=None, *, max_sentences: int = 3, attempts: int = 3) -> dict:
    """
    Return `{"summary": str, "key_points": list[str]}`.

    `client` is injected so this function can be tested without a network call.
    In production it defaults to a real provider client.
    """
    if client is None:  # pragma: no cover - needs a key and a network
        from openai import OpenAI

        client = OpenAI()

    # Refusing an oversized document is not an optimisation, it is a cost
    # control: the provider bills the input whether the answer is useful or not.
    if len(document) > MAX_CHARACTERS:
        raise ValueError(f"document longer than {MAX_CHARACTERS} characters")

    # An empty document has no summary, and asking for one costs the same as
    # asking for a real one.
    if not document.strip():
        return {"summary": "", "key_points": []}

    prompt = PROMPT.format(sentences=max_sentences, document=document)
    return _ask(client, prompt, attempts)


def _ask(client, prompt: str, attempts: int) -> dict:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero: two identical documents that summarise
            # differently cannot be reviewed, and cannot be cached either.
            answer = client.complete(prompt=prompt, temperature=0)
            return _decode(json.loads(answer))
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise SummaryUnavailable(str(last_error))


def _decode(parsed) -> dict:
    """
    Accept only the shape that was asked for.

    Returning a half-built answer would hand the caller a summary that is
    silently empty, which reads exactly like a document with nothing in it.
    """
    if not isinstance(parsed, dict):
        raise ValueError("the model answered something that is not an object")
    summary = parsed.get("summary")
    if not isinstance(summary, str) or not summary.strip():
        raise ValueError("the model answered without a summary")
    points = parsed.get("key_points", [])
    if not isinstance(points, list):
        raise ValueError("the model answered with key points that are not a list")
    return {
        "summary": summary.strip(),
        "key_points": [str(point) for point in points],
    }

/**
 * Summarise a long document by asking a general-purpose model.
 *
 * Rung N3. This is the option people reach for first. It is here so you can
 * see what it costs, not because this entry recommends it.
 *
 * What it buys over N2 is real: no weights to host, no machine to keep warm,
 * and an answer that follows an instruction — three sentences, or a list of
 * points, or both — without anyone fine-tuning anything.
 *
 * What it costs is in this file. Cap the input, because the provider charges
 * by the token and a document nobody meant to send is money gone. Retry,
 * because the call goes over a network. Parse an answer that is only probably
 * the JSON you asked for. Refuse an answer of the wrong shape rather than
 * passing half of one to the caller. That plumbing is what your tests can
 * cover.
 *
 * What no test here can cover: whether the summary is true of the document.
 * The model will write a fluent, plausible sentence the document never
 * supported, and nothing below can tell that sentence from a good one. See the
 * test.
 */

const PROMPT = [
  'Summarise the document below in at most {sentences} sentences.',
  'Use only what the document says, and add nothing to it.',
  'Answer with JSON only: an object with the key `summary`, a string, and',
  'the key `key_points`, a list of short strings.',
  '',
  'Document:',
].join('\n');

export const MAX_CHARACTERS = 40000;

export class SummaryUnavailable extends Error {}

/**
 * Return `{ summary, keyPoints }`.
 *
 * @param {string} document
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {number} [options.maxSentences]
 * @param {number} [options.attempts]
 */
export async function summarise(document, { client, maxSentences = 3, attempts = 3 } = {}) {
  if (!client) {
    // Needs a key and a network, so it is never reached in the tests.
    const { OpenAI } = await import('openai');
    client = new OpenAI();
  }

  // Refusing an oversized document is not an optimisation, it is a cost
  // control: the provider bills the input whether the answer is useful or not.
  if (document.length > MAX_CHARACTERS) {
    throw new RangeError(`document longer than ${MAX_CHARACTERS} characters`);
  }

  // An empty document has no summary, and asking for one costs the same as
  // asking for a real one.
  if (!document.trim()) return { summary: '', keyPoints: [] };

  const prompt = `${PROMPT.replace('{sentences}', String(maxSentences))}\n${document}`;
  return ask(client, prompt, attempts);
}

async function ask(client, prompt, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      // Temperature zero: two identical documents that summarise differently
      // cannot be reviewed, and cannot be cached either.
      const answer = await client.complete({ prompt, temperature: 0 });
      return decode(JSON.parse(answer));
    } catch (error) {
      lastError = error;
    }
  }
  throw new SummaryUnavailable(String(lastError));
}

/**
 * Accept only the shape that was asked for.
 *
 * Returning a half-built answer would hand the caller a summary that is
 * silently empty, which reads exactly like a document with nothing in it.
 */
function decode(parsed) {
  if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
    throw new Error('the model answered something that is not an object');
  }
  const { summary, key_points: points = [] } = parsed;
  if (typeof summary !== 'string' || !summary.trim()) {
    throw new Error('the model answered without a summary');
  }
  if (!Array.isArray(points)) {
    throw new Error('the model answered with key points that are not a list');
  }
  return { summary: summary.trim(), keyPoints: points.map(String) };
}