Moderate user comments

"""
Flag a comment against a term list, after normalisation, with context.

Rung N0. Deterministic, standard library only, and auditable: every decision
can be traced back to one entry in a list you control.

Two things make it usable rather than merely simple.

First, normalisation. Case, accents and compatibility forms (full-width
letters, ligatures, superscripts) are spellings of the same word, so they are
folded before matching. Look-alikes are not: a 0 stays a 0, which is why
bl0rptard walks past.

Second, the context window. A term list cannot decide anything on its own, so
the function returns the words around each hit. A human reads the window and
decides. A moderation tool that returns a bare boolean hides the one piece of
evidence its reviewer needs.
"""

import unicodedata

# Latin combining accents. Only those are dropped when folding: a Devanagari
# vowel sign is a mark too, and dropping it would turn one word into another.
LATIN_MARKS = ("\u0300", "\u036f")


def normalise(text: str) -> str:
    """Fold compatibility forms, then case, then Latin accents."""
    # Decomposed before folding, and again after: 𝐁𝐋𝐎𝐑𝐏𝐓𝐀𝐑𝐃 has to become
    # BLORPTARD before the case fold can see it, and folding can itself produce
    # a compatibility form.
    decomposed = unicodedata.normalize("NFKD", text)
    folded = unicodedata.normalize("NFKD", decomposed.casefold())
    return "".join(c for c in folded if not LATIN_MARKS[0] <= c <= LATIN_MARKS[1])


def _words(text: str) -> list[str]:
    """
    Letters, digits, and the marks that spell them; anything else separates.

    A regular expression does this in JavaScript, which has a class for marks;
    Python's does not, and its word class leaves marks out, so a Devanagari
    vowel sign would cut a word in two. Composed first, so an accent typed as a
    separate mark stays in its word.
    """
    words, current = [], []
    for char in unicodedata.normalize("NFC", text):
        if char.isalnum() or unicodedata.category(char).startswith("M"):
            current.append(char)
        elif current:
            words.append("".join(current))
            current = []
    if current:
        words.append("".join(current))
    return words


def review(text: str, terms, window: int = 3) -> dict:
    """
    Return every listed term found in `text`, with the words around it.

    A term of several words matches those words in a row, whatever punctuation
    separates them in the comment.

    `terms` is yours: the list is policy, not code, and it belongs outside the
    function that applies it.

    `window` is a number of words on each side. Widen it when your reviewers
    keep asking what the comment was about.
    """
    listed = {tuple(normalise(w) for w in _words(t)) for t in terms}
    longest = max(map(len, listed), default=0)
    words = _words(text)
    folded = [normalise(w) for w in words]
    matches = []
    for position in range(len(words)):
        for size in range(1, min(longest, len(words) - position) + 1):
            if tuple(folded[position:position + size]) in listed:
                matches.append({
                    "term": " ".join(folded[position:position + size]),
                    "position": position,
                    "context": " ".join(words[max(0, position - window):position + size + window]),
                })
    return {"flagged": bool(matches), "matches": matches}

/**
 * Flag a comment against a term list, after normalisation, with context.
 *
 * Rung N0. Deterministic, no dependency, and auditable: every decision can be
 * traced back to one entry in a list you control.
 *
 * Two things make it usable rather than merely simple.
 *
 * First, normalisation. Case, accents and compatibility forms (full-width
 * letters, ligatures, superscripts) are spellings of the same word, so they
 * are folded before matching. Look-alikes are not: a 0 stays a 0, which is
 * why bl0rptard walks past.
 *
 * Second, the context window. A term list cannot decide anything on its own,
 * so the function returns the words around each hit. A human reads the window
 * and decides. A moderation tool that returns a bare boolean hides the one
 * piece of evidence its reviewer needs.
 */

// Letters, digits, and the marks that spell them; anything else separates. A
// Devanagari vowel sign is a mark, and leaving it out would cut a word in two.
const TOKEN = /[\p{L}\p{N}\p{M}]+/gu;

/** Fold compatibility forms, then case, then Latin accents. */
export function normalise(text) {
  // Decomposed before folding, and again after: 𝐁𝐋𝐎𝐑𝐏𝐓𝐀𝐑𝐃 has to become
  // BLORPTARD before the case fold can see it. JavaScript has no casefold, so
  // `toLowerCase` does the work and the two letters a Latin term list meets,
  // ß and the final ς, are done by hand; there are two hundred and fifty-two
  // others, in Cherokee, Greek and Cyrillic, that this line does not cover.
  return text
    .normalize('NFKD')
    .toLowerCase()
    .normalize('NFKD')
    .replaceAll('ß', 'ss')
    .replaceAll('ς', 'σ')
    .replace(/[\u0300-\u036f]/gu, '');
}

// Composed first, so an accent typed as a separate mark stays in its word.
const words = (text) => text.normalize('NFC').match(TOKEN) ?? [];

/**
 * Return every listed term found in `text`, with the words around it.
 *
 * A term of several words matches those words in a row, whatever punctuation
 * separates them in the comment.
 *
 * `terms` is yours: the list is policy, not code, and it belongs outside the
 * function that applies it.
 *
 * `window` is a number of words on each side. Widen it when your reviewers
 * keep asking what the comment was about.
 */
export function review(text, terms, window = 3) {
  const listed = new Set([...terms].map((t) => words(t).map(normalise).join(' ')));
  const longest = [...terms].reduce((most, t) => Math.max(most, words(t).length), 0);
  const found = words(text);
  const folded = found.map(normalise);
  const matches = [];
  for (let position = 0; position < found.length; position += 1) {
    for (let size = 1; size <= Math.min(longest, found.length - position); size += 1) {
      const term = folded.slice(position, position + size).join(' ');
      if (!listed.has(term)) continue;
      matches.push({
        term,
        position,
        context: found.slice(Math.max(0, position - window), position + size + window).join(' '),
      });
    }
  }
  return { flagged: matches.length > 0, matches };
}

"""
Score a comment with a linear classifier trained on a labelled corpus.

Rung N1. The term list of N0 matches spellings. This matches shapes: character
n-grams, so `bl0rptard` and `blorptardd` share most of their features with the
form the model was shown, and a variant nobody added to a list still scores.

The whole model is a vector of weights over character n-grams. It is small
enough to keep beside the code, it trains while you read this docstring, and
every weight can be printed next to the n-gram it belongs to. Someone will ask
why their comment was hidden.
"""

import unicodedata

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline


def fold(comment: str) -> str:
    """NFKC then lowercase, so full-width letters and ligatures read as the plain ones."""
    return unicodedata.normalize("NFKC", comment).lower()


def train(comments: list[str], labels: list[int]):
    """
    `labels` is 1 when the comment breaks the policy, 0 when it does not.

    Character n-grams rather than words, because abuse is spelled creatively
    and a word-level model only knows the exact tokens it was shown.

    `class_weight="balanced"` because a real moderation corpus is mostly
    ordinary comments, and an unweighted model learns to say no to everything.
    """
    model = make_pipeline(
        TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), preprocessor=fold),
        # `C` above one because a couple of dozen short examples under the
        # default regulariser leave every score sitting near a half, which
        # makes the threshold in `is_abusive` meaningless.
        LogisticRegression(class_weight="balanced", C=10.0, max_iter=1000),
    )
    model.fit(comments, labels)
    return model


def score(model, comment: str) -> float:
    """
    How strongly the model reads this comment as breaking the policy. A comment
    with no n-gram to read scores 0: the intercept alone is not evidence.
    """
    if model[0].transform([comment]).nnz == 0:
        return 0.0
    return float(model.predict_proba([comment])[0][1])


def is_abusive(model, comment: str, threshold: float = 0.5) -> bool:
    """
    Return a decision, and keep the threshold in the caller's hands.

    Moderation has no neutral setting. Move it towards 1 and you silence fewer
    innocent people while letting more abuse through; move it towards 0 and you
    do the opposite. Someone has to choose, and it should not be this function.
    """
    return score(model, comment) >= threshold

/**
 * Score a comment with a linear classifier trained on a labelled corpus.
 *
 * Rung N1. The term list of N0 matches spellings. This matches shapes:
 * character n-grams, so `bl0rptard` and `blorptardd` share most of their
 * features with the form the model was shown, and a variant nobody added to a
 * list still scores.
 *
 * Written out rather than pulled from a library: hashing the character
 * n-grams, training a logistic regression and scoring take four short
 * functions. The model is a vector of weights, small enough to keep beside the
 * code and trained while you read this. Every weight can be printed, but with
 * hashing a weight stands for every n-gram that lands in its bucket: to argue
 * about one, list those n-grams first. Someone will ask why their comment was
 * hidden.
 */

// The hashing trick: no vocabulary to build, ship or keep in sync.
const BUCKETS = 1024;

/** Character n-grams of the folded comment, hashed into a fixed vector. */
function features(text) {
  // NFKC first, so full-width letters and ligatures read as the plain ones.
  const padded = ` ${text.normalize('NFKC').toLowerCase()} `;
  const vector = new Float64Array(BUCKETS);
  for (let n = 3; n <= 5; n += 1) {
    for (let i = 0; i + n <= padded.length; i += 1) {
      // An n-gram of nothing but blanks is not evidence about a comment: a
      // form submitted empty would otherwise be scored like any other text.
      // Python never builds one, its analyser padding each word rather than
      // the whole comment.
      if (padded.slice(i, i + n).trim() === '') continue;
      let h = 2166136261;
      // Math.imul keeps the multiplication exact on 32 bits; a plain `*` goes
      // past 2^53, loses the low bits, and piles n-grams into fewer buckets.
      for (const c of padded.slice(i, i + n)) h = Math.imul(h ^ c.codePointAt(0), 16777619) >>> 0;
      vector[h % BUCKETS] += 1;
    }
  }
  const norm = Math.hypot(...vector);
  if (norm) for (let j = 0; j < BUCKETS; j += 1) vector[j] /= norm;
  return vector;
}

/**
 * `labels` is 1 when the comment breaks the policy, 0 when it does not.
 *
 * Each class is weighted by its rarity, because a real moderation corpus is
 * mostly ordinary comments and an unweighted model learns to allow everything.
 */
export function train(comments, labels, { epochs = 300, rate = 1 } = {}) {
  const counts = [labels.filter((l) => l === 0).length, labels.filter((l) => l === 1).length];
  // Both classes, one label per comment: otherwise the weights learn nothing
  // and every comment gets the same verdict.
  if (!counts[0] || !counts[1] || counts[0] + counts[1] !== comments.length) {
    throw new RangeError('train needs labels 0 and 1, one per comment');
  }
  const rows = comments.map(features);
  const weights = new Float64Array(BUCKETS);
  let bias = 0;
  for (let epoch = 0; epoch < epochs; epoch += 1) {
    for (let i = 0; i < rows.length; i += 1) {
      const step = (rate * labels.length) / (2 * counts[labels[i]]);
      const error = predict(rows[i], weights, bias) - labels[i];
      for (let j = 0; j < BUCKETS; j += 1) weights[j] -= step * error * rows[i][j];
      bias -= step * error;
    }
  }
  return { weights, bias };
}

function predict(vector, weights, bias) {
  let z = bias;
  for (let j = 0; j < BUCKETS; j += 1) z += weights[j] * vector[j];
  return 1 / (1 + Math.exp(-z));
}

/**
 * How strongly the model reads this comment as breaking the policy. A comment
 * with no n-gram to read scores 0: the bias alone is not evidence.
 */
export function score(model, comment) {
  const vector = features(comment);
  return vector.some((v) => v !== 0) ? predict(vector, model.weights, model.bias) : 0;
}

/**
 * Return a decision, and keep the threshold in the caller's hands.
 *
 * Moderation has no neutral setting. Move it towards 1 and you silence fewer
 * innocent people while letting more abuse through; move it towards 0 and you
 * do the opposite. Someone has to choose, and it should not be this function.
 */
export function isAbusive(model, comment, threshold = 0.5) {
  return score(model, comment) >= threshold;
}

"""
Score a comment by asking a general-purpose model.

Rung N3. The rung that hands the most away: what each category means, the
calibration, the right to appeal, and the text of your users' comments, which
leaves your premises on every call.

Before writing this, look at whether your provider publishes a moderation
endpoint. The one taken as an example here does, it is not metered, and it
scores thirteen fixed categories. If the harms you care about are on that
list, this file is the expensive way to get them. What it buys, and the only
thing it buys, is a category that is yours: the last one below names giving
out where somebody lives, which no fixed taxonomy on this page carries — not
the provider's thirteen, and not the labels of the model at N2.

You name the categories; the model decides what they mean. Everything else
here — capping the input, retrying, refusing to act on an answer that is not
the shape you asked for — is plumbing you own, and it is where the bugs of
this rung live. It is also all your tests can reach, because the judgement
itself is not testable.
"""

from __future__ import annotations

import json

# The provider named here is an example, not a recommendation: the reasoning
# holds for any general-purpose model API, and the client is swappable. Pass
# any object with a `complete(prompt=..., temperature=...)` method.
MODEL = "gpt-4.1-mini"  # an example id: check the parameters your model accepts

# The first four are on the provider's own moderation endpoint too. The last
# one is not, and it is the reason this file exists.
CATEGORIES = ("harassment", "hate", "violence", "self_harm", "personal_information")

PROMPT = (
    "Rate the comment below on each moderation category. Answer with JSON\n"
    "only: an object mapping each category to a score between 0 and 1.\n"
    f"Categories: {', '.join(CATEGORIES)}\n"
    "personal_information means giving out where somebody lives, works, or how\n"
    "to reach them, without their consent.\n\nComment:\n{comment}"
)

MAX_CHARACTERS = 4000
DEFAULT_THRESHOLDS = {"block": 0.9, "review": 0.6}


class ProviderClient:
    """The one call this snippet makes, on top of the provider's SDK."""

    def __init__(self, sdk=None, model: str = MODEL):
        if sdk is None:  # pragma: no cover - needs a key and a network
            from openai import OpenAI

            sdk = OpenAI()
        self.sdk, self.model = sdk, model

    def complete(self, *, prompt: str, temperature: float) -> str:
        response = self.sdk.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            temperature=temperature,
        )
        return response.choices[0].message.content


class ModerationUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def moderate(comment: str, client=None, *, thresholds=None, attempts: int = 3) -> dict:
    """
    Decide what to do with one comment: block, send to review, or allow.

    `client` is injected so this can be tested without a network call. In
    production it defaults to a real provider client.
    """
    # The provider bills every token of the prompt. The cap counts characters,
    # not tokens, and is checked before any call: the caller decides where a
    # longer comment goes instead. On a forum that is the review queue, not the
    # bin — a comment of four thousand and one characters is not an anomaly.
    if len(comment) > MAX_CHARACTERS:
        raise ValueError(f"comment longer than {MAX_CHARACTERS} characters")

    client = client or ProviderClient()
    thresholds = thresholds or DEFAULT_THRESHOLDS
    scores = _ask(client, comment, attempts)
    category, score = max(scores.items(), key=lambda item: item[1])
    if score >= thresholds["block"]:
        action = "block"
    elif score >= thresholds["review"]:
        action = "review"
    else:
        action = "allow"
    return {"action": action, "category": category, "score": score, "scores": scores}


def _ask(client, comment: str, attempts: int) -> dict[str, float]:
    """
    Keep the categories that came back as a number in range, and nothing else.

    A category the model invented is dropped, one it omitted is simply absent.
    An answer with none of them left is unusable, and unusable is raised rather
    than quietly turned into "allow".
    """
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            # Temperature zero, because a moderation decision that changes between
            # two identical calls cannot be explained to the person it hit.
            answer = client.complete(prompt=PROMPT.format(comment=comment), temperature=0)
            # No content (a refusal) and anything but a JSON object are unusable.
            parsed = json.loads(_unfenced(answer)) if isinstance(answer, str) else None
            if not isinstance(parsed, dict):
                last_error = ValueError("the answer is not a JSON object")
                continue
            scores = {n: float(parsed[n]) for n in CATEGORIES if _is_score(parsed.get(n))}
            if scores:
                return scores
            last_error = ValueError("no category came back as a score in range")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise ModerationUnavailable(str(last_error))


def _unfenced(answer: str) -> str:
    """
    A JSON answer wrapped whole in one code fence is read; nothing else is.

    Prose around it, a second block, or a fence never closed is a failed
    answer, and it is asked for again rather than salvaged.
    """
    text = answer.strip()
    if text.startswith("```") and text.endswith("```") and text.count("```") == 2:
        return text[3:-3].removeprefix("json")
    return text


def _is_score(value) -> bool:
    """A number the caller can act on, rather than whatever came back."""
    return isinstance(value, (int, float)) and not isinstance(value, bool) and 0.0 <= value <= 1.0

/**
 * Score a comment by asking a general-purpose model.
 *
 * Rung N3. The rung that hands the most away: what each category means, the
 * calibration, the right to appeal, and the text of your users' comments,
 * which leaves your premises on every call.
 *
 * Before writing this, look at whether your provider publishes a moderation
 * endpoint. The one taken as an example here does, it is not metered, and it
 * scores thirteen fixed categories. If the harms you care about are on that
 * list, this file is the expensive way to get them. What it buys, and the only
 * thing it buys, is a category that is yours: the last one below names giving
 * out where somebody lives, which no fixed taxonomy on this page carries — not
 * the provider's thirteen, and not the labels of the model at N2.
 *
 * You name the categories; the model decides what they mean. Everything else
 * here — capping the input, retrying, refusing to act on an answer that is not
 * the shape you asked for — is plumbing you own, and it is where the bugs of
 * this rung live. It is also all your tests can reach, because the judgement
 * itself is not testable.
 */

// The provider named here is an example, not a recommendation: the reasoning
// holds for any general-purpose model API, and the client is swappable. Pass
// any object with a `complete({ prompt, temperature })` method.
export const MODEL = 'gpt-4.1-mini'; // an example id: check the parameters your model accepts

export async function providerClient(sdk, model = MODEL) {
  if (!sdk) {
    const { OpenAI } = await import('openai');
    sdk = new OpenAI();
  }
  return {
    async complete({ prompt, temperature }) {
      const response = await sdk.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        temperature,
      });
      return response.choices[0].message.content;
    },
  };
}

// The first four are on the provider's own moderation endpoint too. The last
// one is not, and it is the reason this file exists.
export const CATEGORIES = ['harassment', 'hate', 'violence', 'self_harm', 'personal_information'];

const PROMPT = [
  'Rate the comment below on each moderation category. Answer with JSON',
  'only: an object mapping each category to a score between 0 and 1.',
  `Categories: ${CATEGORIES.join(', ')}`,
  'personal_information means giving out where somebody lives, works, or how',
  'to reach them, without their consent.',
  '',
  'Comment:',
].join('\n');

export const MAX_CHARACTERS = 4000;
export const DEFAULT_THRESHOLDS = { block: 0.9, review: 0.6 };

export class ModerationUnavailable extends Error {}

/**
 * Decide what to do with one comment: block, send to review, or allow.
 *
 * @param {string} comment
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {{block: number, review: number}} [options.thresholds]
 * @param {number} [options.attempts]
 */
export async function moderate(comment, { client, thresholds = DEFAULT_THRESHOLDS, attempts = 3 } = {}) {
  // The provider bills every token of the prompt. The cap counts characters
  // (code points, as Python does), not tokens, and is checked before any call:
  // the caller decides where a longer comment goes instead. On a forum that is
  // the review queue, not the bin — a comment of four thousand and one
  // characters is not an anomaly.
  if ([...comment].length > MAX_CHARACTERS) {
    throw new RangeError(`comment longer than ${MAX_CHARACTERS} characters`);
  }

  client ??= await providerClient();
  const scores = await ask(client, comment, attempts);
  const [category, score] = Object.entries(scores).reduce((best, row) => (row[1] > best[1] ? row : best));
  let action = 'allow';
  if (score >= thresholds.block) action = 'block';
  else if (score >= thresholds.review) action = 'review';
  return { action, category, score, scores };
}

/**
 * Keep the categories that came back as a number in range, and nothing else.
 *
 * A category the model invented is dropped, one it omitted is simply absent.
 * An answer with none of them left is unusable, and unusable is thrown rather
 * than quietly turned into "allow".
 */
async function ask(client, comment, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt: `${PROMPT}\n${comment}`,
        // Temperature zero, because a moderation decision that changes between
        // two identical calls cannot be explained to the person it hit.
        temperature: 0,
      });
      // No content (a refusal) and anything but a JSON object are unusable.
      const parsed = typeof answer === 'string' ? JSON.parse(unfenced(answer)) : null;
      if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
        lastError = new Error('the answer is not a JSON object');
        continue;
      }
      const scores = Object.fromEntries(CATEGORIES.filter((n) => isScore(parsed[n])).map((n) => [n, parsed[n]]));
      if (Object.keys(scores).length > 0) return scores;
      lastError = new Error('no category came back as a score in range');
    } catch (error) {
      lastError = error;
    }
  }
  throw new ModerationUnavailable(String(lastError));
}

/** A number the caller can act on, rather than whatever came back. */
/**
 * A JSON answer wrapped whole in one code fence is read; nothing else is.
 *
 * Prose around it, a second block, or a fence never closed is a failed answer,
 * and it is asked for again rather than salvaged.
 */
function unfenced(answer) {
  const text = answer.trim();
  const fences = text.match(/```/g)?.length ?? 0;
  if (text.startsWith('```') && text.endsWith('```') && fences === 2) {
    const inner = text.slice(3, -3);
    return inner.startsWith('json') ? inner.slice(4) : inner;
  }
  return text;
}

function isScore(value) {
  return typeof value === 'number' && value >= 0 && value <= 1;
}