Moderate user comments

"""
Flag a comment against a term list, after normalisation, with context.

Rung N0. Deterministic, standard library only, and auditable: every decision
can be traced back to one word in a list you control.

Two things make it usable rather than merely simple.

First, normalisation. Accents and case are spellings of the same word, so they
are folded before matching. Nothing else is touched: folding further would
start inventing matches.

Second, the context window. A term list cannot decide anything on its own, so
the function returns the words around each hit. A human reads the window and
decides. A moderation tool that returns a bare boolean hides the one piece of
evidence its reviewer needs.
"""

import re
import unicodedata

# Letters and digits, in any script. Punctuation and underscores separate.
TOKEN = re.compile(r"[^\W_]+")


def normalise(text: str) -> str:
    """Fold case and strip accents, so one entry matches its spellings."""
    decomposed = unicodedata.normalize("NFKD", text.casefold())
    return "".join(c for c in decomposed if not unicodedata.combining(c))


def review(text: str, terms, window: int = 3) -> dict:
    """
    Return every listed term found in `text`, with the words around it.

    `terms` is yours: the list is policy, not code, and it belongs outside the
    function that applies it.

    `window` is a number of words on each side. Widen it when your reviewers
    keep asking what the comment was about.
    """
    listed = {normalise(t) for t in terms}
    words = TOKEN.findall(text)
    matches = []
    for position, word in enumerate(words):
        if normalise(word) in listed:
            start = max(0, position - window)
            matches.append({
                "term": normalise(word),
                "position": position,
                "context": " ".join(words[start:position + window + 1]),
            })
    return {"flagged": bool(matches), "matches": matches}

/**
 * Flag a comment against a term list, after normalisation, with context.
 *
 * Rung N0. Deterministic, no dependency, and auditable: every decision can be
 * traced back to one word in a list you control.
 *
 * Two things make it usable rather than merely simple.
 *
 * First, normalisation. Accents and case are spellings of the same word, so
 * they are folded before matching. Nothing else is touched: folding further
 * would start inventing matches.
 *
 * Second, the context window. A term list cannot decide anything on its own,
 * so the function returns the words around each hit. A human reads the window
 * and decides. A moderation tool that returns a bare boolean hides the one
 * piece of evidence its reviewer needs.
 */

// Letters and digits, in any script. Punctuation and underscores separate.
const TOKEN = /[\p{L}\p{N}]+/gu;

/** Fold case and strip accents, so one entry matches its spellings. */
export function normalise(text) {
  return text.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase();
}

/**
 * Return every listed term found in `text`, with the words around it.
 *
 * `terms` is yours: the list is policy, not code, and it belongs outside the
 * function that applies it.
 *
 * `window` is a number of words on each side. Widen it when your reviewers
 * keep asking what the comment was about.
 */
export function review(text, terms, window = 3) {
  const listed = new Set([...terms].map(normalise));
  const words = text.match(TOKEN) ?? [];
  const matches = [];
  for (const [position, word] of words.entries()) {
    if (!listed.has(normalise(word))) continue;
    matches.push({
      term: normalise(word),
      position,
      context: words.slice(Math.max(0, position - window), position + window + 1).join(' '),
    });
  }
  return { flagged: matches.length > 0, matches };
}

"""
Score a comment with a linear classifier trained on a labelled corpus.

Rung N1. The term list of N0 matches spellings. This matches shapes: character
n-grams, so `bl0rptard` and `blorptardd` share most of their features with the
form the model was shown, and a variant nobody added to a list still scores.

The whole model is a vector of weights over character n-grams. It is small
enough to keep beside the code, it trains while you read this docstring, and
every weight can be printed and argued about. That last property is worth more in
moderation than a point of accuracy: someone will ask why a comment was hidden.
"""

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline


def train(comments: list[str], labels: list[int]):
    """
    `labels` is 1 when the comment breaks the policy, 0 when it does not.

    Character n-grams rather than words, because abuse is spelled creatively
    and a word-level model only knows the exact tokens it was shown.

    `class_weight="balanced"` because a real moderation corpus is mostly
    ordinary comments, and an unweighted model learns to say no to everything.
    """
    model = make_pipeline(
        TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), min_df=1),
        LogisticRegression(class_weight="balanced", max_iter=1000),
    )
    model.fit(comments, labels)
    return model


def score(model, comment: str) -> float:
    """How strongly the model reads this comment as breaking the policy."""
    return float(model.predict_proba([comment])[0][1])


def is_abusive(model, comment: str, threshold: float = 0.5) -> bool:
    """
    Return a decision, and keep the threshold in the caller's hands.

    Moderation has no neutral setting. Move it towards 1 and you silence fewer
    innocent people while letting more abuse through; move it towards 0 and you
    do the opposite. Someone has to choose, and it should not be this function.
    """
    return score(model, comment) >= threshold

/**
 * Score a comment with a linear classifier trained on a labelled corpus.
 *
 * Rung N1. The term list of N0 matches spellings. This matches shapes:
 * character n-grams, so `bl0rptard` and `blorptardd` share most of their
 * features with the form the model was shown, and a variant nobody added to a
 * list still scores.
 *
 * Written out rather than pulled from a library, because logistic regression
 * on hashed character n-grams is forty lines. The model is a vector of
 * weights: small enough to keep beside the code, trained while you read this,
 * and every weight can be printed and argued about. In moderation that last
 * property is worth more than a point of accuracy, because sooner or later
 * someone asks why their comment was hidden.
 */

// The hashing trick: no vocabulary to build, ship or keep in sync.
const BUCKETS = 1024;

/** Character n-grams of the lowercased comment, hashed into a fixed vector. */
function features(text) {
  const padded = ` ${text.toLowerCase()} `;
  const vector = new Float64Array(BUCKETS);
  for (let n = 3; n <= 5; n += 1) {
    for (let i = 0; i + n <= padded.length; i += 1) {
      let h = 2166136261;
      for (const c of padded.slice(i, i + n)) h = ((h ^ c.codePointAt(0)) * 16777619) >>> 0;
      vector[h % BUCKETS] += 1;
    }
  }
  const norm = Math.hypot(...vector);
  if (norm) for (let j = 0; j < BUCKETS; j += 1) vector[j] /= norm;
  return vector;
}

/**
 * `labels` is 1 when the comment breaks the policy, 0 when it does not.
 *
 * Each class is weighted by its rarity, because a real moderation corpus is
 * mostly ordinary comments and an unweighted model learns to allow everything.
 */
export function train(comments, labels, { epochs = 300, rate = 1 } = {}) {
  const rows = comments.map(features);
  const counts = [labels.filter((l) => l === 0).length, labels.filter((l) => l === 1).length];
  const weights = new Float64Array(BUCKETS);
  let bias = 0;
  for (let epoch = 0; epoch < epochs; epoch += 1) {
    for (let i = 0; i < rows.length; i += 1) {
      const step = (rate * labels.length) / (2 * counts[labels[i]]);
      const error = predict(rows[i], weights, bias) - labels[i];
      for (let j = 0; j < BUCKETS; j += 1) weights[j] -= step * error * rows[i][j];
      bias -= step * error;
    }
  }
  return { weights, bias };
}

function predict(vector, weights, bias) {
  let z = bias;
  for (let j = 0; j < BUCKETS; j += 1) z += weights[j] * vector[j];
  return 1 / (1 + Math.exp(-z));
}

/** How strongly the model reads this comment as breaking the policy. */
export function score(model, comment) {
  return predict(features(comment), model.weights, model.bias);
}

/**
 * Return a decision, and keep the threshold in the caller's hands.
 *
 * Moderation has no neutral setting. Move it towards 1 and you silence fewer
 * innocent people while letting more abuse through; move it towards 0 and you
 * do the opposite. Someone has to choose, and it should not be this function.
 */
export function isAbusive(model, comment, threshold = 0.5) {
  return score(model, comment) >= threshold;
}

"""
Route comments with a self-hosted toxicity classifier.

Rung N2. A distilled encoder fine-tuned on a moderation corpus, running on
your own machine. It reads context the n-grams of N1 cannot, and it stays
inside your infrastructure, which matters when the text you are sending away
is the abuse one of your users just received.

What you own on this rung is not the model, it is everything around it: the
batching, the thresholds, and the answer to "what does the code do when the
model says nothing usable". The model itself is a black box with a fixed list
of labels, and the last function below is where that becomes your problem.
"""

from __future__ import annotations

MODEL_NAME = "unitary/unbiased-toxic-roberta"

# Two thresholds, not one: between "obviously fine" and "obviously not" there
# is a band that belongs to a human, and pretending otherwise is how automated
# moderation earns its reputation.
DEFAULT_THRESHOLDS = {"block": 0.9, "review": 0.6}


class ToxicityModel:
    """The real model, loaded once and kept in memory for the process."""

    def __init__(self, name: str = MODEL_NAME) -> None:
        from transformers import pipeline  # a large download, done once

        self._pipe = pipeline("text-classification", model=name, top_k=None)

    def predict(self, comments: list[str]) -> list[dict[str, float]]:
        """One label-to-score mapping per comment, in the order given."""
        return [{r["label"]: r["score"] for r in row} for row in self._pipe(comments)]


class ModerationUnavailable(Exception):
    """The model answered something no caller can act on."""


def moderate(comments, classifier=None, thresholds=None) -> list[dict]:
    """
    Decide what to do with each comment: block, send to review, or allow.

    `classifier` is injected so this can be tested without downloading the
    weights. In production it defaults to the real model above.

    The whole batch goes in one call. Feeding comments one by one is the usual
    way this rung is made slow, because a batch of a hundred is one pass
    through the model and a hundred calls are a hundred passes.
    """
    classifier = classifier or ToxicityModel()
    thresholds = thresholds or DEFAULT_THRESHOLDS
    comments = list(comments)
    scored = classifier.predict(comments)
    if len(scored) != len(comments):
        raise ModerationUnavailable("the model returned one row per comment, and did not")
    return [_decide(row, thresholds) for row in scored]


def _decide(scores, thresholds: dict[str, float]) -> dict:
    """
    Keep the strongest label, and fall back to a human when nothing is usable.

    Falling back to "allow" would be the tempting shortcut, and it would mean
    that a model failure silently publishes everything it was asked about.
    """
    usable = {label: v for label, v in (scores or {}).items() if _is_score(v)}
    if not usable:
        return {"action": "review", "label": None, "score": None}
    label, value = max(usable.items(), key=lambda item: item[1])
    if value >= thresholds["block"]:
        action = "block"
    elif value >= thresholds["review"]:
        action = "review"
    else:
        action = "allow"
    return {"action": action, "label": label, "score": value}


def _is_score(value) -> bool:
    """A number the caller can act on, rather than whatever came back."""
    return isinstance(value, (int, float)) and not isinstance(value, bool) and 0.0 <= value <= 1.0

/**
 * Route comments with a self-hosted toxicity classifier.
 *
 * Rung N2. A distilled encoder fine-tuned on a moderation corpus, running on
 * your own machine. It reads context the n-grams of N1 cannot, and it stays
 * inside your infrastructure, which matters when the text you are sending
 * away is the abuse one of your users just received.
 *
 * What you own on this rung is not the model, it is everything around it: the
 * batching, the thresholds, and the answer to "what does the code do when the
 * model says nothing usable". The model itself is a black box with a fixed
 * list of labels, and the last function below is where that becomes your
 * problem.
 */

export const MODEL_NAME = 'Xenova/toxic-bert';

// Two thresholds, not one: between "obviously fine" and "obviously not" there
// is a band that belongs to a human, and pretending otherwise is how automated
// moderation earns its reputation.
export const DEFAULT_THRESHOLDS = { block: 0.9, review: 0.6 };

export class ModerationUnavailable extends Error {}

/** The real model, loaded once and kept in memory for the process. */
export class ToxicityModel {
  static async load(name = MODEL_NAME) {
    const { pipeline } = await import('@huggingface/transformers'); // a large download, done once
    return new ToxicityModel(await pipeline('text-classification', name));
  }

  constructor(pipe) {
    this.pipe = pipe;
  }

  /** One label-to-score mapping per comment, in the order given. */
  async predict(comments) {
    const rows = await this.pipe(comments, { top_k: null });
    return rows.map((row) => Object.fromEntries(row.map((r) => [r.label, r.score])));
  }
}

/**
 * Decide what to do with each comment: block, send to review, or allow.
 *
 * `classifier` is injected so this can be tested without downloading the
 * weights. In production it defaults to the real model above.
 *
 * The whole batch goes in one call. Feeding comments one by one is the usual
 * way this rung is made slow, because a batch of a hundred is one pass through
 * the model and a hundred calls are a hundred passes.
 */
export async function moderate(comments, classifier, thresholds = DEFAULT_THRESHOLDS) {
  const model = classifier ?? (await ToxicityModel.load());
  const batch = [...comments];
  const scored = await model.predict(batch);
  if (scored.length !== batch.length) {
    throw new ModerationUnavailable('the model returned one row per comment, and did not');
  }
  return scored.map((row) => decide(row, thresholds));
}

/**
 * Keep the strongest label, and fall back to a human when nothing is usable.
 *
 * Falling back to "allow" would be the tempting shortcut, and it would mean
 * that a model failure silently publishes everything it was asked about.
 */
function decide(scores, thresholds) {
  const usable = Object.entries(scores ?? {}).filter(
    ([, v]) => typeof v === 'number' && v >= 0 && v <= 1,
  );
  if (usable.length === 0) return { action: 'review', label: null, score: null };
  const [label, score] = usable.reduce((best, row) => (row[1] > best[1] ? row : best));
  let action = 'allow';
  if (score >= thresholds.block) action = 'block';
  else if (score >= thresholds.review) action = 'review';
  return { action, label, score };
}

"""
Score a comment through a provider's moderation endpoint.

Rung N3. The shortest code on the ladder to write, and the one that hands the
most away: the taxonomy, the calibration, the right to appeal, and the text of
your users' comments, which leaves your premises on every call.

The endpoint returns a number per category. Everything else here — capping the
input, retrying, refusing to act on an answer that is not the shape you asked
for — is plumbing you own, and it is where the bugs of this rung live. It is
also all your tests can reach, because the judgement itself is not testable.
"""

from __future__ import annotations

import json

CATEGORIES = ("harassment", "hate", "violence", "self_harm")

PROMPT = (
    "Rate the comment below on each moderation category. Answer with JSON\n"
    "only: an object mapping each category to a score between 0 and 1.\n"
    f"Categories: {', '.join(CATEGORIES)}\n\nComment:\n{{comment}}"
)

MAX_CHARACTERS = 4000
DEFAULT_THRESHOLDS = {"block": 0.9, "review": 0.6}


class ModerationUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def moderate(comment: str, client=None, *, thresholds=None, attempts: int = 3) -> dict:
    """
    Decide what to do with one comment: block, send to review, or allow.

    `client` is injected so this can be tested without a network call. In
    production it defaults to a real provider client.
    """
    if client is None:  # pragma: no cover - needs a key and a network
        from openai import OpenAI

        client = OpenAI()

    # An endpoint charges by the token, and a comment that long is a bug or an
    # attack. Refusing it is a cost control, not an optimisation.
    if len(comment) > MAX_CHARACTERS:
        raise ValueError(f"comment longer than {MAX_CHARACTERS} characters")

    thresholds = thresholds or DEFAULT_THRESHOLDS
    scores = _ask(client, comment, attempts)
    category, score = max(scores.items(), key=lambda item: item[1])
    if score >= thresholds["block"]:
        action = "block"
    elif score >= thresholds["review"]:
        action = "review"
    else:
        action = "allow"
    return {"action": action, "category": category, "score": score, "scores": scores}


def _ask(client, comment: str, attempts: int) -> dict[str, float]:
    """
    Keep the categories that came back as a number in range, and nothing else.

    A category the model invented is dropped, one it omitted is simply absent.
    An answer with none of them left is unusable, and unusable is raised rather
    than quietly turned into "allow".
    """
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            answer = client.complete(prompt=PROMPT.format(comment=comment), temperature=0)
            parsed = json.loads(answer)
            scores = {n: float(parsed[n]) for n in CATEGORIES if _is_score(parsed.get(n))}
            if scores:
                return scores
            last_error = ValueError("no category came back as a score in range")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise ModerationUnavailable(str(last_error))


def _is_score(value) -> bool:
    """A number the caller can act on, rather than whatever came back."""
    return isinstance(value, (int, float)) and not isinstance(value, bool) and 0.0 <= value <= 1.0

/**
 * Score a comment through a provider's moderation endpoint.
 *
 * Rung N3. The shortest code on the ladder to write, and the one that hands
 * the most away: the taxonomy, the calibration, the right to appeal, and the
 * text of your users' comments, which leaves your premises on every call.
 *
 * The endpoint returns a number per category. Everything else here — capping
 * the input, retrying, refusing to act on an answer that is not the shape you
 * asked for — is plumbing you own, and it is where the bugs of this rung live.
 * It is also all your tests can reach, because the judgement itself is not
 * testable.
 */

export const CATEGORIES = ['harassment', 'hate', 'violence', 'self_harm'];

const PROMPT = [
  'Rate the comment below on each moderation category. Answer with JSON',
  'only: an object mapping each category to a score between 0 and 1.',
  `Categories: ${CATEGORIES.join(', ')}`,
  '',
  'Comment:',
].join('\n');

export const MAX_CHARACTERS = 4000;
export const DEFAULT_THRESHOLDS = { block: 0.9, review: 0.6 };

export class ModerationUnavailable extends Error {}

/**
 * Decide what to do with one comment: block, send to review, or allow.
 *
 * @param {string} comment
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be
 *   tested without a network call; defaults to a real provider client
 * @param {{block: number, review: number}} [options.thresholds]
 * @param {number} [options.attempts]
 */
export async function moderate(comment, { client, thresholds = DEFAULT_THRESHOLDS, attempts = 3 } = {}) {
  if (!client) {
    // Needs a key and a network, so it is never reached in the tests.
    const { OpenAI } = await import('openai');
    client = new OpenAI();
  }

  // An endpoint charges by the token, and a comment that long is a bug or an
  // attack. Refusing it is a cost control, not an optimisation.
  if (comment.length > MAX_CHARACTERS) {
    throw new RangeError(`comment longer than ${MAX_CHARACTERS} characters`);
  }

  const scores = await ask(client, comment, attempts);
  const [category, score] = Object.entries(scores).reduce((best, row) => (row[1] > best[1] ? row : best));
  let action = 'allow';
  if (score >= thresholds.block) action = 'block';
  else if (score >= thresholds.review) action = 'review';
  return { action, category, score, scores };
}

/**
 * Keep the categories that came back as a number in range, and nothing else.
 *
 * A category the model invented is dropped, one it omitted is simply absent.
 * An answer with none of them left is unusable, and unusable is thrown rather
 * than quietly turned into "allow".
 */
async function ask(client, comment, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt: `${PROMPT}\n${comment}`,
        // Temperature zero, because a moderation decision that changes between
        // two identical calls cannot be explained to the person it hit.
        temperature: 0,
      });
      const parsed = JSON.parse(answer);
      const scores = Object.fromEntries(CATEGORIES.filter((n) => isScore(parsed[n])).map((n) => [n, parsed[n]]));
      if (Object.keys(scores).length > 0) return scores;
      lastError = new Error('no category came back as a score in range');
    } catch (error) {
      lastError = error;
    }
  }
  throw new ModerationUnavailable(String(lastError));
}

/** A number the caller can act on, rather than whatever came back. */
function isScore(value) {
  return typeof value === 'number' && value >= 0 && value <= 1;
}