Découper une adresse en champs

"""
Parse a postal address into fields: regular expressions anchored on the
postcode.

Rung N0. Deterministic, standard library only, and short enough to read in one
sitting.

Two things make this work.

First, the postcode is the anchor. Five digits in a row cut a French address in
two: what comes before is the street line, what comes after is the town. Trying
to recognise the town by its name would mean shipping a list of communes and
keeping it up to date.

Second, the street type is read from a dictionary rather than guessed, so the
abbreviations people actually type — « av. », « bd », « imp. » — come out as one
canonical spelling.
"""

import re
import unicodedata

# The street types of a French address, with the abbreviations people type.
# This is trade knowledge, not example data, so it belongs in the snippet.
STREET_TYPES = {
    "r": "rue", "rue": "rue",
    "av": "avenue", "ave": "avenue", "avenue": "avenue",
    "bd": "boulevard", "bld": "boulevard", "boul": "boulevard", "boulevard": "boulevard",
    "imp": "impasse", "impasse": "impasse",
    "all": "allée", "allee": "allée",
    "ch": "chemin", "chemin": "chemin",
    "pl": "place", "place": "place",
    "rte": "route", "route": "route",
    "quai": "quai", "cours": "cours", "crs": "cours",
    "sq": "square", "square": "square",
    "voie": "voie", "passage": "passage", "sentier": "sentier",
    "chaussee": "chaussée", "fbg": "faubourg", "faubourg": "faubourg",
    "cite": "cité", "villa": "villa", "esplanade": "esplanade",
}

FIELDS = ("number", "street_type", "street", "postcode", "city")

# A house number, and the repetition index that may follow it: 8, 8 bis, 12B.
HOUSE_NUMBER = re.compile(r"^(\d{1,4})\s*(bis|ter|quater|[a-z])?\b", re.IGNORECASE)

# A French postcode: five digits standing alone.
POSTCODE = re.compile(r"\b\d{5}\b")


def normalise(text: str) -> str:
    """Reduce commas, line breaks and exotic spaces to a single plain space."""
    text = unicodedata.normalize("NFKC", text).replace(",", " ")
    return re.sub(r"\s+", " ", text).strip()


def fold(word: str) -> str:
    """Lowercase, drop the accents and the trailing dot, for lookup only."""
    decomposed = unicodedata.normalize("NFD", word.lower())
    return "".join(c for c in decomposed if not unicodedata.combining(c)).strip(".")


def parse(address: str) -> dict:
    """
    Split an address into number, street type, street, postcode and town.

    Every field is a string, empty when the address does not carry it. Returning
    an empty string rather than nothing at all keeps the caller from having to
    test each field before printing it.
    """
    fields = dict.fromkeys(FIELDS, "")
    text = normalise(address)

    # The anchor. Take the last postcode: a street name may carry a year, and a
    # town never comes before its postcode in the French convention.
    postcodes = list(POSTCODE.finditer(text))
    if postcodes:
        found = postcodes[-1]
        fields["postcode"] = found.group()
        fields["city"] = text[found.end():].strip()
        text = text[: found.start()].strip()

    number = HOUSE_NUMBER.match(text)
    if number:
        fields["number"] = " ".join(part for part in number.groups() if part)
        text = text[number.end():].strip()

    words = text.split()
    if words:
        canonical = STREET_TYPES.get(fold(words[0]))
        if canonical:
            # Rewrite the abbreviation, so two spellings of one street compare
            # equal downstream.
            fields["street_type"] = canonical
            words[0] = canonical
        fields["street"] = " ".join(words)
    return fields

/**
 * Parse a postal address into fields: regular expressions anchored on the
 * postcode.
 *
 * Rung N0. Deterministic, no dependency, and short enough to read in one
 * sitting.
 *
 * Two things make this work.
 *
 * First, the postcode is the anchor. Five digits in a row cut a French address
 * in two: what comes before is the street line, what comes after is the town.
 * Recognising the town by its name would mean shipping a list of communes and
 * keeping it up to date.
 *
 * Second, the street type is read from a dictionary rather than guessed, so the
 * abbreviations people actually type — « av. », « bd », « imp. » — come out as
 * one canonical spelling.
 */

// The street types of a French address, with the abbreviations people type.
// This is trade knowledge, not example data, so it belongs in the snippet.
export const STREET_TYPES = {
  r: 'rue', rue: 'rue',
  av: 'avenue', ave: 'avenue', avenue: 'avenue',
  bd: 'boulevard', bld: 'boulevard', boul: 'boulevard', boulevard: 'boulevard',
  imp: 'impasse', impasse: 'impasse',
  all: 'allée', allee: 'allée',
  ch: 'chemin', chemin: 'chemin',
  pl: 'place', place: 'place',
  rte: 'route', route: 'route',
  quai: 'quai', cours: 'cours', crs: 'cours',
  sq: 'square', square: 'square',
  voie: 'voie', passage: 'passage', sentier: 'sentier',
  chaussee: 'chaussée', fbg: 'faubourg', faubourg: 'faubourg',
  cite: 'cité', villa: 'villa', esplanade: 'esplanade',
};

export const FIELDS = ['number', 'street_type', 'street', 'postcode', 'city'];

// A house number, and the repetition index that may follow it: 8, 8 bis, 12B.
const HOUSE_NUMBER = /^(\d{1,4})\s*(bis|ter|quater|[a-z])?\b/i;

// A French postcode: five digits standing alone.
const POSTCODE = /\b\d{5}\b/g;

/** Reduce commas, line breaks and exotic spaces to a single plain space. */
export function normalise(text) {
  return text.normalize('NFKC').replaceAll(',', ' ').replace(/\s+/g, ' ').trim();
}

/** Lowercase, drop the accents and the trailing dot, for lookup only. */
export function fold(word) {
  return word.toLowerCase().normalize('NFD').replace(/\p{M}/gu, '').replace(/\.+$/, '');
}

/**
 * Split an address into number, street type, street, postcode and town.
 *
 * Every field is a string, empty when the address does not carry it. Returning
 * an empty string rather than nothing at all keeps the caller from having to
 * test each field before printing it.
 */
export function parse(address) {
  const fields = Object.fromEntries(FIELDS.map((name) => [name, '']));
  let text = normalise(address);

  // The anchor. Take the last postcode: a street name may carry a year, and a
  // town never comes before its postcode in the French convention.
  const postcodes = [...text.matchAll(POSTCODE)];
  if (postcodes.length > 0) {
    const found = postcodes.at(-1);
    fields.postcode = found[0];
    fields.city = text.slice(found.index + found[0].length).trim();
    text = text.slice(0, found.index).trim();
  }

  const number = HOUSE_NUMBER.exec(text);
  if (number) {
    fields.number = number.slice(1).filter(Boolean).join(' ');
    text = text.slice(number[0].length).trim();
  }

  const words = text.split(' ').filter(Boolean);
  if (words.length > 0) {
    const canonical = STREET_TYPES[fold(words[0])];
    if (canonical) {
      // Rewrite the abbreviation, so two spellings of one street compare equal
      // downstream.
      fields.street_type = canonical;
      words[0] = canonical;
    }
    fields.street = words.join(' ');
  }
  return fields;
}

"""
Label every token of an address with a logistic regression on context traits.

Rung N1. N0 reads the address it expects; this one reads the address it is
given. Each word is labelled on its own — number, street type, street name,
complement, postcode, town — from what it looks like and from what sits on
either side of it. Fields are then rebuilt from the labels.

That is the whole gain: a complement in the middle of the line no longer
swallows the street, because « Bâtiment » is a word the model has seen in that
position, not an unexpected token in a fixed pattern.

The price is a labelled training set. A few dozen addresses tagged by hand are
enough to start, and every convention absent from that set is a convention the
model does not know.
"""

import re
import unicodedata

from sklearn.feature_extraction import DictVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline

# The labels are the fields, so the mapping back is a grouping and nothing more.
LABELS = ("number", "street_type", "street", "complement", "postcode", "city")

TOKEN = re.compile(r"[^\W_]+")


def tokenise(address: str) -> list[str]:
    """Words and numbers, punctuation dropped."""
    return TOKEN.findall(unicodedata.normalize("NFKC", address))


def fold(word: str) -> str:
    """Lowercase and accent-free, so « Allée » and « allee » share a trait."""
    decomposed = unicodedata.normalize("NFD", word.lower())
    return "".join(c for c in decomposed if not unicodedata.combining(c))


def features(tokens: list[str], i: int) -> dict:
    """
    What the token looks like, and what surrounds it.

    The neighbours carry most of the signal: five digits followed by one
    capitalised word is a postcode and a town, wherever it sits in the line.
    """
    token = tokens[i]
    return {
        f"token={fold(token)}": 1,
        f"previous={fold(tokens[i - 1]) if i else '<start>'}": 1,
        f"next={fold(tokens[i + 1]) if i + 1 < len(tokens) else '<end>'}": 1,
        "digits": float(token.isdigit()),
        "five_digits": float(token.isdigit() and len(token) == 5),
        "short_number": float(token.isdigit() and len(token) <= 3),
        "capitalised": float(token[:1].isupper()),
        "position": i / len(tokens),
        "last": float(i == len(tokens) - 1),
    }


def train(examples):
    """
    `examples` pairs an address with one label per token, in reading order.

    Tagging is the work of this rung, and mistagging it is the usual way the
    rung is made to fail, so a misaligned example is rejected rather than
    quietly learnt.
    """
    rows, targets = [], []
    for address, labels in examples:
        tokens = tokenise(address)
        if len(tokens) != len(labels):
            raise ValueError(f"{len(tokens)} tokens for {len(labels)} labels: {address!r}")
        rows += [features(tokens, i) for i in range(len(tokens))]
        targets += list(labels)
    model = make_pipeline(
        DictVectorizer(),
        LogisticRegression(max_iter=1000, class_weight="balanced"),
    )
    model.fit(rows, targets)
    return model


def parse(model, address: str) -> dict:
    """Group the labelled tokens back into fields, in reading order."""
    fields = dict.fromkeys(LABELS, "")
    tokens = tokenise(address)
    if not tokens:
        return fields
    predicted = model.predict([features(tokens, i) for i in range(len(tokens))])
    for token, label in zip(tokens, predicted):
        fields[label] = f"{fields[label]} {token}".strip()
    # The street type stays available on its own, and also opens the street, so
    # the output can be compared field by field with the rung below.
    fields["street"] = f"{fields['street_type']} {fields['street']}".strip()
    return fields

/**
 * Label every token of an address with a logistic regression on context traits.
 *
 * Rung N1. N0 reads the address it expects; this one reads the address it is
 * given. Each word is labelled on its own — number, street type, street name,
 * complement, postcode, town — from what it looks like and from what sits on
 * either side of it. Fields are then rebuilt from the labels.
 *
 * That is the whole gain: a complement in the middle of the line no longer
 * swallows the street, because « Bâtiment » is a word the model has seen in
 * that position, not an unexpected token in a fixed pattern.
 *
 * One binary regression per label, trained by plain gradient descent, and the
 * strongest one wins. Written out rather than pulled from a library, because
 * that is forty lines and it is the argument of this rung.
 */

// The labels are the fields, so the mapping back is a grouping and nothing more.
export const LABELS = ['number', 'street_type', 'street', 'complement', 'postcode', 'city'];

const TOKEN = /[\p{L}\p{N}]+/gu;

/** Words and numbers, punctuation dropped. */
export function tokenise(address) {
  return address.normalize('NFKC').match(TOKEN) ?? [];
}

/** Lowercase and accent-free, so « Allée » and « allee » share a trait. */
export function fold(word) {
  return word.toLowerCase().normalize('NFD').replace(/\p{M}/gu, '');
}

const isDigits = (token) => /^\d+$/.test(token);

/**
 * What the token looks like, and what surrounds it.
 *
 * The neighbours carry most of the signal: five digits followed by one
 * capitalised word is a postcode and a town, wherever it sits in the line.
 */
export function features(tokens, i) {
  const token = tokens[i];
  return {
    [`token=${fold(token)}`]: 1,
    [`previous=${i > 0 ? fold(tokens[i - 1]) : '<start>'}`]: 1,
    [`next=${i + 1 < tokens.length ? fold(tokens[i + 1]) : '<end>'}`]: 1,
    digits: Number(isDigits(token)),
    five_digits: Number(isDigits(token) && token.length === 5),
    short_number: Number(isDigits(token) && token.length <= 3),
    capitalised: Number(token[0] !== token[0].toLowerCase()),
    position: i / tokens.length,
    last: Number(i === tokens.length - 1),
  };
}

/**
 * `examples` pairs an address with one label per token, in reading order.
 *
 * Tagging is the work of this rung, and mistagging it is the usual way the rung
 * is made to fail, so a misaligned example is rejected rather than quietly
 * learnt.
 */
export function train(examples, { epochs = 300, rate = 0.3 } = {}) {
  const rows = [];
  const targets = [];
  for (const [address, labels] of examples) {
    const tokens = tokenise(address);
    if (tokens.length !== labels.length) {
      throw new Error(`${tokens.length} tokens for ${labels.length} labels: ${address}`);
    }
    tokens.forEach((_, i) => {
      rows.push(features(tokens, i));
      targets.push(labels[i]);
    });
  }

  // One column per trait seen in training. A trait absent from this map is a
  // word the model never met, and it simply contributes nothing at prediction.
  const columns = new Map();
  const vectors = rows.map((row) =>
    Object.entries(row).map(([name, value]) => {
      if (!columns.has(name)) columns.set(name, columns.size);
      return [columns.get(name), value];
    }));

  const weights = {};
  for (const label of new Set(targets)) {
    const w = new Float64Array(columns.size + 1); // the last cell is the bias
    for (let epoch = 0; epoch < epochs; epoch += 1) {
      for (let i = 0; i < vectors.length; i += 1) {
        const target = targets[i] === label ? 1 : 0;
        const error = 1 / (1 + Math.exp(-score(w, vectors[i]))) - target;
        for (const [column, value] of vectors[i]) w[column] -= rate * error * value;
        w[columns.size] -= rate * error;
      }
    }
    weights[label] = w;
  }
  return { columns, weights };
}

/** Group the labelled tokens back into fields, in reading order. */
export function parse(model, address) {
  const fields = Object.fromEntries(LABELS.map((name) => [name, '']));
  const tokens = tokenise(address);
  tokens.forEach((token, i) => {
    const vector = vectorise(features(tokens, i), model.columns);
    const label = Object.keys(model.weights).reduce((best, candidate) =>
      score(model.weights[candidate], vector) > score(model.weights[best], vector) ? candidate : best);
    fields[label] = `${fields[label]} ${token}`.trim();
  });
  // The street type stays available on its own, and also opens the street, so
  // the output can be compared field by field with the rung below.
  fields.street = `${fields.street_type} ${fields.street}`.trim();
  return fields;
}

function vectorise(row, columns) {
  return Object.entries(row)
    .filter(([name]) => columns.has(name))
    .map(([name, value]) => [columns.get(name), value]);
}

function score(weights, vector) {
  let total = weights[weights.length - 1];
  for (const [column, value] of vector) total += weights[column] * value;
  return total;
}

"""
Parse an address with a self-hosted statistical parser.

Rung N2. libpostal is a parser trained on tens of millions of addresses from
open worldwide data. It labels a token from its context, like N1, but its
training set is the planet: the German number that comes after the street, the
British postcode that is not five digits, the Japanese order that starts with
the prefecture. It runs on your machine, so no address ever leaves it.

What you own on this rung is not the model. It is the batch, the size cap, the
mapping from its label set to yours, and the answer to « what does the code do
when the parser returns something we cannot use ». The model is a black box
with a fixed list of labels and no confidence score, and the last function
below is where that becomes your problem.
"""

from __future__ import annotations

# A postal address is a short line. Anything longer is a paste, and feeding it
# to the parser only produces confident nonsense more slowly.
MAX_CHARACTERS = 300

FIELDS = ("number", "street", "complement", "postcode", "city")

# libpostal's label set is its own, and wider than ours. Two of its labels can
# land in one of our fields, and everything unmapped is dropped on purpose: an
# unmapped label that silently became a field would be a surprise in a letter.
COMPONENTS = {
    "house_number": "number",
    "road": "street",
    "unit": "complement",
    "level": "complement",
    "staircase": "complement",
    "entrance": "complement",
    "postcode": "postcode",
    "city": "city",
}


class ParsingUnavailable(Exception):
    """The parser failed, or answered something no caller can act on."""


class LibpostalParser:
    """The real parser: a C library and its data files, loaded once."""

    def __init__(self) -> None:
        from postal.parser import parse_address  # a large local install

        self._parse_address = parse_address

    def predict(self, addresses: list[str]) -> list[dict]:
        """One label-to-value mapping per address, in the order given."""
        return [self._merge(self._parse_address(a)) for a in addresses]

    @staticmethod
    def _merge(components) -> dict:
        """libpostal yields (value, label) pairs, and repeats a label freely."""
        merged: dict[str, str] = {}
        for value, label in components:
            merged[label] = f"{merged.get(label, '')} {value}".strip()
        return merged


def parse_addresses(addresses, parser=None, *, attempts: int = 2) -> list[dict]:
    """
    Parse a batch of addresses into fields.

    `parser` is injected so this can be tested without installing the model. In
    production it defaults to the real one above.

    The whole batch goes in one call. Parsing addresses one by one is the usual
    way this rung is made slow, because the model is loaded once and a batch of
    a hundred is one pass through it.
    """
    parser = parser or LibpostalParser()
    batch = list(addresses)
    for address in batch:
        if len(address) > MAX_CHARACTERS:
            raise ValueError(f"address longer than {MAX_CHARACTERS} characters")
    if not batch:
        return []

    rows = _predict(parser, batch, attempts)
    if len(rows) != len(batch):
        raise ParsingUnavailable("the parser returned one row per address, and did not")
    return [_to_fields(row) for row in rows]


def _predict(parser, batch: list[str], attempts: int) -> list[dict]:
    """Retry once: loading the data files is the call that fails, and once."""
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            return parser.predict(batch)
        except Exception as error:  # noqa: BLE001 - any model failure is retried
            last_error = error
    raise ParsingUnavailable(str(last_error))


def _to_fields(row) -> dict:
    """Keep the labels we mapped, join those that share a field, drop the rest."""
    fields = dict.fromkeys(FIELDS, "")
    for label, value in (row or {}).items():
        field = COMPONENTS.get(label)
        if field and isinstance(value, str) and value.strip():
            fields[field] = f"{fields[field]} {value.strip()}".strip()
    return fields

/**
 * Parse an address with a self-hosted statistical parser.
 *
 * Rung N2. libpostal is a parser trained on tens of millions of addresses from
 * open worldwide data. It labels a token from its context, like N1, but its
 * training set is the planet: the German number that comes after the street,
 * the British postcode that is not five digits, the Japanese order that starts
 * with the prefecture. It runs on your machine, so no address ever leaves it.
 *
 * What you own on this rung is not the model. It is the batch, the size cap,
 * the mapping from its label set to yours, and the answer to « what does the
 * code do when the parser returns something we cannot use ». The model is a
 * black box with a fixed list of labels and no confidence score, and the last
 * function below is where that becomes your problem.
 */

// A postal address is a short line. Anything longer is a paste, and feeding it
// to the parser only produces confident nonsense more slowly.
export const MAX_CHARACTERS = 300;

export const FIELDS = ['number', 'street', 'complement', 'postcode', 'city'];

// libpostal's label set is its own, and wider than ours. Two of its labels can
// land in one of our fields, and everything unmapped is dropped on purpose: an
// unmapped label that silently became a field would be a surprise in a letter.
const COMPONENTS = {
  house_number: 'number',
  road: 'street',
  unit: 'complement',
  level: 'complement',
  staircase: 'complement',
  entrance: 'complement',
  postcode: 'postcode',
  city: 'city',
};

export class ParsingUnavailable extends Error {}

/** The real parser: a native binding and its data files, loaded once. */
export class LibpostalParser {
  static async load() {
    const postal = await import('node-postal'); // a large local install
    return new LibpostalParser((address) => postal.parser.parse_address(address));
  }

  constructor(parseAddress) {
    this.parseAddress = parseAddress;
  }

  /** One label-to-value mapping per address, in the order given. */
  async predict(addresses) {
    return addresses.map((address) => merge(this.parseAddress(address)));
  }
}

/** libpostal yields {component, value} pairs, and repeats a component freely. */
function merge(components) {
  const merged = {};
  for (const { component, value } of components) {
    merged[component] = `${merged[component] ?? ''} ${value}`.trim();
  }
  return merged;
}

/**
 * Parse a batch of addresses into fields.
 *
 * `parser` is injected so this can be tested without installing the model. In
 * production it defaults to the real one above.
 *
 * The whole batch goes in one call. Parsing addresses one by one is the usual
 * way this rung is made slow, because the model is loaded once and a batch of a
 * hundred is one pass through it.
 */
export async function parseAddresses(addresses, parser, { attempts = 2 } = {}) {
  const model = parser ?? (await LibpostalParser.load());
  const batch = [...addresses];
  for (const address of batch) {
    if (address.length > MAX_CHARACTERS) {
      throw new RangeError(`address longer than ${MAX_CHARACTERS} characters`);
    }
  }
  if (batch.length === 0) return [];

  const rows = await predict(model, batch, attempts);
  if (rows.length !== batch.length) {
    throw new ParsingUnavailable('the parser returned one row per address, and did not');
  }
  return rows.map(toFields);
}

/** Retry once: loading the data files is the call that fails, and once. */
async function predict(parser, batch, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      return await parser.predict(batch);
    } catch (error) {
      lastError = error;
    }
  }
  throw new ParsingUnavailable(String(lastError));
}

/** Keep the labels we mapped, join those that share a field, drop the rest. */
function toFields(row) {
  const fields = Object.fromEntries(FIELDS.map((name) => [name, '']));
  for (const [label, value] of Object.entries(row ?? {})) {
    const field = COMPONENTS[label];
    if (field && typeof value === 'string' && value.trim()) {
      fields[field] = `${fields[field]} ${value.trim()}`.trim();
    }
  }
  return fields;
}

"""
Split an address by asking a general-purpose model.

Rung N3. This is the option people reach for first. It is here so you can see
what it costs, not because this entry recommends it.

Note what the code has to do that N0 did not: cap the input size, retry on
failure, parse an answer that is only probably valid JSON, and check that the
fields it hands back were actually in the address. That last point is specific
to extraction: a model asked for a postcode and given none will happily supply
a plausible one, and a plausible postcode is worse than an empty field because
nothing downstream will ever question it.

That plumbing is the real cost of this rung, and it is the part your tests have
to cover, because the model itself is not testable.
"""

from __future__ import annotations

import json
import re
import unicodedata

FIELDS = ("number", "street", "complement", "postcode", "city")

PROMPT = (
    "Split the postal address below into fields.\n"
    "Answer with JSON only: an object with the keys `number`, `street`,\n"
    "`complement`, `postcode` and `city`. Copy the text exactly as it is\n"
    "written, and leave a key empty when the address does not carry it.\n\n"
    "Address:\n{address}"
)

# An address is a short line. A cap is not an optimisation here, it is a cost
# control: a model charges by the token, on the way in as well as out.
MAX_CHARACTERS = 300


class ParsingUnavailable(Exception):
    """The provider could not be reached, or answered something unusable."""


def parse(address: str, client=None, *, attempts: int = 3) -> dict:
    """
    Split an address into number, street, complement, postcode and town.

    `client` is injected so this function can be tested without a network call.
    In production it defaults to a real provider client.
    """
    if client is None:  # pragma: no cover - needs a key and a network
        from openai import OpenAI

        client = OpenAI()

    if len(address) > MAX_CHARACTERS:
        raise ValueError(f"address longer than {MAX_CHARACTERS} characters")

    answer = _ask(client, address, attempts)
    source = _fold(address)
    fields = dict.fromkeys(FIELDS, "")
    for key in FIELDS:
        value = answer.get(key)
        if isinstance(value, str) and value.strip() and _fold(value) in source:
            # Kept only if the model copied it from the address. What it made
            # up is dropped, and an empty field is a question a human can see.
            fields[key] = value.strip()
    return fields


def _ask(client, address: str, attempts: int) -> dict:
    last_error: Exception | None = None
    for _ in range(attempts):
        try:
            answer = client.complete(prompt=PROMPT.format(address=address), temperature=0)
            parsed = json.loads(answer)
            if isinstance(parsed, dict):
                return parsed
            last_error = ValueError("the model answered something that is not an object")
        except Exception as error:  # noqa: BLE001 - any provider failure is retried
            last_error = error
    raise ParsingUnavailable(str(last_error))


def _fold(text: str) -> str:
    """Case and spacing are the model's to change; the words are not."""
    return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", text).lower()).strip()

/**
 * Split an address by asking a general-purpose model.
 *
 * Rung N3. This is the option people reach for first. It is here so you can see
 * what it costs, not because this entry recommends it.
 *
 * Note what the code has to do that N0 did not: cap the input size, retry on
 * failure, parse an answer that is only probably valid JSON, and check that the
 * fields it hands back were actually in the address. That last point is
 * specific to extraction: a model asked for a postcode and given none will
 * happily supply a plausible one, and a plausible postcode is worse than an
 * empty field because nothing downstream will ever question it.
 *
 * That plumbing is the real cost of this rung, and it is the part your tests
 * have to cover, because the model itself is not testable.
 */

export const FIELDS = ['number', 'street', 'complement', 'postcode', 'city'];

const PROMPT = [
  'Split the postal address below into fields.',
  'Answer with JSON only: an object with the keys `number`, `street`,',
  '`complement`, `postcode` and `city`. Copy the text exactly as it is',
  'written, and leave a key empty when the address does not carry it.',
  '',
  'Address:',
].join('\n');

// An address is a short line. A cap is not an optimisation here, it is a cost
// control: a model charges by the token, on the way in as well as out.
export const MAX_CHARACTERS = 300;

export class ParsingUnavailable extends Error {}

/**
 * Split an address into number, street, complement, postcode and town.
 *
 * @param {string} address
 * @param {object} options
 * @param {{complete: Function}} [options.client] injected so this can be tested
 *   without a network call; defaults to a real provider client
 * @param {number} [options.attempts]
 */
export async function parse(address, { client, attempts = 3 } = {}) {
  if (!client) {
    // Needs a key and a network, so it is never reached in the tests.
    const { OpenAI } = await import('openai');
    client = new OpenAI();
  }

  if (address.length > MAX_CHARACTERS) {
    throw new RangeError(`address longer than ${MAX_CHARACTERS} characters`);
  }

  const answer = await ask(client, address, attempts);
  const source = fold(address);
  const fields = Object.fromEntries(FIELDS.map((name) => [name, '']));
  for (const key of FIELDS) {
    const value = answer[key];
    if (typeof value === 'string' && value.trim() && source.includes(fold(value))) {
      // Kept only if the model copied it from the address. What it made up is
      // dropped, and an empty field is a question a human can see.
      fields[key] = value.trim();
    }
  }
  return fields;
}

async function ask(client, address, attempts) {
  let lastError;
  for (let i = 0; i < attempts; i += 1) {
    try {
      const answer = await client.complete({
        prompt: `${PROMPT}\n${address}`,
        // Temperature zero, because an address that splits differently between
        // two identical calls cannot be reconciled with anything.
        temperature: 0,
      });
      const parsed = JSON.parse(answer);
      if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed;
      lastError = new Error('the model answered something that is not an object');
    } catch (error) {
      lastError = error;
    }
  }
  throw new ParsingUnavailable(String(lastError));
}

/** Case and spacing are the model's to change; the words are not. */
function fold(text) {
  return text.normalize('NFKC').toLowerCase().replace(/\s+/g, ' ').trim();
}