Découper des lignes de journal en champs

"""
Découper une ligne de journal en champs, avec un motif nommé.

Niveau N0. Une ligne de journal n'est pas du texte libre : elle a été écrite
par un programme, dans un format fixé par ce programme, et souvent par une
norme. Il n'y a rien à deviner — la forme est connue avant la première ligne,
ce qui est exactement la situation pour laquelle une expression régulière a
été inventée.

Ce qui fait la différence au volume n'est pas l'expression mais le
vocabulaire autour d'elle. Grok, la notation que Logstash a rendue courante,
nomme les morceaux une fois — `%{IP:client}`, `%{INT:status}` — et permet de
décrire une ligne par ses champs plutôt que par un mur de crochets. La
douzaine de lignes ci-dessous, c'est cette idée, avec les morceaux dont cette
fiche a besoin ; les vraies bibliothèques en livrent quelques centaines.

Deux règles pour lesquelles le reste de ce fichier existe. Une ligne qui ne
correspond pas n'est jamais jetée : elle revient dans `rejected`, avec son
numéro, parce qu'un analyseur de journal qui perd en silence une ligne sur
vingt est pire qu'un analyseur qui n'en lit aucune. Et l'horodatage est rendu
tel qu'il est écrit, pas comme une date : le format syslog de la RFC 3164 ne
porte ni année ni fuseau, et faire de « Oct 10 13:55:36 » un instant, c'est
inventer les deux.
"""

from __future__ import annotations

import re

# The pieces a pattern is built from. Deliberately few and deliberately loose:
# a log parser that refuses a line because an address was unusual has lost the
# line, and the line is the point.
PIECES = {
    "IP": r"[0-9A-Fa-f.:]+",
    "WORD": r"\S+",
    "INT": r"-?\d+",
    "DATA": r".*?",
    # A quoted field, escapes included: `mod_log_config` writes a `\"` inside
    # the request and the agent, and scanners produce them every day. Reading
    # up to the first quote would send those lines to the reject list.
    "QUOTED": r"(?:[^\"\\]|\\.)*",
    "BRACKETED": r"[^\]]*",
    "NOTCOLON": r"[^:]+",
    "SYSLOGDATE": r"\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}",
    "GREEDY": r".*",
}

# A loose piece is one that can match anything, so it cuts wherever the rest of
# the pattern lets it. Placed before the end, it moves the boundary of every
# field that follows without rejecting a single line — which is this entry's
# breaking point. `loose_pieces` in the report names them.
LOOSE = ("DATA", "GREEDY")

# Three formats that cover most of what lands in a log directory. A pattern is
# written once, in this notation, and compiled for the language that reads it —
# Python spells a named group `(?P<name>…)` and JavaScript `(?<name>…)`, which
# is reason enough not to write either by hand in a file read by both.
#
# The syslog priority between angle brackets exists on the wire — the UDP 514
# packet — and the collector strips it before writing. A line of
# `/var/log/syslog`, of `/var/log/auth.log` or of `journalctl` has none, so the
# piece is optional: a pattern that required it rejected every line of the log
# directory of every machine.
PATTERNS = {
    "apache-combined": (
        r"%{IP:client} %{WORD:ident} %{WORD:user} \[%{BRACKETED:timestamp}\]"
        r' "%{QUOTED:request}" %{INT:status} %{WORD:size}'
        r' "%{QUOTED:referrer}" "%{QUOTED:agent}"'
    ),
    "syslog-3164": (
        r"(?:<%{INT:priority}>)?%{SYSLOGDATE:timestamp} %{WORD:host}"
        r" %{NOTCOLON:tag}: %{GREEDY:message}"
    ),
    "nginx-error": (
        r"%{BRACKETED:timestamp} \[%{BRACKETED:level}\] %{WORD:pid}: %{GREEDY:message}"
    ),
}

NAMED = re.compile(r"%\{(\w+):(\w+)\}")


def compile_pattern(pattern: str):
    """Turn the named notation into a regular expression of this language."""
    def piece(match):
        kind, name = match.group(1), match.group(2)
        if kind not in PIECES:
            raise KeyError(f"unknown piece {kind}")
        return f"(?P<{name}>{PIECES[kind]})"

    return re.compile(f"^{NAMED.sub(piece, pattern)}$")


def loose_pieces(pattern: str) -> list:
    """
    The loose pieces of a pattern that are not its last one.

    A loose piece in last position stops where the line does, which is what it
    is for. Anywhere else it decides the boundary of the fields that follow,
    and it decides it silently.
    """
    found = NAMED.findall(pattern)
    return [name for kind, name in found[:-1] if kind in LOOSE]


def parse_lines(lines, pattern: str) -> dict:
    """
    Every line cut into its fields, and every line that could not be.

    `pattern` is a name from `PATTERNS` or a pattern written in the same
    notation. `rejected` carries the line number and the line itself: that list
    is what tells you the format changed under you.

    `loose_pieces` names the pieces that can cut anywhere. Nothing is rejected
    because of them — that is exactly the problem — so they are named instead.
    """
    written = PATTERNS.get(pattern, pattern)
    try:
        expression = compile_pattern(written)
    except (KeyError, re.error):
        return {"parsed": [], "rejected": [], "loose_pieces": [],
                "reason": "this pattern is not usable"}

    parsed, rejected = [], []
    for number, line in enumerate(lines, start=1):
        if not isinstance(line, str):
            rejected.append({"line": number, "text": None})
            continue
        found = expression.match(line.rstrip("\r\n"))
        if found:
            parsed.append({"line": number, **found.groupdict()})
        else:
            rejected.append({"line": number, "text": line.rstrip("\r\n")})
    return {"parsed": parsed, "rejected": rejected,
            "loose_pieces": loose_pieces(written), "reason": None}

/**
 * Découper une ligne de journal en champs, avec un motif nommé.
 *
 * Niveau N0. Une ligne de journal n'est pas du texte libre : elle a été écrite
 * par un programme, dans un format fixé par ce programme, et souvent par une
 * norme. Il n'y a rien à deviner — la forme est connue avant la première ligne,
 * ce qui est exactement la situation pour laquelle une expression régulière a
 * été inventée.
 *
 * Ce qui fait la différence au volume n'est pas l'expression mais le
 * vocabulaire autour d'elle. Grok, la notation que Logstash a rendue courante,
 * nomme les morceaux une fois — `%{IP:client}`, `%{INT:status}` — et permet de
 * décrire une ligne par ses champs plutôt que par un mur de crochets. La
 * douzaine de lignes ci-dessous, c'est cette idée, avec les morceaux dont cette
 * fiche a besoin ; les vraies bibliothèques en livrent quelques centaines.
 *
 * Deux règles pour lesquelles le reste de ce fichier existe. Une ligne qui ne
 * correspond pas n'est jamais jetée : elle revient dans `rejected`, avec son
 * numéro, parce qu'un analyseur de journal qui perd en silence une ligne sur
 * vingt est pire qu'un analyseur qui n'en lit aucune. Et l'horodatage est rendu
 * tel qu'il est écrit, pas comme une date : le format syslog de la RFC 3164 ne
 * porte ni année ni fuseau, et faire de « Oct 10 13:55:36 » un instant, c'est
 * inventer les deux.
 */

// The pieces a pattern is built from. Deliberately few and deliberately loose:
// a log parser that refuses a line because an address was unusual has lost the
// line, and the line is the point.
export const PIECES = {
  IP: '[0-9A-Fa-f.:]+',
  WORD: '\\S+',
  INT: '-?\\d+',
  DATA: '.*?',
  // A quoted field, escapes included: `mod_log_config` writes a `\"` inside
  // the request and the agent, and scanners produce them every day. Reading up
  // to the first quote would send those lines to the reject list.
  QUOTED: '(?:[^"\\\\]|\\\\.)*',
  BRACKETED: '[^\\]]*',
  NOTCOLON: '[^:]+',
  SYSLOGDATE: '\\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}',
  GREEDY: '.*',
};

// A loose piece is one that can match anything, so it cuts wherever the rest
// of the pattern lets it. Placed before the end, it moves the boundary of
// every field that follows without rejecting a single line — which is this
// entry's breaking point. `loose_pieces` in the report names them.
export const LOOSE = ['DATA', 'GREEDY'];

// Three formats that cover most of what lands in a log directory. A pattern is
// written once, in this notation, and compiled for the language that reads it —
// Python spells a named group `(?P<name>…)` and JavaScript `(?<name>…)`, which
// is reason enough not to write either by hand in a file read by both.
//
// The syslog priority between angle brackets exists on the wire — the UDP 514
// packet — and the collector strips it before writing. A line of
// `/var/log/syslog`, of `/var/log/auth.log` or of `journalctl` has none, so
// the piece is optional: a pattern that required it rejected every line of the
// log directory of every machine.
export const PATTERNS = {
  'apache-combined':
    '%{IP:client} %{WORD:ident} %{WORD:user} \\[%{BRACKETED:timestamp}\\]'
    + ' "%{QUOTED:request}" %{INT:status} %{WORD:size}'
    + ' "%{QUOTED:referrer}" "%{QUOTED:agent}"',
  'syslog-3164':
    '(?:<%{INT:priority}>)?%{SYSLOGDATE:timestamp} %{WORD:host}'
    + ' %{NOTCOLON:tag}: %{GREEDY:message}',
  'nginx-error':
    '%{BRACKETED:timestamp} \\[%{BRACKETED:level}\\] %{WORD:pid}: %{GREEDY:message}',
};

const NAMED = /%\{(\w+):(\w+)\}/g;

/** Turn the named notation into a regular expression of this language. */
export function compilePattern(pattern) {
  const body = pattern.replace(NAMED, (whole, kind, name) => {
    if (!(kind in PIECES)) throw new Error(`unknown piece ${kind}`);
    return `(?<${name}>${PIECES[kind]})`;
  });
  return new RegExp(`^${body}$`);
}

/**
 * Every line cut into its fields, and every line that could not be.
 *
 * `pattern` is a name from `PATTERNS` or a pattern written in the same
 * notation. `rejected` carries the line number and the line itself: that list
 * is what tells you the format changed under you.
 *
 * `loose_pieces` names the pieces that can cut anywhere. Nothing is rejected
 * because of them — that is exactly the problem — so they are named instead.
 *
 * @param {Iterable<string>} lines
 * @param {string} pattern
 */
export function parseLines(lines, pattern) {
  const written = PATTERNS[pattern] ?? pattern;
  let expression;
  try {
    expression = compilePattern(written);
  } catch {
    return {
      parsed: [], rejected: [], loose_pieces: [], reason: 'this pattern is not usable',
    };
  }

  const parsed = [];
  const rejected = [];
  let number = 0;
  for (const line of lines) {
    number += 1;
    if (typeof line !== 'string') {
      rejected.push({ line: number, text: null });
      continue;
    }
    const text = line.replace(/\r?\n$/, '');
    const found = expression.exec(text);
    // An optional piece that did not match is `undefined` here and `None` in
    // Python; both sides of this entry answer null.
    if (found) {
      const fields = Object.fromEntries(
        Object.entries(found.groups).map(([name, value]) => [name, value ?? null]),
      );
      parsed.push({ line: number, ...fields });
    } else rejected.push({ line: number, text });
  }
  return { parsed, rejected, loose_pieces: loosePieces(written), reason: null };
}

/**
 * The loose pieces of a pattern that are not its last one.
 *
 * A loose piece in last position stops where the line does, which is what it
 * is for. Anywhere else it decides the boundary of the fields that follow, and
 * it decides it silently.
 */
export function loosePieces(pattern) {
  const found = [...pattern.matchAll(NAMED)];
  return found.slice(0, -1).filter(([, kind]) => LOOSE.includes(kind)).map(([, , name]) => name);
}