#!/usr/bin/env python3
"""pattern_ngrams — call each pattern a thing, then cluster tracks by shared ones.

Flat sound-token sets ("this track uses kick, cp, jazz") lose what livecoding is
made of: PHRASES. ParVagues reuses mini-notation phrases across tracks (a groove,
a fill, a bassline figure) and within a track (a section that comes back). This
tool lifts the catalog from flat tokens to **patterns**:

  • a PATTERN ("a thing") = a normalized mini-notation phrase — the `"…"` content
    of a sound source, with sample indices (`:4`) and whitespace canonicalized so
    `"kick:4 ~ kick:4"` and `"kick:2  ~ kick:2"` are the SAME thing. Each gets a
    stable id and a scope:
        unique    — appears once, in one track
        repeated  — appears ≥2× in ONE track  → a duplicated *section*
        shared    — appears in ≥2 tracks       → a shared *idiom*
  • track CLUSTERING — per track we shingle the sound-token sequence of its
    phrases into n-grams (n=2,3), drop boilerplate shingles (present in a large
    fraction of tracks — the TidalCycles equivalent of stop-words), and score
    track↔track similarity = Jaccard of the surviving shingle sets → neighbors.

This is the n-gram layer the locate-matrix L3 fingerprint (#34) wants, and the
substrate for a "sections" view. Pure functions + a validated artifact
(PatternRegistry, see models.py), tested mechanically (parsers-over-copy).

    python3 pattern_ngrams.py            # write pattern_registry.json + summary
    python3 pattern_ngrams.py <a.tidal>  # print one track's phrases + neighbors
"""
from __future__ import annotations
import glob
import hashlib
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

import tidal_score as ts
from tidal_score import SOURCE_CTX, _blocks, _strip_comment
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "tools"))
from sample_tfidf import SPLIT, sound_vocab  # noqa: E402

HERE = Path(__file__).parent
REPO = HERE.parent.parent
LIVES = Path("/home/pln/Work/Web/www/content/lives")  # app hoisted out of next/ in 6e51fff
OUT = HERE / "pattern_registry.json"
AS_OF = "2026-06-06"

NGRAM_NS = (2, 3)            # shingle sizes for clustering
MIN_PHRASE_TOKENS = 2       # a "thing" needs ≥2 alnum tokens (skip a lone sound name)
BOILERPLATE_DF = 0.15       # a shingle in >15% of tracks is a stop-word → ignored
MIN_TRACK_SHINGLES = 4      # below this a track's similarity is degenerate → no neighbors
NEIGHBOR_SIM = 0.12         # Jaccard floor to list a neighbor
NEIGHBOR_K = 6              # max neighbors per track
_INDEX = re.compile(r":\d+")
_WS = re.compile(r"\s+")
QUOTED = re.compile(r'"([^"]*)"')          # every quoted string, any context
# mini-notation structure markers — a string is a PHRASE (vs a bare label/path/synth
# name) if it carries musical structure: whitespace-separated steps or an operator.
_OP = set("~*<>[]!@/")


# ── normalization ─────────────────────────────────────────────────────────────
def normalize(q: str) -> str:
    """Canonical form of a mini-notation phrase: lowercase, drop sample indices
    (`kick:4`→`kick`), collapse whitespace. Keeps operators/structure (~ * [ ] < >
    . ! / ) so the rhythm/structure survives — only the surface noise is removed."""
    s = _INDEX.sub("", q.lower())
    return _WS.sub(" ", s).strip()


def phrase_tokens(norm: str) -> list[str]:
    """Alnum tokens of a normalized phrase, in order — sounds AND numbers (note/
    euclid steps carry the melodic/rhythmic shape), for n-gram shingling. Operators
    and rests are stripped by SPLIT; only structure-via-sequence survives."""
    return [t for t in SPLIT.split(norm) if t]


def is_phrase(q: str) -> bool:
    """A quoted string is a PHRASE (worth registering) when it carries musical
    structure — multiple steps or a mini-notation operator — rather than being a
    bare label / synth name / file path (e.g. `"moog"`, `"superpiano"`)."""
    norm = normalize(q)
    if not norm:
        return False
    if any(ch in _OP for ch in norm):
        return True
    return len(phrase_tokens(norm)) >= MIN_PHRASE_TOKENS


def shingles(tokens: list[str]):
    """All n=2,3 contiguous token n-grams of a token sequence."""
    out = set()
    for n in NGRAM_NS:
        for i in range(len(tokens) - n + 1):
            out.add(" ".join(tokens[i:i + n]))
    return out


def pid(norm: str) -> str:
    return "p" + hashlib.sha1(norm.encode("utf-8")).hexdigest()[:6]


# ── extraction ─────────────────────────────────────────────────────────────────
def track_phrases(path: Path):
    """All structured mini-notation phrases in a .tidal (sound sources, note/`n`
    patterns, gFunc arguments — every quoted string carrying structure), with
    orbit attribution where the phrase sits inside a dN block. Returns a list of
    (orbit|None, raw, norm). Comments are stripped first."""
    text = path.read_text(errors="ignore")
    counted = defaultdict(int)          # (orbit, raw) seen, to avoid double-count
    out = []
    # inside dN/pN blocks → orbit attribution
    for n, block in _blocks(text):
        for line in block.splitlines():
            for q in QUOTED.findall(line):     # block lines are already comment-stripped
                if is_phrase(q):
                    out.append((n, q, normalize(q)))
                    counted[(n, q)] += 1
    # phrases outside any dN block (let-defs, once/all, fx busses) — orbit unknown.
    # Re-scan whole file, attribute the EXTRA (uncounted) hits to orbit=None.
    whole = defaultdict(int)
    for raw in text.splitlines():
        for q in QUOTED.findall(_strip_comment(raw)):
            if is_phrase(q):
                whole[q] += 1
    in_block = defaultdict(int)
    for (n, q) in counted:
        in_block[q] += counted[(n, q)]
    for q, total in whole.items():
        extra = total - in_block.get(q, 0)
        for _ in range(max(0, extra)):
            out.append((None, q, normalize(q)))
    return out


# ── build ──────────────────────────────────────────────────────────────────────
def build():
    """Pure: scan every cited .tidal → PatternRegistry dict."""
    # corner-C names + the set of tracks the catalog actually cites
    names: dict[str, str] = {}
    files: set[str] = set()
    for f in glob.glob(str(LIVES / "**/tracks.json"), recursive=True):
        try:
            d = json.load(open(f))
        except Exception:
            continue
        for tr in (d.get("tracks") if isinstance(d, dict) else d) or []:
            fp = tr.get("file")
            if fp:
                files.add(fp)
                if tr.get("name"):
                    names.setdefault(fp, tr["name"])

    # registry: norm -> occurrences; per-track shingle sets + phrase-id sets
    reg: dict[str, dict] = {}
    track_shingles: dict[str, set] = {}
    track_phrase_ids: dict[str, set] = {}
    for fp in sorted(files):
        path = REPO / fp
        if not path.exists():
            continue
        shg: set = set()
        pids: set = set()
        per_track_count: dict[str, int] = defaultdict(int)
        reps_raw: dict[str, str] = {}
        reps_orbit: dict[str, int | None] = {}
        for orbit, raw, norm in track_phrases(path):
            toks = phrase_tokens(norm)
            shg |= shingles(toks)
            if len(toks) < MIN_PHRASE_TOKENS:
                continue
            per_track_count[norm] += 1
            reps_raw.setdefault(norm, raw)
            reps_orbit.setdefault(norm, orbit)
            pids.add(pid(norm))
        track_shingles[fp] = shg
        track_phrase_ids[fp] = pids
        for norm, cnt in per_track_count.items():
            e = reg.setdefault(norm, {"occ": [], "n_total": 0})
            e["occ"].append({"track": fp, "name": names.get(fp, Path(fp).stem),
                             "orbit": reps_orbit[norm], "count": cnt,
                             "raw": reps_raw[norm]})
            e["n_total"] += cnt

    # df over shingles → boilerplate stop-words excluded from similarity
    n_tracks = len([fp for fp in files if (REPO / fp).exists()])
    sdf: dict[str, int] = defaultdict(int)
    for shg in track_shingles.values():
        for s in shg:
            sdf[s] += 1
    cap = max(2, int(BOILERPLATE_DF * n_tracks))
    sig_shingles = {fp: {s for s in shg if sdf[s] <= cap}
                    for fp, shg in track_shingles.items()}

    # pattern entries
    patterns = []
    for norm, e in reg.items():
        occ = e["occ"]
        n_tr = len(occ)
        scope = ("shared" if n_tr >= 2
                 else "repeated" if any(o["count"] >= 2 for o in occ)
                 else "unique")
        patterns.append({
            "id": pid(norm), "norm": norm,
            "n_tokens": len(phrase_tokens(norm)),
            "n_tracks": n_tr, "n_total": e["n_total"], "scope": scope,
            "occurrences": sorted(occ, key=lambda o: (-o["count"], o["track"])),
        })
    patterns.sort(key=lambda p: (-p["n_tracks"], -p["n_total"], p["norm"]))

    # phrase ids that are genuinely shared (for neighbor "shared_patterns")
    shared_pids = {p["id"] for p in patterns if p["scope"] == "shared"}

    # track clustering — Jaccard of significant shingle sets
    fps = sorted(sig_shingles)
    tracks_out = []
    for a in fps:
        sa = sig_shingles[a]
        nb = []
        # tracks with too few distinctive shingles produce degenerate (1.0) matches
        for b in (fps if len(sa) >= MIN_TRACK_SHINGLES else []):
            if b == a:
                continue
            sb = sig_shingles[b]
            if len(sb) < MIN_TRACK_SHINGLES:
                continue
            inter = len(sa & sb)
            if not inter:
                continue
            sim = inter / len(sa | sb)
            if sim >= NEIGHBOR_SIM:
                shared = sorted(track_phrase_ids[a] & track_phrase_ids[b] & shared_pids)
                nb.append({"track": b, "name": names.get(b, Path(b).stem),
                           "similarity": round(sim, 3), "shared_patterns": shared})
        nb.sort(key=lambda x: -x["similarity"])
        n_shared_here = len(track_phrase_ids[a] & shared_pids)
        tracks_out.append({
            "track": a, "name": names.get(a, Path(a).stem),
            "n_phrases": len(track_phrase_ids[a]),
            "n_shared": n_shared_here,
            "neighbors": nb[:NEIGHBOR_K],
        })

    return {
        "schema": "pattern-registry v1 (mini-notation phrases as things + "
                  "n-gram track clustering)",
        "as_of": AS_OF,
        "n_tracks": n_tracks,
        "n_patterns": len(patterns),
        "n_shared": sum(1 for p in patterns if p["scope"] == "shared"),
        "n_repeated": sum(1 for p in patterns if p["scope"] == "repeated"),
        "patterns": patterns,
        "tracks": tracks_out,
    }


def main():
    if len(sys.argv) > 1 and sys.argv[1].endswith(".tidal"):
        _one(sys.argv[1])
        return
    out = build()
    from models import PatternRegistry            # DRY contract: validate on emit
    PatternRegistry.model_validate(out)
    OUT.write_text(json.dumps(out, ensure_ascii=False, indent=1))
    print(f"✓ {OUT}")
    print(f"  tracks {out['n_tracks']}  ·  distinct patterns {out['n_patterns']}  "
          f"·  shared {out['n_shared']}  ·  repeated {out['n_repeated']}")
    top = [p for p in out["patterns"] if p["scope"] == "shared"][:10]
    if top:
        print("\n  most-shared phrases (a thing → tracks):")
        for p in top:
            print(f"    {p['id']}  ×{p['n_tracks']:<2} {p['norm'][:54]!r}")
    # tightest track pairs
    pairs = []
    seen = set()
    for t in out["tracks"]:
        for nb in t["neighbors"]:
            k = tuple(sorted((t["track"], nb["track"])))
            if k in seen:
                continue
            seen.add(k)
            pairs.append((nb["similarity"], t["name"], nb["name"]))
    pairs.sort(reverse=True)
    if pairs:
        print("\n  closest track pairs (shared n-gram Jaccard):")
        for sim, a, b in pairs[:10]:
            print(f"    {sim:.2f}  {a}  ~  {b}")


def _one(path):
    fp = str(Path(path))
    out = build()
    t = next((x for x in out["tracks"] if x["track"].endswith(Path(fp).name)), None)
    ph = [p for p in out["patterns"]
          if any(o["track"].endswith(Path(fp).name) for o in p["occurrences"])]
    print(f"\n{Path(fp).name} — {len(ph)} phrases\n")
    for p in sorted(ph, key=lambda p: -p["n_total"]):
        mark = {"shared": "↔", "repeated": "⟳", "unique": " "}[p["scope"]]
        print(f"  {mark} {p['id']}  {p['norm'][:60]!r}")
    if t:
        print(f"\n  neighbors of {t['name']}:")
        for nb in t["neighbors"]:
            print(f"    {nb['similarity']:.2f}  {nb['name']}  "
                  f"({len(nb['shared_patterns'])} shared things)")


if __name__ == "__main__":
    main()
