#!/usr/bin/env python3
"""Phase-1 data-scientist EDA over the L'Armada corpus — find the stories before any viz.

Loads the generated artifacts (catalog_view, pattern_registry, tokens ontology) plus the
canonical site gigs and the Dirt-Samples symlink dates, and computes tidy cuts:

  tempo      TRUE bpm parsed from each track's `setcps` (corner A / the score), the
             metadata-bpm claim (corner C), and the A↔C delta — never trust the label.
  cadence    all 37 canonical gigs by year (not just the 23 with tracklists).
  styles     style mix + label-fragmentation (nujazz/nu-jazz, breaks/breakbeat…).
  palette    most-used sounds; sample-family split via the fleet classifier (DRY).
  idioms     shared/repeated mini-notation phrases; the gMask/gMute signature ubiquity.
  recurrence most-performed tracks (gig appearances) = the set-staples.
  agreement  the A↔C reconciliation taxonomy.
  collab     per-collaborator track counts.
  vocabulary custom-sample packs entering the vocabulary over time (symlink mtime).

Emits `eda_report.json` (tidy, typed-ish) AND prints a narrative. Honest about coverage
(BPM/style cuts only span gigs with a tracklist). parsers-over-copy; no invented values.
"""
import json
import re
import sys
import glob
import subprocess
import statistics as st
import unicodedata
from collections import Counter, defaultdict
from pathlib import Path

HERE = Path(__file__).parent
ROOT = HERE.parent.parent                      # …/Sound/Tidal
sys.path.insert(0, str(ROOT / "tools"))
import models as M  # noqa: E402

LIVES = Path("/home/pln/Work/Web/www/content/lives")  # app hoisted out of next/ in 6e51fff
DIRT = Path.home() / ".local/share/SuperCollider/downloaded-quarks/Dirt-Samples"
CV = HERE / "catalog_view.json"
PR = HERE / "pattern_registry.json"
OUT = HERE / "eda_report.json"
AS_OF = "2026-06-06"


# ── tempo: parse TRUE bpm from setcps (corner A) ──────────────────────────────
def _evalnum(s):
    """Eval a tiny arithmetic blob like '99 - 20' or '99'. Safe: digits/+-/space only."""
    s = s.strip().strip("()").strip()
    if re.fullmatch(r"[\d\s.+\-]+", s):
        try:
            return float(eval(s, {"__builtins__": {}}))  # noqa: S307 — constrained charset
        except Exception:
            return None
    return None


def parse_tempo(src):
    """Return {bpm, lo, hi, morph} from a .tidal source, or None. bpm = primary tempo."""
    if not src:
        return None
    # drop Tidal line comments (`-- …`) so commented-out experiments don't count
    # (e.g. ton_numero has a `-- # cps (range …)` morpher that was never live)
    src = re.sub(r"--[^\n]*", "", src)
    # all setcps / cps / cpsbus / "# cps" statements
    cands = []
    for m in re.finditer(r"(?:setcps|cpsbus\s*\d+|#\s*cps|\bcps)\s*\(?(.+)", src):
        expr = m.group(1)
        rng = re.search(r"range\s*\(?\s*([\d.\s+\-]+?)\)?\s+\(?\s*([\d.\s+\-]+?)\)?\s*[\"/)]", expr)
        ndiv = re.search(r"(\d+(?:\.\d+)?)\s*/\s*60\s*/\s*4", expr)
        bare = re.match(r"\s*(0?\.\d+|\d+\.\d+)\s*$", expr)
        if rng:
            lo, hi = _evalnum(rng.group(1)), _evalnum(rng.group(2))
            if lo and hi:
                cands.append({"bpm": (lo + hi) / 2, "lo": lo, "hi": hi, "morph": True})
                continue
        if ndiv:
            cands.append({"bpm": float(ndiv.group(1)), "lo": None, "hi": None, "morph": False})
        elif bare:
            cands.append({"bpm": float(bare.group(1)) * 240, "lo": None, "hi": None, "morph": False})
    if not cands:
        return None
    # primary BPM = the fixed setcps (the declared base tempo) if there is one,
    # else the first range's midpoint. But a track that ALSO has a `cps (range …)`
    # morpher is a morpher (sept1 declares 120 then sweeps 60→180): keep the base
    # bpm for sorting, flag the morph, and carry its span.
    fixed = [c for c in cands if not c["morph"]]
    morphs = [c for c in cands if c["morph"]]
    out = dict((fixed or cands)[0])
    if morphs:
        out["morph"] = True
        out["lo"], out["hi"] = morphs[0]["lo"], morphs[0]["hi"]
    return out


# ── sample-family classifier (DRY: the canonical SSOT classifier in models.py) ─
sample_family = M.classify_sample_family


def gig_year_index():
    """All canonical gigs (every .md), date+year — the honest denominator."""
    gigs = {}
    for f in glob.glob(str(LIVES / "20*/*.md")):
        txt = Path(f).read_text(errors="ignore")
        m = re.search(r"^date:\s*\"?(\d{4}-\d{2}-\d{2})", txt, re.M)
        if m:
            slug = str(Path(f).relative_to(LIVES))[:-3]
            gigs[slug] = m.group(1)
    return gigs


def metadata_bpm():
    """Per-gig metadata bpm/style from tracks.json (corner C)."""
    rows = []
    for f in glob.glob(str(LIVES / "**/tracks.json"), recursive=True):
        d = json.load(open(f))
        slug = str(Path(f).parent.relative_to(LIVES))
        for t in (d.get("tracks") or []):
            rows.append({"gig": slug, "date": (d.get("date") or "")[:10],
                         "file": t.get("file"), "name": t.get("name"),
                         "bpm": t.get("bpm"), "style": M.norm_style(t.get("style"))})
    return rows


def vocabulary_growth():
    """Custom-sample packs (symlinks) entering the vocab, by month (link mtime)."""
    import os
    months = Counter()
    if not DIRT.exists():
        return months
    for p in DIRT.iterdir():
        if p.is_symlink():
            ts = p.lstat().st_mtime
            import time
            months[time.strftime("%Y-%m", time.localtime(ts))] += 1
    return months


def git_creation_dates():
    """Earliest git add-date per .tidal (true 'studio creation' date, not mtime which
    every reformat touches). git log is reverse-chron, so the LAST occurrence wins."""
    out = subprocess.run(
        ["git", "-C", str(ROOT), "log", "--diff-filter=A", "--format=%as",
         "--name-only", "--", "*.tidal"],
        capture_output=True, text=True).stdout
    date, first = None, {}
    for line in out.splitlines():
        if re.fullmatch(r"\d{4}-\d{2}-\d{2}", line.strip()):
            date = line.strip()
        elif line.strip().endswith(".tidal") and date:
            first[line.strip()] = date          # overwrite → ends at earliest
    return first


def collab_fingerprint(T, tempo):
    """Per-collaborator tell: bpm profile + distinctive samples (lift vs corpus) + idioms.
    'nova' = ParVagues solo (live/midi/nova); collab/<who> = co-writes."""
    def who(track):
        m = re.match(r"live/collab/([^/]+)/", track)
        if m:
            return m.group(1)
        if track.startswith("live/midi/nova/"):
            return "nova(solo)"
        return None

    groups = defaultdict(list)
    for t in T:
        w = who(t["track"])
        if w:
            groups[w].append(t)
    global_snd = Counter()
    for t in T:
        for s in set(t["score_sounds"]):
            global_snd[s] += 1
    n_all = len(T)

    out = {}
    for w, tracks in groups.items():
        if len(tracks) < 2:
            continue
        bpms = [tempo[t["track"]]["bpm"] for t in tracks if t["track"] in tempo]
        gsnd = Counter()
        for t in tracks:
            for s in set(t["score_sounds"]):
                gsnd[s] += 1
        n = len(tracks)
        # lift = (freq in group) / (freq in corpus); keep samples used ≥2× in group
        distinctive = sorted(
            ((s, gsnd[s] / n / max(global_snd[s] / n_all, 1e-9), gsnd[s])
             for s in gsnd if gsnd[s] >= 2),
            key=lambda x: -x[1])[:6]
        out[w] = {
            "n": n,
            "bpm_median": round(st.median(bpms)) if bpms else None,
            "bpm_min": min(bpms) if bpms else None,
            "bpm_max": max(bpms) if bpms else None,
            "tracks": sorted(({"name": t["name"], "bpm": tempo[t["track"]]["bpm"]}
                              for t in tracks if t["track"] in tempo),
                             key=lambda x: x["bpm"]),
            "distinctive_samples": [{"sound": s, "lift": round(l, 1), "n": c}
                                    for s, l, c in distinctive],
        }
    return out


def cooccurrence(T):
    """Sound co-occurrence within tracks → pair counts + per-sound partner index.
    Returns (pair_counts:Counter[(a,b)], partners:dict[sound]->Counter, doc_freq)."""
    from itertools import combinations
    pair = Counter()
    partners = defaultdict(Counter)
    df = Counter()
    for t in T:
        sounds = sorted(set(t["score_sounds"]))
        for s in sounds:
            df[s] += 1
        for a, b in combinations(sounds, 2):
            pair[(a, b)] += 1
            partners[a][b] += 1
            partners[b][a] += 1
    return pair, partners, df


def pairings_for(prefix, T, top=12):
    """What pairs well with `prefix`* — lift-weighted partners (the 🍷 question)."""
    pair, partners, df = cooccurrence(T)
    n = len(T)
    hits = [s for s in df if s.lower().startswith(prefix.lower())]
    if not hits:
        return None, []
    agg = Counter()
    base = 0
    for s in hits:
        base += df[s]
        for p, c in partners[s].items():
            agg[p] += c
    # lift = P(partner|hit) / P(partner) ; require co-occur ≥2
    ranked = sorted(
        ((p, c, (c / base) / max(df[p] / n, 1e-9)) for p, c in agg.items() if c >= 1
         and not p.lower().startswith(prefix.lower())),
        key=lambda x: (-x[2], -x[1]))[:top]
    return hits, ranked


def build():
    cv = json.load(open(CV))
    pr = json.load(open(PR))
    T = cv["tracks"]
    gigs = gig_year_index()
    meta = metadata_bpm()
    meta_by_file = {m["file"]: m for m in meta if m.get("file")}

    # tempo per track (corner A) + A↔C delta
    tempo = {}
    ac_delta = []
    for t in T:
        tp = parse_tempo(t.get("source"))
        if tp:
            tempo[t["track"]] = tp
            mc = meta_by_file.get(t["track"])
            if mc and isinstance(mc.get("bpm"), (int, float)) and not tp["morph"]:
                ac_delta.append({"track": t["name"], "score_bpm": tp["bpm"],
                                 "meta_bpm": mc["bpm"], "delta": tp["bpm"] - mc["bpm"]})

    bpms = [tp["bpm"] for tp in tempo.values()]
    morph = [k for k, v in tempo.items() if v["morph"]]

    # creation-BPM story: tempo vs git-creation date (studio tempo, not stage tempo)
    created = git_creation_dates()
    by_creation = []
    for t in T:
        cdate = created.get(t["track"])
        if cdate and t["track"] in tempo:
            by_creation.append({"name": t["name"], "track": t["track"],
                                "created": cdate, "bpm": tempo[t["track"]]["bpm"],
                                "morph": tempo[t["track"]]["morph"]})
    by_creation.sort(key=lambda x: x["created"])
    creation_year_bpm = defaultdict(list)
    for x in by_creation:
        creation_year_bpm[x["created"][:4]].append(x["bpm"])
    creation_tempo = {y: round(st.median(v)) for y, v in sorted(creation_year_bpm.items())}

    # stage-BPM story: tempo of what was actually PERFORMED each year (club lens).
    # Attribute a track's score-tempo to every gig-year it appears in (one vote per
    # performance), then take the year median. The "in the club" counterpart to the
    # studio (creation) line above — same tracks, weighted by when they hit a stage.
    stage_year_bpm = defaultdict(list)
    for t in T:
        tp = tempo.get(t["track"])
        if not tp:
            continue
        for g in t.get("gigs", []):
            gdate = gigs.get(g)
            if gdate:
                stage_year_bpm[gdate[:4]].append(tp["bpm"])
    stage_tempo = {y: round(st.median(v)) for y, v in sorted(stage_year_bpm.items())}

    # palette + families
    snd = Counter()
    for t in T:
        for s in t["score_sounds"]:
            snd[s] += 1
    fam, unclassified = Counter(), Counter()
    for t in T:
        for s in t["score_sounds"]:
            k = sample_family(s)
            (fam if k else unclassified)[k or s] += 1

    # styles (corner C)
    sty = Counter(m["style"] for m in meta if m.get("style"))

    # idioms
    shared = sorted((p for p in pr["patterns"] if p["scope"] == "shared"),
                    key=lambda p: -p["n_tracks"])
    repeated = [p for p in pr["patterns"] if p["scope"] == "repeated"]

    # recurrence
    recurrence = sorted(({"name": t["name"], "gigs": len(t["gigs"]),
                          "track": t["track"], "gig_slugs": sorted(t.get("gigs", []))}
                         for t in T),
                        key=lambda x: -x["gigs"])

    # collab dimension (live/collab/<who>/…)
    collab = Counter()
    for t in T:
        m = re.match(r"live/collab/([^/]+)/", t["track"])
        if m:
            collab[m.group(1)] += 1

    report = {
        "schema": "eda phase-1 (exploratory; tidy cuts for viz)", "as_of": AS_OF,
        "coverage": {"tracks": len(T), "canonical_gigs": len(gigs),
                     "gigs_with_tracklist": cv["stats"]["with_metadata"],
                     "tracks_with_tempo": len(tempo),
                     "tracks_with_creation_date": len(by_creation)},
        "tempo": {
            "n": len(bpms), "min": min(bpms), "max": max(bpms),
            "median": st.median(bpms), "mean": round(st.mean(bpms), 1),
            "morphing_tracks": len(morph),
            "histogram": dict(sorted(Counter(int(b // 10) * 10 for b in bpms).items())),
            "ac_delta": sorted(ac_delta, key=lambda x: -abs(x["delta"])),
            "creation_tempo_by_year": creation_tempo,
            "stage_tempo_by_year": stage_tempo,
            "by_creation": by_creation,
        },
        "collab_fingerprint": collab_fingerprint(T, tempo),
        "pairings_top": [{"a": a, "b": b, "n": c}
                         for (a, b), c in cooccurrence(T)[0].most_common(20)],
        "cadence": dict(sorted(Counter(d[:4] for d in gigs.values()).items())),
        "styles": dict(sty.most_common()),
        "palette_top": dict(snd.most_common(25)),
        "families": dict(fam.most_common()),
        # honest coverage: a sample-NAME only encodes an instrument sometimes; kits,
        # drum machines and source-named chops (wap, take5…) need analysis, not a guess.
        "classified_uses": sum(fam.values()),
        "unclassified_uses": sum(unclassified.values()),
        "unclassified_names": len(unclassified),
        "unclassified_top": dict(unclassified.most_common(15)),
        "idioms_top": [{"norm": p["norm"], "n_tracks": p["n_tracks"]} for p in shared[:15]],
        "idioms_counts": {"shared": len(shared), "repeated": len(repeated),
                          "total": pr["n_patterns"]},
        "recurrence_top": recurrence[:20],
        "collab": dict(collab.most_common()),
        "vocabulary_growth": dict(sorted(vocabulary_growth().items())),
        # which inputs fed this build (shown in the viz footer; append as we add more)
        "sources": [
            {"key": "score", "label": ".tidal scores",
             "detail": "tempo, sample palette, phrases"},
            {"key": "site", "label": "site gig metadata",
             "detail": "style tags, bpm, setlists"},
            {"key": "git", "label": "git history", "detail": "track creation dates"},
            {"key": "dirt", "label": "Dirt-Samples links",
             "detail": "sample-folder import dates"},
        ],
        # near-future sources (kept honest: shown as "coming", not faked)
        "roadmap": ["platform play counts", "git edit counts", "per-set durations"],
    }
    return report


def _bar(n, mx, w=32):
    return "█" * round(w * n / mx) if mx else ""


def main():
    r = build()
    OUT.write_text(json.dumps(r, ensure_ascii=False, indent=1))
    c = r["coverage"]
    print(f"✓ {OUT}")
    print(f"\ncoverage: {c['tracks']} tracks · {c['canonical_gigs']} canonical gigs · "
          f"{c['tracks_with_tempo']} with parsed tempo\n")

    t = r["tempo"]
    print(f"── TEMPO (true, from setcps) ── {t['min']:.0f}–{t['max']:.0f} bpm, "
          f"median {t['median']:.0f}, {t['morphing_tracks']} morphing")
    mx = max(t["histogram"].values())
    for b, n in t["histogram"].items():
        print(f"   {b:>3}s {_bar(n, mx, 24)} {n}")
    print("   biggest score↔metadata bpm gaps:")
    for d in t["ac_delta"][:6]:
        print(f"     {d['track']:<26} score {d['score_bpm']:.0f} vs meta {d['meta_bpm']:.0f}  ({d['delta']:+.0f})")

    print("\n── GIG CADENCE (all 37 canonical) ──")
    mx = max(r["cadence"].values())
    for y, n in r["cadence"].items():
        print(f"   {y} {_bar(n, mx, 24)} {n}")

    print("\n── SAMPLE FAMILIES ──")
    tot = sum(r["families"].values())
    for k, n in r["families"].items():
        print(f"   {k:<7} {_bar(n, tot, 24)} {n:>3} ({100*n/tot:.0f}%)")
    print(f"   unclassified: {sum(r['unclassified_top'].values())}+ uses "
          f"(top: {', '.join(list(r['unclassified_top'])[:6])})")

    print("\n── SIGNATURE IDIOMS (shared phrases) ──")
    for p in r["idioms_top"][:8]:
        print(f"   {p['n_tracks']:>2} tracks  «{p['norm'][:48]}»")

    print("\n── SET-STAPLES (most performed) ──")
    for x in r["recurrence_top"][:10]:
        print(f"   {x['gigs']:>2} gigs  {x['name']}")

    print("\n── CREATION TEMPO (studio, by git-add year) vs stage tempo ──")
    for y, b in r["tempo"]["creation_tempo_by_year"].items():
        print(f"   {y}  median {b} bpm")

    print("\n── COLLAB FINGERPRINT (who betrays themselves) ──")
    fp = r["collab_fingerprint"]
    for w in sorted(fp, key=lambda k: -(fp[k]["bpm_median"] or 0)):
        f = fp[w]
        tells = ", ".join(f"{d['sound']}(×{d['n']})" for d in f["distinctive_samples"][:4])
        rng = f"{f['bpm_min']:.0f}–{f['bpm_max']:.0f}" if f["bpm_min"] else "—"
        print(f"   {w:<14} n={f['n']:<2} bpm~{f['bpm_median']} [{rng}]  tells: {tells}")

    print("\n── COLLAB ──", "  ".join(f"{k}:{v}" for k, v in r["collab"].items()))
    print("\n── VOCABULARY GROWTH (custom packs/month) ──")
    vg = r["vocabulary_growth"]
    mx = max(vg.values())
    for mth, n in vg.items():
        print(f"   {mth} {_bar(n, mx, 28)} {n}")


def cmd_pair(prefix):
    """Ad-hoc: `tide_eda.py pair bogdan` → what `bogdan*` pairs well with 🍷."""
    T = json.load(open(CV))["tracks"]
    hits, ranked = pairings_for(prefix, T)
    if not hits:
        print(f"no sound matches «{prefix}*»")
        return
    print(f"«{prefix}*» = {', '.join(hits)}  →  pairs best with (lift-ranked):")
    for p, c, lift in ranked:
        fam = sample_family(p) or "?"
        print(f"   {c:>2}× lift {lift:>4.1f}  {p:<18} [{fam}]")


if __name__ == "__main__":
    if len(sys.argv) > 2 and sys.argv[1] == "pair":
        cmd_pair(sys.argv[2])
    else:
        main()
