Commit 2e77398b by PLN (Algolia)

fix(tide-table): I reintroduced the reference-size bias in a new place, and measured it

`locate` normalises by reference size. `crossmatch` did not, and the number said
so: correlation(reference length, seconds won) = +0.586. Ceci (524s) and Piment
(506s) won most of the set; WAP and REVOLUTION won nothing at all. The cause is
that per-track score is a MAX over that track's windows, so a 524s reference
simply gets more chances at a high max than a 154s one.

Same asymmetry, second occurrence, new location — which is the pattern in
feedback_count_what_you_bound: a bias survives a fix when the metric moves.

The correction turns "how similar is this window to track T" into "how UNUSUALLY
similar, for track T", by standardising each track's column over the whole
query. A track generically close to everything now has a high mean and wins
nothing; a track that spikes in one place wins there. It assumes each track
occupies a minority of the recording — true for 15-30 tracks over 63 minutes,
false for a 2-track set, so it is stated rather than buried, and --no-standardise
shows the raw bias.

Two more things, both from PLN's "almost the same lineup" observation:

Montreuil26's tracks_bandcamp adds 15 more renders, and six of them are tracks
OPAL never had — Jeudi Drill, PunkAChien, L'Or Bleu, Premier Septembre, Quand on
decolle, Techno Orage. That directly explains the two disagreements between the
methods: `locate` was confident about rose and love_parade, and crossmatch
mislabelled both, because rose_rouge and livecode_parade are absent from the OPAL
reference set and it can only pick the nearest thing it holds.

Six tracks now appear in BOTH gigs, so their names must merge or one track splits
its own vote and each half loses. canon_track does that with an explicit alias
table rather than fuzzy matching: a wrong explicit merge is auditable, whereas a
silent fuzzy merge of two different tracks would be invisible in the output.

Also caches the query profile keyed on mtime and window geometry, because
recomputing 63 minutes of chroma_cqt per run costs five minutes and this tool
exists to be iterated on.
parent 22a9fc46
......@@ -451,20 +451,22 @@ def main(argv=None) -> int:
q = sub.add_parser("crossmatch", help="match against PRIOR RENDERS of the same tracks")
q.add_argument("recording")
q.add_argument("--refdir", default="/home/pln/Work/Sound/Prod/Opal26_master/tracks")
q.add_argument("--refdir", nargs="+", default=[
"/home/pln/Work/Sound/Prod/Opal26_master/tracks",
"/home/pln/Work/Sound/Prod/Montreuil26_master/tracks_bandcamp"])
q.add_argument("--no-standardise", dest="standardise", action="store_false",
help="disable the per-track standardisation (to see the raw bias)")
q.add_argument("--window", type=float, default=15.0)
q.add_argument("--hop-s", type=float, default=5.0)
q.add_argument("--smooth-s", type=float, default=45.0)
q.add_argument("--min-seg-s", type=float, default=40.0)
q.add_argument("--margin-warn", type=float, default=0.03)
q.add_argument("--no-cache", action="store_true")
q.set_defaults(func=cmd_crossmatch)
a = p.parse_args(argv)
return a.func(a)
if __name__ == "__main__":
sys.exit(main())
# --------------------------------------------------------------- crossmatch ----
#
......@@ -525,13 +527,67 @@ def _window_profiles(path, win, hop_s, sr=SR):
return out
# The same track appears in more than one gig's renders under different spellings:
# "07 - Piment Bresilien" (OPAL) and "02-Piment_brésilien" (Montreuil). Left alone
# they split one track's vote in two and each half loses. Merging them also doubles
# that track's reference mass, and gives a free cross-performance check.
#
# Explicit, not fuzzy: an alias table can be audited and a wrong merge is visible.
# A silent fuzzy match that merges two DIFFERENT tracks would be undetectable in
# the output, which is the failure mode that matters here.
_ALIASES = {
"cecinestpasunebombe": "bombe",
"quandondecolle": "quand_on_decolle",
"pimentbresilien": "piment_bresilien",
"takefivedrops": "take5_drops", "take5drops": "take5_drops",
"wap": "wap", "wipwap": "wap",
"youmysunshine": "you_my_sunshine",
"desire": "desire",
"larevolutionserasamplee": "revolution", "revolution": "revolution",
"mafiasansserif": "mafia", "mafia": "mafia",
"theressomethingaboutdrums": "something_about_drums",
"amidoingitright": "am_i_doing_it_right",
"ehouaisjefunk": "ouais_je_funk",
"perfect": "perfect",
"gimmeacid": "gimme_acid",
"ghostsinthetoilets": "ghosts_in_the_toilets",
"vaguedecrime": "vague_de_crime",
"jeudidrill": "jeudrill",
"punkachien": "punkachien",
"lorbleu": "blue_gold",
"premierseptembre": "sept1",
"technoorage": "techno_orage",
"plosive": "plosive",
"supersunnysideup": "super_sunny_side_up",
}
def canon_track(stem: str) -> str:
"""'07 - Piment Bresilien' / '02-Piment_brésilien' -> 'piment_bresilien'."""
import re as _re
import unicodedata as _ud
t = _ud.normalize("NFKD", stem)
t = "".join(c for c in t if not _ud.combining(c))
t = _re.sub(r"^\s*\d+\s*[-_.]*\s*", "", t) # leading track number
key = _re.sub(r"[^a-z0-9]", "", t.lower())
if key in _ALIASES:
return _ALIASES[key]
return _re.sub(r"[^a-z0-9]+", "_", t.lower()).strip("_")
def cmd_crossmatch(args) -> int:
refdir = pathlib.Path(args.refdir)
refs = sorted(p for p in refdir.iterdir()
if p.suffix.lower() in (".flac", ".wav", ".mp3", ".aiff"))
refdirs = [pathlib.Path(x) for x in args.refdir]
refs = []
for rd in refdirs:
if not rd.is_dir():
print(f" WARNING: {rd} is not a directory, skipping")
continue
refs += sorted(p for p in rd.iterdir()
if p.suffix.lower() in (".flac", ".wav", ".mp3", ".aiff"))
if not refs:
sys.exit(f"set_finder: no audio in {refdir}")
print(f"references: {len(refs)} tracks from {refdir.name}")
sys.exit(f"set_finder: no audio in {refdirs}")
refdir = refdirs[0]
print(f"references: {len(refs)} renders from {len(refdirs)} gig(s)")
print(f"features: {FEAT_DIM_NOTE}")
print(f"windows: {args.window}s every {args.hop_s}s\n")
......@@ -539,14 +595,31 @@ def cmd_crossmatch(args) -> int:
R_vecs, R_track = [], []
for p in refs:
prof = _window_profiles(p, args.window, args.hop_s)
ct = canon_track(p.stem)
for _, v in prof:
R_vecs.append(v); R_track.append(p.stem)
print(f" {p.stem[:44]:<46} {len(prof):>4} windows")
R_vecs.append(v); R_track.append(ct)
print(f" {p.stem[:40]:<42} -> {ct[:22]:<24} {len(prof):>4} windows")
R = np.stack(R_vecs)
print(f"\n{len(R)} reference windows in {time.time()-t0:.0f}s")
print("\nprofiling the query …")
q = _window_profiles(args.recording, args.window, args.hop_s)
# Cache the query profile. Recomputing 63 minutes of chroma_cqt on every run
# costs ~5 minutes and buys nothing — and this tool exists to be iterated on.
# Keyed on mtime and window geometry so a changed file or changed parameters
# invalidate it rather than silently returning a stale answer.
qp = pathlib.Path(args.recording)
ck = REPO / "armada" / "tide-table" / (
f".qprofile_{qp.stem}_{int(qp.stat().st_mtime)}"
f"_{args.window:g}_{args.hop_s:g}.npz")
if ck.exists() and not args.no_cache:
z = np.load(ck)
q = list(zip(z["t"].tolist(), list(z["v"])))
print(f"\nquery profile from cache ({ck.name}, {len(q)} windows)")
else:
print("\nprofiling the query …")
q = _window_profiles(args.recording, args.window, args.hop_s)
np.savez_compressed(ck, t=np.array([t for t, _ in q]),
v=np.stack([v for _, v in q]))
print(f" cached -> {ck.name}")
Q = np.stack([v for _, v in q]); Qt = [t for t, _ in q]
print(f" {len(Q)} query windows")
......@@ -570,6 +643,24 @@ def cmd_crossmatch(args) -> int:
cols = ref_of == i
per_track[:, i] = S[:, cols].max(axis=1)
# Measured bias, corrected: correlation(reference length, seconds won) was
# +0.586 on the first run. Per-track score is a MAX over that track's windows,
# so a 524s reference simply gets more chances at a high max than a 154s one —
# the same reference-size asymmetry already normalised out of `locate`, back in
# a new place (feedback_count_what_you_bound).
#
# The fix turns "how similar is this window to track T" into "how UNUSUALLY
# similar, for track T" by standardising each column over the whole query. A
# track that is generically close to everything has a high mean and therefore
# wins nothing; a track that spikes only in one place wins there.
#
# This assumes each track occupies a minority of the recording. True here (15-30
# tracks over 63 min) but it would break on a 2-track set, so it is an
# assumption worth stating rather than burying.
if args.standardise:
per_track = ((per_track - per_track.mean(axis=0))
/ (per_track.std(axis=0) + 1e-8))
# temporal smoothing: a track lasts minutes, so a single-window flicker is noise
k = max(1, int(args.smooth_s / args.hop_s) | 1)
if k > 1:
......@@ -619,3 +710,6 @@ def cmd_crossmatch(args) -> int:
}, indent=1))
print(f"\nwrote {out.name}")
return 0
if __name__ == "__main__":
sys.exit(main())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment