Commit a05d764f by PLN (Algolia)

measures: cadence engine — staccato, triads, antithesis, bookends, sentence cv

Document-level rhythm signals, report-only: they never touch the score.
Antithesis regexes are precision-first; the missed constructs and the
seam for a future smarter pass are documented in the module header.
Quoted spans stay exempt, same rule as the catalog.
parent 1fac43f6
/** buvard package entry — the catalog, the budgets, and the lint core. */ /** buvard package entry — the catalog, the budgets, the lint core, the measures. */
export { CATALOG, CATALOG_VERSION, BUDGETS } from './catalog.mjs'; export { CATALOG, CATALOG_VERSION, BUDGETS } from './catalog.mjs';
export { lint, bodyOf } from './lint.mjs'; export { lint, bodyOf } from './lint.mjs';
export { measuresOf, filterMeasures, MEASURES, MEASURE_GROUP } from './measures.mjs';
...@@ -22,7 +22,7 @@ const QUOTED_SPANS = [ ...@@ -22,7 +22,7 @@ const QUOTED_SPANS = [
/"[^"\n]*"/g, // straight double quotes, same line only /"[^"\n]*"/g, // straight double quotes, same line only
]; ];
function quotedRanges(text) { export function quotedRanges(text) {
const ranges = []; const ranges = [];
for (const re of QUOTED_SPANS) { for (const re of QUOTED_SPANS) {
re.lastIndex = 0; re.lastIndex = 0;
...@@ -32,7 +32,7 @@ function quotedRanges(text) { ...@@ -32,7 +32,7 @@ function quotedRanges(text) {
} }
/** Membership test by binary search over the sorted spans. */ /** Membership test by binary search over the sorted spans. */
function outsideTest(ranges) { export function outsideTest(ranges) {
return (index) => { return (index) => {
let lo = 0; let lo = 0;
let hi = ranges.length - 1; let hi = ranges.length - 1;
......
/**
* buvard cadence measures — document-level rhythm signals, report-only.
*
* The regex catalog judges words; this module measures the shape above them:
* the staccato beat, the triads, the antithesis engine, the bookends, the
* burstiness. These come from the diagnosis of a real article that scored
* 78/100 yet read unmistakably machine-flavored (test fixture:
* test/fixtures/article-excerpt.txt).
*
* PRODUCT RULE: measures are informational signals, never verdicts. They do
* not touch the score. The voice rule applies doubly here — a staccato
* paragraph can be the best thing in a text.
*
* Constructs the antithesis regexes deliberately MISS (high precision, low
* recall, like the catalog): balanced antitheses without "not" ("X here;
* Y there"), postposed contrast ("Y, not X"), "non seulement… mais…",
* negation spread across sentences with no it's/it-is anchor, and any
* contrast carried by morphology or parallelism alone. That is the design
* seam for a future optional smarter pass (dependency-parse or LLM-assisted);
* nothing in this module pretends to close it.
*
* Observed rates on a 15-text pre-2022 Gutenberg human corpus (report-only,
* no firing thresholds): docs/cadence-corpus.md.
*/
import { quotedRanges, outsideTest } from './lint.mjs';
/** Measure ids, their result key, and the group — selection operates on these. */
export const MEASURES = [
{ id: 'staccato', key: 'staccato_ratio', group: 'cadence' },
{ id: 'triads', key: 'triad_density', group: 'cadence' },
{ id: 'antithesis', key: 'antithesis_density', group: 'cadence' },
{ id: 'bookends', key: 'bookends_score', group: 'cadence' },
{ id: 'sentence-cv', key: 'sentence_length_cv', group: 'cadence' },
];
export const MEASURE_GROUP = 'cadence';
/**
* Precision-first triad: three single-word items, "X, Y and Z" / "X, Y, and
* Z" / "X, Y et Z". Multi-word items ("salt, fresh pepper and oil") are not
* caught — underestimate rather than over-catch.
*/
const TRIAD = /\b[\p{L}\p{N}][\p{L}\p{N}'’-]*,\s+[\p{L}\p{N}][\p{L}\p{N}'’-]*,?\s+(?:and|et)\s+[\p{L}\p{N}][\p{L}\p{N}'’-]*/giu;
/**
* Contrastive constructs. Each window is capped so a runaway "not" cannot
* span half a document; quoted spans are exempt (same rule as the catalog).
*/
const ANTITHESIS = [
// EN: "not X; it's / it is / that's / they are Y"
/\bnot\b[^.;!?\n—]{1,60}?\s*;\s*(?:it(?:'s| is)|that(?:'s| is)|they(?:'re| are)|this is)\b/gi,
// EN: "it's not X. It's Y." — the period pivot, anchored on the pronoun
/\bit'?s not\b[^.!?\n]{1,60}\.\s+(?:it'?s|this is|that'?s)\b/gi,
/\bis not\b[^.!?\n]{1,60}\.\s+it is\b/gi,
// EN: "isn't X — it's Y" — the tiret pivot
/\b(?:isn'?t|aren'?t|is not|are not)\b[^.!?\n—]{1,60}—\s*(?:it(?:'s| is)|they(?:'re| are)|but rather)\b/gi,
// FR: "ce n'est pas X, c'est Y" / "ce n'est pas X, mais bien Y"
/\bce n'est pas\b[^.!?\n;]{1,60}?[,;]\s*(?:c'est|mais bien)\b/gi,
// FR: "non pas X, mais Y"
/\bnon pas\b[^.!?\n;]{1,60}?\bmais\b/gi,
// FR: "ne … pas X, mais bien Y"
/\bne\b[^.!?\n;]{1,40}?\bpas\b[^.!?\n;]{1,40}?\bmais bien\b/gi,
];
/**
* Stopwords for the bookends overlap — one hand-rolled EN+FR list, kept
* deliberately small and closed: the goal is to drop function words, not to
* model a linguist's lexicon. "one" stays a content word on purpose (numbers
* carry meaning in codas). Nothing outside this list is filtered.
*/
const STOPWORDS = new Set(
`a an the and or but if then than that this these those it its its is are was were be been being am
do does did done have has had i you he she we they them his her their our your my me us him of in
on at to for from by with as into about over after before between out up down not no nor so too
very can will just dont there here when where what which who whom whose why how all any both each
few more most other some such only own same now also because while during through against above
below again further once
le la les un une des du de d l et ou mais donc ni car que qui quoi dont où ce cet cette ces son sa
ses leur leurs mon ma mes ton ta tes notre nos votre vos il elle ils elles on nous vous je tu me
te se lui y en au aux avec sans sous sur dans par pour pas plus moins très bien encore déjà est
sont était étaient a ont été être fait faire comme tout tous toute toutes même aussi si ne n c j
m t quand alors chez vers depuis pendant entre contre autre autres chose choses`
.split(/\s+/)
.filter(Boolean),
);
/** Paragraphs = blank-line-separated blocks; markdown headings are not prose. */
function paragraphs(text) {
return text
.split(/\n\s*\n/)
.map((p) => p.trim())
.filter((p) => p.length > 0 && !/^#{1,6}\s/.test(p));
}
/** Sentences inside one paragraph: terminal punctuation, whitespace-bounded. */
function sentences(paragraph) {
return paragraph.replace(/\s+/g, ' ').split(/(?<=[.!?…])\s+/).filter(Boolean);
}
function words(text) {
return text.split(/\s+/).filter(Boolean).length;
}
/** Content tokens for the bookends overlap: letters/digits, no stopwords. */
function contentTokens(paragraph) {
return paragraph
.toLowerCase()
.split(/[^\p{L}\p{N}]+/u)
.filter((t) => t.length > 1 && !STOPWORDS.has(t));
}
/**
* Compute all cadence measures for a text. Every value is a plain signal:
* staccato_ratio in [0,1], triads/antitheses per 1000 words, bookends_score
* (content-word Jaccard, first vs last paragraph, 0 when either side has no
* content tokens), sentence_length_cv (stddev/mean of sentence word counts).
*/
export function measuresOf(text) {
const paras = paragraphs(text).map((p) => ({ raw: p, sents: sentences(p), n: words(p) }));
const totalWords = paras.reduce((sum, p) => sum + p.n, 0);
const per1000 = (count) => (totalWords > 0 ? (count * 1000) / totalWords : 0);
const staccato = paras.filter((p) => p.sents.length === 1 && p.n <= 25).length;
const outside = outsideTest(quotedRanges(text));
let triads = 0;
for (const m of text.matchAll(TRIAD)) if (outside(m.index)) triads++;
let antitheses = 0;
for (const re of ANTITHESIS) {
re.lastIndex = 0;
for (const m of text.matchAll(re)) if (outside(m.index)) antitheses++;
}
const first = paras[0]?.raw ?? '';
const last = paras.at(-1)?.raw ?? '';
const head = new Set(contentTokens(first));
const tail = new Set(contentTokens(last));
let overlap = 0;
for (const t of head) if (tail.has(t)) overlap++;
const union = head.size + tail.size - overlap;
const bookends = union > 0 ? overlap / union : 0;
const lengths = paras.flatMap((p) => p.sents.map(words));
const mean = lengths.reduce((a, b) => a + b, 0) / (lengths.length || 1);
const variance = lengths.reduce((a, b) => a + (b - mean) ** 2, 0) / (lengths.length || 1);
const cv = mean > 0 ? Math.sqrt(variance) / mean : 0;
return {
group: MEASURE_GROUP,
words: totalWords,
paragraphs: paras.length,
staccato_count: staccato,
staccato_ratio: paras.length > 0 ? staccato / paras.length : 0,
triad_density: per1000(triads),
antithesis_density: per1000(antitheses),
bookends_score: bookends,
sentence_length_cv: cv,
};
}
/**
* Linter-style selection: tokens are measure ids or the group name
* ("cadence"). `select` restricts, `ignore` removes; ignore wins. Both empty
* means everything runs — the default, because these are report-only.
*/
export function filterMeasures(measures, { select = [], ignore = [] } = {}) {
const wanted = (token) => {
const t = token.toLowerCase();
return t === MEASURE_GROUP || MEASURES.some((m) => m.id === t);
};
const denied = new Set(ignore.filter(wanted));
const allowed = new Set(
MEASURES.filter((m) => !denied.has(m.id) && !denied.has(m.group)).map((m) => m.id),
);
if (select.length > 0) {
const picked = new Set(
select.filter(wanted).flatMap((token) => {
const t = token.toLowerCase();
return t === MEASURE_GROUP ? MEASURES.map((m) => m.id) : [t];
}),
);
for (const id of allowed) if (!picked.has(id)) allowed.delete(id);
}
const out = {};
for (const m of MEASURES) if (allowed.has(m.id)) out[m.id] = measures[m.key];
return out;
}
I gave Spotify Studio's agent every stream since 2009 and asked it the question no friend will answer honestly. It built me a lab, ran a 16-year backtest on my discoveries, and the verdict was not the one I ordered.
Who starts the discovery: you, or the queue? Since 2012 only 5 to 15 percent of first plays begin with a deliberate choice.
First plays split by how they began: a deliberate choice (click, search, my own playlist) versus a track that continued from the previous one. Since 2012 they have been 5 to 15 percent. I have not been discovering music; autoplay has been discovering music at me.
One in five. The queue throws a hundred doors at me a year and I walk back through five of them. But here is the thing: which five is not random, and that is where my defence begins.
My listening sessions are not shuffles; they are deliberate journeys through genre space. Someone is steering. It is just not the person clicking.
My taste is a real, compact, portable object. It has a shape. It is not a mood board; it is a fingerprint.
35,144 dots, each one an artist.
Another time.
The fix has a name and a cron schedule.
## So does it suck?
The humbling part is not that I have no taste. It is that I have a strong taste and a weak will.
The taste is provably mine: sticky, describable, consistent across the whole record. The discovery pipeline, though, has been running me since 2012, and I have been signing the output as "my music". The backtest caught me outsourcing curiosity and calling it identity.
Here is the verdict I ended up with, and it is not the one I ordered when I loaded the export files.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'fs';
import { measuresOf, filterMeasures } from '../lib/index.mjs';
const fixture = readFileSync(new URL('./fixtures/article-excerpt.txt', import.meta.url), 'utf8');
test('staccato counts one-sentence short paragraphs', () => {
const t = [
'The queue threw a hundred doors at me over the whole year, and in the end I only walked back through five of them, which is not many.',
'One in five.',
'Another time.',
].join('\n\n');
const m = measuresOf(t);
assert.equal(m.paragraphs, 3);
assert.equal(m.staccato_count, 2);
assert.ok(Math.abs(m.staccato_ratio - 2 / 3) < 1e-9);
});
test('a two-sentence paragraph is never staccato, whatever its length', () => {
const m = measuresOf('Short here. Still short.');
assert.equal(m.staccato_count, 0);
const m2 = measuresOf('One sentence only, but it stretches well past the twenty-five word ceiling that separates a deliberate beat from a paragraph that simply never learned where to stop and breathe.');
assert.equal(m2.staccato_count, 0);
});
test('triads: X, Y and Z in English and French, Oxford comma included', () => {
assert.ok(measuresOf('mixed across styles, eras and regions.').triad_density > 0);
assert.ok(measuresOf('salt, pepper, and oil.').triad_density > 0);
assert.ok(measuresOf('sel, poivre et huile.').triad_density > 0);
});
test('triads stay precise: multi-word items and plain pairs do not fire', () => {
assert.equal(measuresOf('fresh herbs, good olive oil and a lemon.').triad_density, 0);
assert.equal(measuresOf('bread and butter.').triad_density, 0);
});
test('antithesis: the EN and FR constructs the regexes own', () => {
const cases = [
'My listening sessions are not shuffles; they are deliberate journeys.',
"It's not a mood board. It's a fingerprint.",
'The score is not the point — it is the smoke alarm.',
"Ce n'est pas un outil, c'est un compagnon.",
'Non pas la vérité, mais sa silhouette.',
'Je ne cherche pas la gloire, mais bien le travail.',
];
for (const t of cases) assert.ok(measuresOf(t).antithesis_density > 0, t);
});
test('antithesis misses balanced contrast without "not" (documented gap)', () => {
assert.equal(measuresOf('X here; Y there. The two never meet.').antithesis_density, 0);
});
test('antithesis inside quotes is exempt, like the catalog', () => {
const t = 'Le genre cite : « Ce n’est pas un outil, c’est un compagnon. » Nous, on marche.';
assert.equal(measuresOf(t).antithesis_density, 0);
});
test('bookends: echoed coda scores high, unrelated paragraphs score zero', () => {
const echo = [
'The verdict was not the one I ordered, and I said so on day one.',
'Blah blah, unrelated middle stuff about entirely different matters entirely.',
'Here is the verdict: not the one I ordered.',
].join('\n\n');
assert.ok(measuresOf(echo).bookends_score > 0.3);
const stray = [
'Salt, fat, acid and heat govern every plate in this kitchen.',
'The compiler emits a warning whenever the integer overflows silently.',
].join('\n\n');
assert.equal(measuresOf(stray).bookends_score, 0);
});
test('sentence-length cv separates uniform droning from bursty prose', () => {
const uniform = Array(8).fill('The signal stays perfectly level across every single measured window here.').join(' ');
const bursty = 'Run. Then the whole thing collapses into one enormous, winding sentence that simply refuses to stop, piling clause on clause while the reader holds on. Short. Long again, far past every natural stopping point, wearing out its welcome with pure momentum. Stop.';
assert.ok(measuresOf(bursty).sentence_length_cv > measuresOf(uniform).sentence_length_cv * 2);
});
test('clean plain prose stays low on every measure', () => {
const clean = [
'Tuesday I moved the database to the new machine. The import itself took about four hours, and the row counts came out identical to the old box. One table needed its index rebuilt by hand before anything would query fast.',
'Next week we rerun the same import against the archive. Then we compare the row counts by hand, table by table, and write down what differs. If nothing differs, the migration is done.',
].join('\n\n');
const m = measuresOf(clean);
assert.equal(m.staccato_ratio, 0);
assert.equal(m.triad_density, 0);
assert.equal(m.antithesis_density, 0);
assert.ok(m.bookends_score < 0.15);
assert.ok(m.sentence_length_cv < 0.4);
});
test('the real-article excerpt fires: staccato, antithesis, bookends', () => {
const m = measuresOf(fixture);
assert.equal(m.paragraphs, 12);
assert.ok(Math.abs(m.staccato_ratio - 1 / 3) < 0.01, `staccato ${m.staccato_ratio}`);
assert.ok(m.antithesis_density > 5, `antithesis ${m.antithesis_density}`);
assert.ok(m.bookends_score > 0.1, `bookends ${m.bookends_score}`);
assert.ok(m.sentence_length_cv > 0.3);
});
test('selection: ignore kills a measure or the whole cadence group', () => {
const m = measuresOf(fixture);
assert.deepEqual(Object.keys(filterMeasures(m, { ignore: ['bookends'] })).sort(), [
'antithesis', 'sentence-cv', 'staccato', 'triads',
]);
assert.deepEqual(filterMeasures(m, { ignore: ['cadence'] }), {});
});
test('selection: select restricts to the named ids or group; ignore wins', () => {
const m = measuresOf(fixture);
assert.deepEqual(filterMeasures(m, { select: ['staccato', 'triads'] }), {
staccato: m.staccato_ratio,
triads: m.triad_density,
});
assert.deepEqual(
Object.keys(filterMeasures(m, { select: ['cadence'], ignore: ['staccato'] })).sort(),
['antithesis', 'bookends', 'sentence-cv', 'triads'],
);
});
test('selection defaults to everything, and nonsense tokens change nothing', () => {
const m = measuresOf(fixture);
assert.equal(Object.keys(filterMeasures(m)).length, 5);
assert.equal(Object.keys(filterMeasures(m, { select: ['nope'] })).length, 0);
assert.equal(Object.keys(filterMeasures(m, { ignore: ['nope'] })).length, 5);
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment