Commit 1d7f0b5a by PLN (Algolia)

lint: quoted speech, code spans and fences are exempt; masked occurrences reported

parent 417cea83
......@@ -9,6 +9,44 @@
import { CATALOG, BUDGETS } from './catalog.mjs';
/**
* Quoted spans are exempt: quoted speech, documentation quoting slop on
* purpose, code. A match counts only if it starts outside every span.
* Spans are bounded so an unpaired opener cannot swallow the document.
*/
const QUOTED_SPANS = [
/```[\s\S]*?(?:```|$)/g, // fenced code blocks
/`[^`\n]+`/g, // inline code
/«[^»]{0,400}»/g, // guillemets
/“[^”\n]{0,400}?”/g, // typographic double quotes
/"[^"\n]*"/g, // straight double quotes, same line only
];
function quotedRanges(text) {
const ranges = [];
for (const re of QUOTED_SPANS) {
re.lastIndex = 0;
for (const m of text.matchAll(re)) ranges.push([m.index, m.index + m[0].length]);
}
return ranges.sort((a, b) => a[0] - b[0]);
}
/** Membership test by binary search over the sorted spans. */
function outsideTest(ranges) {
return (index) => {
let lo = 0;
let hi = ranges.length - 1;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
const [s, e] = ranges[mid];
if (index < s) hi = mid - 1;
else if (index >= e) lo = mid + 1;
else return false; // inside a quoted span
}
return true;
};
}
/** Body = everything after the frontmatter fence (frontmatter is data, not prose). */
export function bodyOf(text) {
const parts = text.split('---');
......@@ -21,21 +59,28 @@ function excerpt(text, index) {
export function lint(text) {
const words = text.split(/\s+/).filter(Boolean).length;
const outside = outsideTest(quotedRanges(text));
const hits = [];
let score = 100;
let masked = 0;
for (const [id, w, , , re] of CATALOG) {
for (const m of text.matchAll(re)) {
if (!outside(m.index)) { masked++; continue; }
hits.push({ id, w, at: excerpt(text, m.index) });
score -= w;
}
}
for (const { re, perWords, floor, id, weight } of BUDGETS) {
const budget = Math.max(floor, Math.floor(words / perWords));
const n = (text.match(re) || []).length;
for (let i = budget; i < n; i++) {
hits.push({ id: `budget ${id} (>${budget}/${words} mots)`, w: weight, at: `occurrence ${i + 1}` });
let excess = 0;
for (const m of text.matchAll(re)) {
if (!outside(m.index)) { masked++; continue; }
if (excess >= budget) {
hits.push({ id: `budget ${id} (>${budget}/${words} mots)`, w: weight, at: `occurrence ${excess + 1}` });
score -= weight;
}
excess++;
}
}
return { words, hits, score: Math.max(0, score) };
return { words, hits, masked, score: Math.max(0, score) };
}
......@@ -36,3 +36,45 @@ test('empty body scores 100 and zero hits', () => {
const r = lint('');
assert.deepEqual([r.score, r.hits.length, r.words], [100, 0, 0]);
});
test('quoted speech is exempt', () => {
const slop = "Ce n'est pas un outil, c'est un compagnon de voyage";
const t = `Le document cite le genre : « ${slop} », et l'original : "${slop}".`;
const r = lint(t);
assert.deepEqual(r.hits, []);
assert.equal(r.masked, 2);
});
test('the same sentence outside quotes still counts', () => {
const r = lint("Ce n'est pas un outil, c'est un compagnon de voyage.");
assert.ok(r.hits.some((h) => h.id === 'contraste binaire'));
});
test('inline code and fenced blocks are exempt', () => {
const t = [
'Le motif est `Ce n\'est pas un outil, c\'est un compagnon` en ligne,',
'et en bloc :',
'```',
'Soyons honnêtes : le montage cassait.',
'De plus, il marche.',
'```',
'Fin.',
].join('\n');
const r = lint(t);
assert.deepEqual(r.hits, []);
assert.equal(r.masked, 3);
});
test('an unpaired guillemet does not swallow the rest', () => {
const t = '« citation jamais fermée. Soyons honnêtes : la suite compte quand même.';
const r = lint(t);
assert.ok(r.hits.some((h) => h.id === 'ouverture-vide'));
});
test('dashes inside quotes stay off the budget', () => {
const words = Array(600).fill('mot').join(' ');
const r = lint(`${words} « un — deux — trois » et un vrai — dehors.`);
// three dashes: two masked, one outside — within budget
assert.equal(r.hits.filter((h) => h.id.startsWith('budget tiret')).length, 0);
assert.equal(r.masked, 2);
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment