Commit a6320e43 by PLN (Algolia)

les-femmes: portraits from Wikipedia lead images — content-type plus magic-bytes…

les-femmes: portraits from Wikipedia lead images — content-type plus magic-bytes check rejects non-images (the NASA-html lesson), fr titles with en fallbacks, Commons credit per file, monogram fallback where no portrait is reliable
parent bb12d4e8
#!/usr/bin/env node
// Télécharge les portraits des femmes depuis l'API Wikipédia (image de tête
// de l'article) vers public/images/les-femmes/. Rejette tout ce qui n'est pas
// une image (content-type + magic bytes — leçon des downloads NASA en HTML).
// Usage: node scripts/les-femmes-images.mjs [--dry]
//
// Les titres d'articles viennent de app/les-femmes/data.ts (champ wikipedia).
import { mkdirSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { readFile } from "node:fs/promises";
import path from "node:path";
const ROOT = path.resolve(import.meta.dirname, "..");
const OUT_DIR = path.join(ROOT, "public", "images", "les-femmes");
const DRY = process.argv.includes("--dry");
// slug -> { lang, title } extrait à la main de data.ts (petit volume, pas de parse TS).
// Titres EN de secours quand l'article FR n'a pas d'image de tête (re-run du
// script pour les récupérer ; les fichiers existants sont sautés).
const FALLBACKS = [
["diotime-aspasie", "en", "Diotima of Mantinea"],
["maria-celeste", "en", "Sister Maria Celeste"],
];
const TARGETS = [
["jocaste-et-antigone", "fr", "Jocaste"],
["themistoclea-theano", "fr", "Théano"],
["diotime-aspasie", "fr", "Diotime de Mantinée"],
["sages-femmes", "fr", "Agnodice"],
["maria-celeste", "fr", "Maria Celeste"],
["elisabeth-de-boheme", "en", "Elisabeth of the Palatinate"],
["soeurs-pascal", "fr", "Jacqueline Pascal"],
["emilie-du-chatelet", "fr", "Émilie du Châtelet"],
["anna-maria-della-pieta", "en", "Anna Maria della Pietà"],
["jane-mecom", "en", "Jane Mecom"],
["geoffrin-lespinasse", "fr", "Marie Thérèse Rodet Geoffrin"],
["simonne-evrard", "fr", "Simonne Évrard"],
["olympe-de-gouges", "fr", "Olympe de Gouges"],
["nannette-streicher", "en", "Nannette Streicher"],
["johanna-et-adele-schopenhauer", "fr", "Johanna Schopenhauer"],
["clemence-royer", "fr", "Clémence Royer"],
["flora-tristan-helene-demuth", "fr", "Flora Tristan"],
["elisabeth-forster-nietzsche", "fr", "Elisabeth Förster-Nietzsche"],
["spielrein-pappenheim", "fr", "Sabina Spielrein"],
["marie-curie", "fr", "Marie Curie"],
["mileva-maric", "fr", "Mileva Marić"],
["dora-maar-francoise-gilot", "fr", "Dora Maar"],
["emily-gerard", "en", "Emily Gerard"],
["rosemonde-gerard", "fr", "Rosemonde Gérard"],
];
const UA = "les-femmes.me.nech.pl build script (contact: pln@nech.pl)";
async function fetchJson(url) {
const res = await fetch(url, { headers: { "User-Agent": UA } });
if (!res.ok) return { status: res.status };
return { status: res.status, json: await res.json() };
}
// Vérifie les magic bytes : refuse un HTML sauvé en .jpg (cf. commit dec2d0f).
function sniffImage(buf) {
if (buf.length < 12) return null;
if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg";
if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) return "image/png";
if (buf.slice(0, 3).toString("latin1") === "GIF") return "image/gif";
if (
buf.slice(8, 12).toString("latin1") === "WEBP" &&
buf.slice(0, 4).toString("latin1") === "RIFF"
)
return "image/webp";
return null;
}
async function download(url) {
const res = await fetch(url, { headers: { "User-Agent": UA } });
const ctype = res.headers.get("content-type") || "";
const buf = Buffer.from(await res.arrayBuffer());
const magic = sniffImage(buf);
if (!res.ok || !magic) {
throw new Error(`rejected: status=${res.status} content-type=${ctype} magic=${magic}`);
}
if (magic && ctype && !ctype.startsWith("image/")) {
throw new Error(`rejected: content-type ${ctype} but magic says ${magic}`);
}
return { buf, mime: magic };
}
const report = [];
mkdirSync(OUT_DIR, { recursive: true });
const ALL = [...TARGETS, ...FALLBACKS];
for (const [slug, lang, title] of ALL) {
const dest = path.join(OUT_DIR, `${slug}.jpg`);
if (existsSync(dest)) {
report.push({ slug, title, status: "skip (exists)" });
continue;
}
try {
// Action API : pageimages=name donne le nom de fichier Commons (pour le crédit).
const api = `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&prop=pageimages&piprop=name|original&redirects=1&titles=${encodeURIComponent(title)}`;
const { json, status } = await fetchJson(api);
if (status !== 200 || !json) throw new Error(`api status ${status}`);
const pages = json.query?.pages ?? {};
const page = Object.values(pages)[0];
if (!page?.pageimage || !page?.original?.source) {
throw new Error("no page image");
}
if (DRY) {
report.push({ slug, title, status: `would fetch: ${page.pageimage}` });
continue;
}
const { buf, mime } = await download(page.original.source);
const ext = mime === "image/png" ? "png" : mime === "image/webp" ? "webp" : "jpg";
const file = path.join(OUT_DIR, `${slug}.${ext}`);
writeFileSync(file, buf);
report.push({
slug,
title,
status: "ok",
file: path.basename(file),
bytes: buf.length,
commons: page.pageimage,
});
} catch (err) {
report.push({ slug, title, status: `FAIL: ${err.message}` });
}
}
console.table(report);
const failed = report.filter((r) => r.status.startsWith("FAIL"));
console.log(`${report.length - failed.length}/${report.length} ok, ${failed.length} failed`);
if (failed.length) process.exitCode = 1;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment