mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(resilience): cross-index benchmark — INFORM/ND-GAIN/WRI/FSI (Phase 2 T2.4) (#2985)
* feat(resilience): cross-index benchmark — INFORM/ND-GAIN/WRI/FSI (Phase 2 T2.4) * fix(resilience): INFORM scoreCol guard + Redis check + cleanup (#2985 review)
This commit is contained in:
Elie Habib
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"generatedAt": 0,
|
||||
"license": "FSI data: Fund for Peace, non-commercial academic license. For internal validation only.",
|
||||
"hypotheses": [],
|
||||
"correlations": {},
|
||||
"outliers": [],
|
||||
"sourceStatus": {},
|
||||
"_note": "Placeholder. Run `node scripts/benchmark-resilience-external.mjs` to populate with live data."
|
||||
}
|
||||
449
scripts/benchmark-resilience-external.mjs
Normal file
449
scripts/benchmark-resilience-external.mjs
Normal file
@@ -0,0 +1,449 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Cross-index benchmark: compares WorldMonitor resilience scores against
|
||||
// INFORM Global, ND-GAIN, WorldRiskIndex, and FSI using Spearman/Pearson.
|
||||
//
|
||||
// FSI data sourced from the Fund for Peace under non-commercial academic license.
|
||||
// WorldMonitor uses FSI scores for internal validation benchmarking only.
|
||||
// FSI scores are NOT displayed in the product UI or included in the public ranking.
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { loadEnvFile, getRedisCredentials, CHROME_UA } from './_seed-utils.mjs';
|
||||
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const VALIDATION_DIR = join(__dirname, '..', 'docs', 'methodology', 'country-resilience-index', 'validation');
|
||||
const REFERENCE_DIR = join(VALIDATION_DIR, 'reference-data');
|
||||
const REDIS_KEY = 'resilience:benchmark:external:v1';
|
||||
const REDIS_TTL = 7 * 24 * 60 * 60;
|
||||
|
||||
const INFORM_CSV_URL = 'https://drmkc.jrc.ec.europa.eu/inform-index/Portals/0/InfoRM/INFORM_Composite_2024.csv';
|
||||
const NDGAIN_CSV_URL = 'https://gain.nd.edu/assets/522870/nd_gain_countryindex_2023data.csv';
|
||||
const WRI_CSV_URL = 'https://weltrisikobericht.de/download/2944/';
|
||||
const FSI_CSV_URL = 'https://fragilestatesindex.org/wp-content/uploads/2024/06/fsi-2024.csv';
|
||||
|
||||
export const HYPOTHESES = [
|
||||
{ index: 'INFORM', pillar: 'overall', direction: 'negative', minSpearman: 0.60 },
|
||||
{ index: 'ND-GAIN', pillar: 'structural-readiness', direction: 'positive', minSpearman: 0.65 },
|
||||
{ index: 'WorldRiskIndex', pillar: 'overall', direction: 'negative', minSpearman: 0.55 },
|
||||
{ index: 'FSI', pillar: 'overall', direction: 'negative', minSpearman: 0.60 },
|
||||
];
|
||||
|
||||
const ISO3_TO_ISO2 = buildIso3ToIso2Map();
|
||||
|
||||
function buildIso3ToIso2Map() {
|
||||
const mapping = {
|
||||
AFG:'AF',ALB:'AL',DZA:'DZ',AND:'AD',AGO:'AO',ATG:'AG',ARG:'AR',ARM:'AM',AUS:'AU',AUT:'AT',
|
||||
AZE:'AZ',BHS:'BS',BHR:'BH',BGD:'BD',BRB:'BB',BLR:'BY',BEL:'BE',BLZ:'BZ',BEN:'BJ',BTN:'BT',
|
||||
BOL:'BO',BIH:'BA',BWA:'BW',BRA:'BR',BRN:'BN',BGR:'BG',BFA:'BF',BDI:'BI',KHM:'KH',CMR:'CM',
|
||||
CAN:'CA',CPV:'CV',CAF:'CF',TCD:'TD',CHL:'CL',CHN:'CN',COL:'CO',COM:'KM',COG:'CG',COD:'CD',
|
||||
CRI:'CR',CIV:'CI',HRV:'HR',CUB:'CU',CYP:'CY',CZE:'CZ',DNK:'DK',DJI:'DJ',DMA:'DM',DOM:'DO',
|
||||
ECU:'EC',EGY:'EG',SLV:'SV',GNQ:'GQ',ERI:'ER',EST:'EE',SWZ:'SZ',ETH:'ET',FJI:'FJ',FIN:'FI',
|
||||
FRA:'FR',GAB:'GA',GMB:'GM',GEO:'GE',DEU:'DE',GHA:'GH',GRC:'GR',GRD:'GD',GTM:'GT',GIN:'GN',
|
||||
GNB:'GW',GUY:'GY',HTI:'HT',HND:'HN',HUN:'HU',ISL:'IS',IND:'IN',IDN:'ID',IRN:'IR',IRQ:'IQ',
|
||||
IRL:'IE',ISR:'IL',ITA:'IT',JAM:'JM',JPN:'JP',JOR:'JO',KAZ:'KZ',KEN:'KE',KIR:'KI',PRK:'KP',
|
||||
KOR:'KR',KWT:'KW',KGZ:'KG',LAO:'LA',LVA:'LV',LBN:'LB',LSO:'LS',LBR:'LR',LBY:'LY',LIE:'LI',
|
||||
LTU:'LT',LUX:'LU',MDG:'MG',MWI:'MW',MYS:'MY',MDV:'MV',MLI:'ML',MLT:'MT',MHL:'MH',MRT:'MR',
|
||||
MUS:'MU',MEX:'MX',FSM:'FM',MDA:'MD',MCO:'MC',MNG:'MN',MNE:'ME',MAR:'MA',MOZ:'MZ',MMR:'MM',
|
||||
NAM:'NA',NRU:'NR',NPL:'NP',NLD:'NL',NZL:'NZ',NIC:'NI',NER:'NE',NGA:'NG',MKD:'MK',NOR:'NO',
|
||||
OMN:'OM',PAK:'PK',PLW:'PW',PAN:'PA',PNG:'PG',PRY:'PY',PER:'PE',PHL:'PH',POL:'PL',PRT:'PT',
|
||||
QAT:'QA',ROU:'RO',RUS:'RU',RWA:'RW',KNA:'KN',LCA:'LC',VCT:'VC',WSM:'WS',STP:'ST',SAU:'SA',
|
||||
SEN:'SN',SRB:'RS',SYC:'SC',SLE:'SL',SGP:'SG',SVK:'SK',SVN:'SI',SLB:'SB',SOM:'SO',ZAF:'ZA',
|
||||
SSD:'SS',ESP:'ES',LKA:'LK',SDN:'SD',SUR:'SR',SWE:'SE',CHE:'CH',SYR:'SY',TWN:'TW',TJK:'TJ',
|
||||
TZA:'TZ',THA:'TH',TLS:'TL',TGO:'TG',TON:'TO',TTO:'TT',TUN:'TN',TUR:'TR',TKM:'TM',TUV:'TV',
|
||||
UGA:'UG',UKR:'UA',ARE:'AE',GBR:'GB',USA:'US',URY:'UY',UZB:'UZ',VUT:'VU',VEN:'VE',VNM:'VN',
|
||||
YEM:'YE',ZMB:'ZM',ZWE:'ZW',PSE:'PS',XKX:'XK',COK:'CK',NIU:'NU',
|
||||
};
|
||||
return mapping;
|
||||
}
|
||||
|
||||
function toIso2(code) {
|
||||
if (!code) return null;
|
||||
const c = code.trim().toUpperCase();
|
||||
if (/^[A-Z]{2}$/.test(c)) return c;
|
||||
if (/^[A-Z]{3}$/.test(c)) return ISO3_TO_ISO2[c] || null;
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseCSV(text) {
|
||||
const lines = text.split('\n').filter(l => l.trim());
|
||||
if (lines.length < 2) return [];
|
||||
const headers = parseCSVLine(lines[0]);
|
||||
return lines.slice(1).map(line => {
|
||||
const values = parseCSVLine(line);
|
||||
const row = {};
|
||||
headers.forEach((h, i) => { row[h.trim()] = (values[i] || '').trim(); });
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
function parseCSVLine(line) {
|
||||
const result = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
for (const ch of line) {
|
||||
if (ch === '"') { inQuotes = !inQuotes; continue; }
|
||||
if (ch === ',' && !inQuotes) { result.push(current); current = ''; continue; }
|
||||
current += ch;
|
||||
}
|
||||
result.push(current);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function fetchCSV(url, label) {
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: { 'User-Agent': CHROME_UA },
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const text = await resp.text();
|
||||
console.log(`[benchmark] Fetched ${label} live (${text.length} bytes)`);
|
||||
return { text, source: 'live' };
|
||||
} catch (err) {
|
||||
console.warn(`[benchmark] Live fetch failed for ${label}: ${err.message}`);
|
||||
const refPath = join(REFERENCE_DIR, `${label.toLowerCase().replace(/[^a-z0-9]/g, '-')}.csv`);
|
||||
if (existsSync(refPath)) {
|
||||
const text = readFileSync(refPath, 'utf8');
|
||||
console.log(`[benchmark] Loaded ${label} from reference CSV (${text.length} bytes)`);
|
||||
return { text, source: 'stub' };
|
||||
}
|
||||
console.warn(`[benchmark] No reference CSV at ${refPath}, skipping ${label}`);
|
||||
return { text: null, source: 'unavailable' };
|
||||
}
|
||||
}
|
||||
|
||||
function findColumn(headers, ...candidates) {
|
||||
const lower = headers.map(h => h.toLowerCase().trim());
|
||||
for (const c of candidates) {
|
||||
const idx = lower.findIndex(h => h.includes(c.toLowerCase()));
|
||||
if (idx >= 0) return headers[idx];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function fetchInformGlobal() {
|
||||
const { text, source } = await fetchCSV(INFORM_CSV_URL, 'INFORM');
|
||||
if (!text) return { scores: new Map(), source };
|
||||
const rows = parseCSV(text);
|
||||
const scores = new Map();
|
||||
let isoCol = findColumn(Object.keys(rows[0] || {}), 'iso3', 'iso', 'country_iso');
|
||||
let scoreCol = findColumn(Object.keys(rows[0] || {}), 'inform_risk', 'inform', 'risk_score', 'composite');
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row);
|
||||
const code = toIso2(row[isoCol || keys[0]]);
|
||||
const val = parseFloat(row[scoreCol || keys[keys.length - 1]]);
|
||||
if (code && !Number.isNaN(val)) scores.set(code, val);
|
||||
}
|
||||
return { scores, source };
|
||||
}
|
||||
|
||||
export async function fetchNdGain() {
|
||||
const { text, source } = await fetchCSV(NDGAIN_CSV_URL, 'ND-GAIN');
|
||||
if (!text) return { scores: new Map(), source };
|
||||
const rows = parseCSV(text);
|
||||
const scores = new Map();
|
||||
const isoCol = findColumn(Object.keys(rows[0] || {}), 'iso3', 'iso', 'country');
|
||||
const scoreCol = findColumn(Object.keys(rows[0] || {}), 'gain', 'nd-gain', 'score', 'readiness', 'index');
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row);
|
||||
const code = toIso2(row[isoCol || keys[0]]);
|
||||
const val = parseFloat(row[scoreCol || keys[keys.length - 1]]);
|
||||
if (code && !Number.isNaN(val)) scores.set(code, val);
|
||||
}
|
||||
return { scores, source };
|
||||
}
|
||||
|
||||
export async function fetchWorldRiskIndex() {
|
||||
const { text, source } = await fetchCSV(WRI_CSV_URL, 'WorldRiskIndex');
|
||||
if (!text) return { scores: new Map(), source };
|
||||
const rows = parseCSV(text);
|
||||
const scores = new Map();
|
||||
const isoCol = findColumn(Object.keys(rows[0] || {}), 'iso3', 'iso', 'country_code');
|
||||
const scoreCol = findColumn(Object.keys(rows[0] || {}), 'worldriskindex', 'wri', 'risk_index', 'score');
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row);
|
||||
const code = toIso2(row[isoCol || keys[0]]);
|
||||
const val = parseFloat(row[scoreCol || keys[keys.length - 1]]);
|
||||
if (code && !Number.isNaN(val)) scores.set(code, val);
|
||||
}
|
||||
return { scores, source };
|
||||
}
|
||||
|
||||
export async function fetchFsi() {
|
||||
const { text, source } = await fetchCSV(FSI_CSV_URL, 'FSI');
|
||||
if (!text) return { scores: new Map(), source };
|
||||
const rows = parseCSV(text);
|
||||
const scores = new Map();
|
||||
const isoCol = findColumn(Object.keys(rows[0] || {}), 'iso', 'country_code', 'code');
|
||||
const scoreCol = findColumn(Object.keys(rows[0] || {}), 'total', 'fsi', 'score', 'fragility');
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row);
|
||||
const code = toIso2(row[isoCol || keys[0]]);
|
||||
const val = parseFloat(row[scoreCol || keys[keys.length - 1]]);
|
||||
if (code && !Number.isNaN(val)) scores.set(code, val);
|
||||
}
|
||||
return { scores, source };
|
||||
}
|
||||
|
||||
export function rankArray(arr) {
|
||||
const sorted = arr.map((v, i) => ({ v, i })).sort((a, b) => a.v - b.v);
|
||||
const ranks = new Array(arr.length);
|
||||
let i = 0;
|
||||
while (i < sorted.length) {
|
||||
let j = i;
|
||||
while (j < sorted.length && sorted[j].v === sorted[i].v) j++;
|
||||
const avgRank = (i + j + 1) / 2;
|
||||
for (let k = i; k < j; k++) ranks[sorted[k].i] = avgRank;
|
||||
i = j;
|
||||
}
|
||||
return ranks;
|
||||
}
|
||||
|
||||
export function spearman(x, y) {
|
||||
if (x.length !== y.length || x.length < 3) return NaN;
|
||||
const rx = rankArray(x);
|
||||
const ry = rankArray(y);
|
||||
return pearson(rx, ry);
|
||||
}
|
||||
|
||||
export function pearson(x, y) {
|
||||
const n = x.length;
|
||||
if (n < 3) return NaN;
|
||||
const mx = x.reduce((s, v) => s + v, 0) / n;
|
||||
const my = y.reduce((s, v) => s + v, 0) / n;
|
||||
let num = 0, dx = 0, dy = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const a = x[i] - mx;
|
||||
const b = y[i] - my;
|
||||
num += a * b;
|
||||
dx += a * a;
|
||||
dy += b * b;
|
||||
}
|
||||
const denom = Math.sqrt(dx * dy);
|
||||
return denom === 0 ? 0 : num / denom;
|
||||
}
|
||||
|
||||
export function detectOutliers(wmScores, extScores, countryCodes) {
|
||||
if (wmScores.length < 5) return [];
|
||||
const rx = rankArray(wmScores);
|
||||
const ry = rankArray(extScores);
|
||||
const n = rx.length;
|
||||
const mRx = rx.reduce((s, v) => s + v, 0) / n;
|
||||
const mRy = ry.reduce((s, v) => s + v, 0) / n;
|
||||
|
||||
let slope_num = 0, slope_den = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
slope_num += (rx[i] - mRx) * (ry[i] - mRy);
|
||||
slope_den += (rx[i] - mRx) ** 2;
|
||||
}
|
||||
const slope = slope_den === 0 ? 0 : slope_num / slope_den;
|
||||
const intercept = mRy - slope * mRx;
|
||||
|
||||
const residuals = rx.map((r, i) => ry[i] - (slope * r + intercept));
|
||||
const meanRes = residuals.reduce((s, v) => s + v, 0) / n;
|
||||
const stdRes = Math.sqrt(residuals.reduce((s, v) => s + (v - meanRes) ** 2, 0) / n);
|
||||
if (stdRes === 0) return [];
|
||||
|
||||
return residuals
|
||||
.map((r, i) => ({ i, z: (r - meanRes) / stdRes }))
|
||||
.filter(({ z }) => Math.abs(z) > 2)
|
||||
.map(({ i, z }) => ({
|
||||
countryCode: countryCodes[i],
|
||||
wmScore: wmScores[i],
|
||||
externalScore: extScores[i],
|
||||
residual: Math.round(z * 100) / 100,
|
||||
}));
|
||||
}
|
||||
|
||||
function generateCommentary(outlier, indexName, wmScores, _extScores) {
|
||||
const { countryCode, residual } = outlier;
|
||||
const wmHigh = outlier.wmScore > median(wmScores);
|
||||
const direction = residual > 0 ? 'higher' : 'lower';
|
||||
|
||||
const templates = {
|
||||
'INFORM': wmHigh
|
||||
? `${countryCode}: WM scores high (fiscal/institutional capacity); INFORM penalizes geographic/hazard exposure`
|
||||
: `${countryCode}: WM scores low (limited structural buffers); INFORM rates risk ${direction} than WM resilience inversion`,
|
||||
'ND-GAIN': wmHigh
|
||||
? `${countryCode}: WM structural readiness aligns with ND-GAIN readiness; external rank ${direction} than expected`
|
||||
: `${countryCode}: WM structural readiness diverges from ND-GAIN; possible data-vintage or indicator-coverage gap`,
|
||||
'WorldRiskIndex': wmHigh
|
||||
? `${countryCode}: WM rates resilience high; WRI emphasizes exposure/vulnerability dimensions differently`
|
||||
: `${countryCode}: WM rates resilience low; WRI susceptibility weighting drives rank ${direction}`,
|
||||
'FSI': wmHigh
|
||||
? `${countryCode}: WM resilience high; FSI fragility captures governance/legitimacy dimensions WM weights differently`
|
||||
: `${countryCode}: WM resilience low; FSI cohesion/economic indicators drive ${direction} fragility rank`,
|
||||
};
|
||||
return templates[indexName] || `${countryCode}: WM diverges from ${indexName} by ${residual} sigma`;
|
||||
}
|
||||
|
||||
function median(arr) {
|
||||
if (arr.length === 0) return 0;
|
||||
const s = [...arr].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
||||
}
|
||||
|
||||
async function readWmScoresFromRedis() {
|
||||
const { url, token } = getRedisCredentials();
|
||||
const rankingResp = await fetch(`${url}/get/${encodeURIComponent('resilience:ranking:v8')}`, {
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!rankingResp.ok) throw new Error(`Failed to read ranking: HTTP ${rankingResp.status}`);
|
||||
const rankingData = await rankingResp.json();
|
||||
if (!rankingData.result) throw new Error('No ranking data in Redis');
|
||||
const ranking = JSON.parse(rankingData.result);
|
||||
const scores = new Map();
|
||||
for (const item of ranking) {
|
||||
if (item.countryCode && typeof item.overallScore === 'number' && item.overallScore > 0) {
|
||||
scores.set(item.countryCode, item.overallScore);
|
||||
}
|
||||
}
|
||||
console.log(`[benchmark] Read ${scores.size} WM resilience scores from Redis`);
|
||||
return scores;
|
||||
}
|
||||
|
||||
function alignScores(wmScores, externalScores) {
|
||||
const commonCodes = [];
|
||||
const wmArr = [];
|
||||
const extArr = [];
|
||||
for (const [code, wm] of wmScores) {
|
||||
const ext = externalScores.get(code);
|
||||
if (ext != null && !Number.isNaN(ext)) {
|
||||
commonCodes.push(code);
|
||||
wmArr.push(wm);
|
||||
extArr.push(ext);
|
||||
}
|
||||
}
|
||||
return { commonCodes, wmArr, extArr };
|
||||
}
|
||||
|
||||
function evaluateHypothesis(hypothesis, sp) {
|
||||
const absSpearman = Math.abs(sp);
|
||||
const directionCorrect = hypothesis.direction === 'negative' ? sp < 0 : sp > 0;
|
||||
return directionCorrect && absSpearman >= hypothesis.minSpearman;
|
||||
}
|
||||
|
||||
export async function runBenchmark(opts = {}) {
|
||||
const wmScores = opts.wmScores || await readWmScoresFromRedis();
|
||||
|
||||
const fetchers = [
|
||||
{ name: 'INFORM', fn: opts.fetchInform || fetchInformGlobal },
|
||||
{ name: 'ND-GAIN', fn: opts.fetchNdGain || fetchNdGain },
|
||||
{ name: 'WorldRiskIndex', fn: opts.fetchWri || fetchWorldRiskIndex },
|
||||
{ name: 'FSI', fn: opts.fetchFsi || fetchFsi },
|
||||
];
|
||||
|
||||
const externalResults = {};
|
||||
const sourceStatus = {};
|
||||
for (const { name, fn } of fetchers) {
|
||||
const result = await fn();
|
||||
externalResults[name] = result.scores;
|
||||
sourceStatus[name] = result.source;
|
||||
}
|
||||
|
||||
const correlations = {};
|
||||
const allOutliers = [];
|
||||
const hypothesisResults = [];
|
||||
|
||||
for (const { name } of fetchers) {
|
||||
const extScores = externalResults[name];
|
||||
if (!extScores || extScores.size === 0) {
|
||||
correlations[name] = { spearman: NaN, pearson: NaN, n: 0 };
|
||||
continue;
|
||||
}
|
||||
|
||||
const { commonCodes, wmArr, extArr } = alignScores(wmScores, extScores);
|
||||
const sp = spearman(wmArr, extArr);
|
||||
const pe = pearson(wmArr, extArr);
|
||||
correlations[name] = {
|
||||
spearman: Math.round(sp * 10000) / 10000,
|
||||
pearson: Math.round(pe * 10000) / 10000,
|
||||
n: commonCodes.length,
|
||||
};
|
||||
|
||||
const outliers = detectOutliers(wmArr, extArr, commonCodes);
|
||||
for (const o of outliers) {
|
||||
const commentary = generateCommentary(o, name, wmArr, extArr);
|
||||
allOutliers.push({ ...o, index: name, commentary });
|
||||
}
|
||||
}
|
||||
|
||||
for (const h of HYPOTHESES) {
|
||||
const corr = correlations[h.index];
|
||||
const sp = corr?.spearman ?? NaN;
|
||||
const pass = !Number.isNaN(sp) && evaluateHypothesis(h, sp);
|
||||
hypothesisResults.push({
|
||||
index: h.index,
|
||||
pillar: h.pillar,
|
||||
direction: h.direction,
|
||||
expected: h.minSpearman,
|
||||
actual: Number.isNaN(sp) ? null : Math.round(sp * 10000) / 10000,
|
||||
pass,
|
||||
});
|
||||
}
|
||||
|
||||
const result = {
|
||||
generatedAt: Date.now(),
|
||||
license: 'FSI data: Fund for Peace, non-commercial academic license. For internal validation only.',
|
||||
hypotheses: hypothesisResults,
|
||||
correlations,
|
||||
outliers: allOutliers,
|
||||
sourceStatus,
|
||||
};
|
||||
|
||||
if (!opts.dryRun) {
|
||||
mkdirSync(VALIDATION_DIR, { recursive: true });
|
||||
writeFileSync(
|
||||
join(VALIDATION_DIR, 'benchmark-results.json'),
|
||||
JSON.stringify(result, null, 2) + '\n',
|
||||
);
|
||||
console.log(`[benchmark] Wrote benchmark-results.json`);
|
||||
|
||||
try {
|
||||
const { url, token } = getRedisCredentials();
|
||||
const payload = JSON.stringify(result);
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(['SET', REDIS_KEY, payload, 'EX', REDIS_TTL]),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!resp.ok) console.warn('[benchmark] Redis write failed:', resp.status);
|
||||
console.log(`[benchmark] Wrote to Redis key ${REDIS_KEY} (TTL ${REDIS_TTL}s)`);
|
||||
} catch (err) {
|
||||
console.warn(`[benchmark] Redis write failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isMain) {
|
||||
runBenchmark()
|
||||
.then(result => {
|
||||
console.log('\n=== Benchmark Results ===');
|
||||
console.log(`Hypotheses: ${result.hypotheses.filter(h => h.pass).length}/${result.hypotheses.length} passed`);
|
||||
for (const h of result.hypotheses) {
|
||||
console.log(` ${h.pass ? 'PASS' : 'FAIL'} ${h.index} (${h.pillar}): expected ${h.direction} >= ${h.expected}, got ${h.actual}`);
|
||||
}
|
||||
console.log(`\nCorrelations:`);
|
||||
for (const [name, c] of Object.entries(result.correlations)) {
|
||||
console.log(` ${name}: spearman=${c.spearman}, pearson=${c.pearson}, n=${c.n}`);
|
||||
}
|
||||
console.log(`\nOutliers: ${result.outliers.length}`);
|
||||
for (const o of result.outliers.slice(0, 10)) {
|
||||
console.log(` ${o.countryCode} (${o.index}): residual=${o.residual} - ${o.commentary}`);
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('[benchmark] Fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
272
tests/benchmark-resilience-external.test.mjs
Normal file
272
tests/benchmark-resilience-external.test.mjs
Normal file
@@ -0,0 +1,272 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
spearman,
|
||||
pearson,
|
||||
rankArray,
|
||||
detectOutliers,
|
||||
HYPOTHESES,
|
||||
runBenchmark,
|
||||
} from '../scripts/benchmark-resilience-external.mjs';
|
||||
|
||||
describe('rankArray', () => {
|
||||
it('assigns sequential ranks for distinct values', () => {
|
||||
assert.deepEqual(rankArray([10, 30, 20]), [1, 3, 2]);
|
||||
});
|
||||
|
||||
it('assigns average ranks for tied values', () => {
|
||||
assert.deepEqual(rankArray([10, 20, 20, 30]), [1, 2.5, 2.5, 4]);
|
||||
});
|
||||
|
||||
it('handles single element', () => {
|
||||
assert.deepEqual(rankArray([5]), [1]);
|
||||
});
|
||||
|
||||
it('handles all tied', () => {
|
||||
assert.deepEqual(rankArray([7, 7, 7]), [2, 2, 2]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pearson', () => {
|
||||
it('returns 1 for perfectly correlated arrays', () => {
|
||||
const r = pearson([1, 2, 3, 4, 5], [2, 4, 6, 8, 10]);
|
||||
assert.ok(Math.abs(r - 1) < 1e-10, `expected ~1, got ${r}`);
|
||||
});
|
||||
|
||||
it('returns -1 for perfectly inversely correlated arrays', () => {
|
||||
const r = pearson([1, 2, 3, 4, 5], [10, 8, 6, 4, 2]);
|
||||
assert.ok(Math.abs(r - (-1)) < 1e-10, `expected ~-1, got ${r}`);
|
||||
});
|
||||
|
||||
it('returns near 0 for uncorrelated arrays', () => {
|
||||
const r = pearson([1, 2, 3, 4, 5, 6], [3, 1, 6, 2, 5, 4]);
|
||||
assert.ok(Math.abs(r) < 0.5, `expected near 0, got ${r}`);
|
||||
});
|
||||
|
||||
it('returns NaN for arrays shorter than 3', () => {
|
||||
assert.ok(Number.isNaN(pearson([1, 2], [3, 4])));
|
||||
});
|
||||
|
||||
it('returns 0 when one array is constant', () => {
|
||||
assert.equal(pearson([1, 1, 1, 1], [1, 2, 3, 4]), 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('spearman', () => {
|
||||
it('returns 1 for monotonically increasing relationship', () => {
|
||||
const r = spearman([1, 2, 3, 4, 5], [10, 20, 30, 40, 50]);
|
||||
assert.ok(Math.abs(r - 1) < 1e-10, `expected ~1, got ${r}`);
|
||||
});
|
||||
|
||||
it('returns -1 for monotonically decreasing relationship', () => {
|
||||
const r = spearman([1, 2, 3, 4, 5], [50, 40, 30, 20, 10]);
|
||||
assert.ok(Math.abs(r - (-1)) < 1e-10, `expected ~-1, got ${r}`);
|
||||
});
|
||||
|
||||
it('handles non-linear monotonic relationship', () => {
|
||||
const r = spearman([1, 2, 3, 4, 5], [1, 4, 9, 16, 25]);
|
||||
assert.ok(Math.abs(r - 1) < 1e-10, `expected ~1 for monotonic, got ${r}`);
|
||||
});
|
||||
|
||||
it('returns NaN for arrays shorter than 3', () => {
|
||||
assert.ok(Number.isNaN(spearman([1], [2])));
|
||||
});
|
||||
});
|
||||
|
||||
describe('detectOutliers', () => {
|
||||
it('returns empty for small arrays', () => {
|
||||
assert.deepEqual(detectOutliers([1, 2, 3], [3, 2, 1], ['A', 'B', 'C']), []);
|
||||
});
|
||||
|
||||
it('detects an outlier in synthetic data', () => {
|
||||
const wm = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
|
||||
const ext = [100, 90, 80, 70, 60, 50, 40, 30, 20, 500];
|
||||
const codes = ['AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ'];
|
||||
const outliers = detectOutliers(wm, ext, codes);
|
||||
assert.ok(outliers.length > 0, 'expected at least one outlier');
|
||||
assert.ok(outliers.some(o => o.countryCode === 'JJ'), 'expected JJ to be an outlier');
|
||||
});
|
||||
|
||||
it('returns empty when relationship is perfectly linear', () => {
|
||||
const wm = [10, 20, 30, 40, 50, 60, 70, 80];
|
||||
const ext = [80, 70, 60, 50, 40, 30, 20, 10];
|
||||
const codes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'];
|
||||
const outliers = detectOutliers(wm, ext, codes);
|
||||
assert.equal(outliers.length, 0, 'perfect linear should have no outliers');
|
||||
});
|
||||
});
|
||||
|
||||
describe('HYPOTHESES', () => {
|
||||
it('has 4 hypothesis entries', () => {
|
||||
assert.equal(HYPOTHESES.length, 4);
|
||||
});
|
||||
|
||||
it('each hypothesis has required fields', () => {
|
||||
for (const h of HYPOTHESES) {
|
||||
assert.ok(h.index, 'missing index');
|
||||
assert.ok(h.pillar, 'missing pillar');
|
||||
assert.ok(['positive', 'negative'].includes(h.direction), `invalid direction: ${h.direction}`);
|
||||
assert.ok(typeof h.minSpearman === 'number', 'minSpearman must be a number');
|
||||
assert.ok(h.minSpearman > 0 && h.minSpearman < 1, `minSpearman out of range: ${h.minSpearman}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('INFORM expects negative correlation', () => {
|
||||
const inform = HYPOTHESES.find(h => h.index === 'INFORM');
|
||||
assert.equal(inform.direction, 'negative');
|
||||
});
|
||||
|
||||
it('ND-GAIN expects positive correlation', () => {
|
||||
const ndgain = HYPOTHESES.find(h => h.index === 'ND-GAIN');
|
||||
assert.equal(ndgain.direction, 'positive');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runBenchmark (mocked)', () => {
|
||||
it('produces correct output shape with mock data', async () => {
|
||||
const wmScores = new Map([
|
||||
['US', 85], ['GB', 78], ['DE', 80], ['FR', 76], ['JP', 82],
|
||||
['IN', 45], ['BR', 50], ['NG', 30], ['SO', 20], ['CH', 88],
|
||||
]);
|
||||
|
||||
const mockInform = async () => ({
|
||||
scores: new Map([
|
||||
['US', 2.1], ['GB', 2.5], ['DE', 2.3], ['FR', 2.8], ['JP', 2.0],
|
||||
['IN', 5.5], ['BR', 4.8], ['NG', 7.2], ['SO', 8.5], ['CH', 1.8],
|
||||
]),
|
||||
source: 'mock',
|
||||
});
|
||||
|
||||
const mockNdGain = async () => ({
|
||||
scores: new Map([
|
||||
['US', 72], ['GB', 70], ['DE', 71], ['FR', 68], ['JP', 73],
|
||||
['IN', 42], ['BR', 45], ['NG', 35], ['SO', 28], ['CH', 75],
|
||||
]),
|
||||
source: 'mock',
|
||||
});
|
||||
|
||||
const mockWri = async () => ({
|
||||
scores: new Map([
|
||||
['US', 3.8], ['GB', 4.2], ['DE', 3.6], ['FR', 4.5], ['JP', 5.1],
|
||||
['IN', 7.1], ['BR', 5.9], ['NG', 9.3], ['SO', 12.1], ['CH', 2.9],
|
||||
]),
|
||||
source: 'mock',
|
||||
});
|
||||
|
||||
const mockFsi = async () => ({
|
||||
scores: new Map([
|
||||
['US', 38], ['GB', 36], ['DE', 30], ['FR', 35], ['JP', 28],
|
||||
['IN', 72], ['BR', 68], ['NG', 98], ['SO', 112], ['CH', 25],
|
||||
]),
|
||||
source: 'mock',
|
||||
});
|
||||
|
||||
const result = await runBenchmark({
|
||||
wmScores,
|
||||
fetchInform: mockInform,
|
||||
fetchNdGain: mockNdGain,
|
||||
fetchWri: mockWri,
|
||||
fetchFsi: mockFsi,
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
assert.ok(result.generatedAt > 0, 'missing generatedAt');
|
||||
assert.ok(result.license, 'missing FSI license note');
|
||||
assert.equal(result.hypotheses.length, 4, 'expected 4 hypotheses');
|
||||
assert.ok(result.correlations.INFORM, 'missing INFORM correlation');
|
||||
assert.ok(result.correlations['ND-GAIN'], 'missing ND-GAIN correlation');
|
||||
assert.ok(result.correlations.WorldRiskIndex, 'missing WorldRiskIndex correlation');
|
||||
assert.ok(result.correlations.FSI, 'missing FSI correlation');
|
||||
assert.ok(Array.isArray(result.outliers), 'outliers must be an array');
|
||||
assert.ok(result.sourceStatus, 'missing sourceStatus');
|
||||
|
||||
for (const [, corr] of Object.entries(result.correlations)) {
|
||||
assert.ok(typeof corr.spearman === 'number', 'spearman must be number');
|
||||
assert.ok(typeof corr.pearson === 'number', 'pearson must be number');
|
||||
assert.ok(typeof corr.n === 'number', 'n must be number');
|
||||
assert.equal(corr.n, 10, `expected 10 countries, got ${corr.n}`);
|
||||
}
|
||||
|
||||
for (const h of result.hypotheses) {
|
||||
assert.ok(typeof h.pass === 'boolean', 'pass must be boolean');
|
||||
assert.ok(h.index, 'hypothesis must have index');
|
||||
assert.ok(h.pillar, 'hypothesis must have pillar');
|
||||
}
|
||||
});
|
||||
|
||||
it('INFORM and FSI show negative correlation with mock data', async () => {
|
||||
const wmScores = new Map([
|
||||
['US', 85], ['GB', 78], ['DE', 80], ['FR', 76], ['JP', 82],
|
||||
['IN', 45], ['BR', 50], ['NG', 30], ['SO', 20], ['CH', 88],
|
||||
]);
|
||||
|
||||
const mockInform = async () => ({
|
||||
scores: new Map([
|
||||
['US', 2.1], ['GB', 2.5], ['DE', 2.3], ['FR', 2.8], ['JP', 2.0],
|
||||
['IN', 5.5], ['BR', 4.8], ['NG', 7.2], ['SO', 8.5], ['CH', 1.8],
|
||||
]),
|
||||
source: 'mock',
|
||||
});
|
||||
const emptyFetcher = async () => ({ scores: new Map(), source: 'mock' });
|
||||
|
||||
const result = await runBenchmark({
|
||||
wmScores,
|
||||
fetchInform: mockInform,
|
||||
fetchNdGain: emptyFetcher,
|
||||
fetchWri: emptyFetcher,
|
||||
fetchFsi: emptyFetcher,
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
const informCorr = result.correlations.INFORM;
|
||||
assert.ok(informCorr.spearman < 0, `INFORM spearman should be negative, got ${informCorr.spearman}`);
|
||||
});
|
||||
|
||||
it('handles empty external indices gracefully', async () => {
|
||||
const wmScores = new Map([['US', 85], ['GB', 78], ['DE', 80]]);
|
||||
const emptyFetcher = async () => ({ scores: new Map(), source: 'unavailable' });
|
||||
|
||||
const result = await runBenchmark({
|
||||
wmScores,
|
||||
fetchInform: emptyFetcher,
|
||||
fetchNdGain: emptyFetcher,
|
||||
fetchWri: emptyFetcher,
|
||||
fetchFsi: emptyFetcher,
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
assert.equal(result.hypotheses.filter(h => h.pass).length, 0, 'no hypotheses should pass with empty data');
|
||||
assert.equal(result.outliers.length, 0, 'no outliers with empty data');
|
||||
});
|
||||
|
||||
it('outlier entries have commentary', async () => {
|
||||
const n = 20;
|
||||
const wmScores = new Map();
|
||||
const informScores = new Map();
|
||||
const codes = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const code = String.fromCharCode(65 + Math.floor(i / 26)) + String.fromCharCode(65 + (i % 26));
|
||||
codes.push(code);
|
||||
wmScores.set(code, 10 + i * 4);
|
||||
informScores.set(code, 9 - i * 0.4);
|
||||
}
|
||||
wmScores.set(codes[n - 1], 10);
|
||||
informScores.set(codes[n - 1], 0.5);
|
||||
|
||||
const result = await runBenchmark({
|
||||
wmScores,
|
||||
fetchInform: async () => ({ scores: informScores, source: 'mock' }),
|
||||
fetchNdGain: async () => ({ scores: new Map(), source: 'mock' }),
|
||||
fetchWri: async () => ({ scores: new Map(), source: 'mock' }),
|
||||
fetchFsi: async () => ({ scores: new Map(), source: 'mock' }),
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
for (const o of result.outliers) {
|
||||
assert.ok(o.commentary, `outlier ${o.countryCode} missing commentary`);
|
||||
assert.ok(typeof o.residual === 'number', 'residual must be number');
|
||||
assert.ok(o.index, 'outlier must have index name');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user