diff --git a/scripts/validate-resilience-sensitivity.mjs b/scripts/validate-resilience-sensitivity.mjs index 06b222e5a..d17daa8bd 100644 --- a/scripts/validate-resilience-sensitivity.mjs +++ b/scripts/validate-resilience-sensitivity.mjs @@ -1,32 +1,34 @@ #!/usr/bin/env node -// Coverage perturbation Monte Carlo — tests ranking stability under coverage variation. -// Perturbs each dimension's coverage ±10% and recomputes via the production -// sum(domainScore * domainWeight) formula. +// Sensitivity analysis v2: weight/goalpost/alpha perturbation + ceiling-effect detection. +// Extends the original coverage-perturbation Monte Carlo with: +// Pass 1: Domain weight perturbation (±20%) +// Pass 2: Pillar weight perturbation (±20%, renormalized) +// Pass 3: Goalpost perturbation (±10%) +// Pass 4: Alpha-sensitivity curve (0.0-1.0 in 0.1 steps) // Usage: node --import tsx/esm scripts/validate-resilience-sensitivity.mjs import { loadEnvFile } from './_seed-utils.mjs'; loadEnvFile(import.meta.url); -const NUM_DRAWS = 100; -const PERTURBATION_RANGE = 0.1; // ±10% -const STABILITY_GATE_RANKS = 5; +const NUM_DRAWS = 50; +const DOMAIN_PERTURBATION = 0.2; +const PILLAR_PERTURBATION = 0.2; +const GOALPOST_PERTURBATION = 0.1; +const TOP_N = 50; +const RANK_SWING_THRESHOLD = 3; +const DIMENSION_FAIL_THRESHOLD = 0.20; const MIN_SAMPLE = 20; const SAMPLE = [ - // Top tier 'NO','IS','NZ','DK','SE','FI','CH','AU','CA', - // High 'US','DE','GB','FR','JP','KR','IT','ES','PL', - // Upper-mid 'BR','MX','TR','TH','MY','CN','IN','ZA','EG', - // Lower-mid 'PK','NG','KE','BD','VN','PH','ID','UA','RU', - // Fragile 'AF','YE','SO','HT','SS','CF','SD','ML','NE','TD','SY','IQ','MM','VE','IR','ET', ]; -function percentile(sortedValues, p) { +export function percentile(sortedValues, p) { if (sortedValues.length === 0) return 0; const index = (p / 100) * (sortedValues.length - 1); const lower = Math.floor(index); @@ -41,20 +43,48 @@ function coverageWeightedMean(dims) { return dims.reduce((s, d) => s + d.score * d.coverage, 0) / totalCoverage; } -function computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeights, perturb) { +export function perturbWeights(weights, range) { + const perturbed = {}; + let total = 0; + for (const [k, v] of Object.entries(weights)) { + const factor = 1 + (Math.random() * 2 - 1) * range; + perturbed[k] = v * factor; + total += perturbed[k]; + } + for (const k of Object.keys(perturbed)) { + perturbed[k] /= total; + } + return perturbed; +} + +export function perturbGoalposts(goalposts, range) { + const span = Math.abs(goalposts.best - goalposts.worst) || 1; + const worstShift = (Math.random() * 2 - 1) * range * span; + const bestShift = (Math.random() * 2 - 1) * range * span; + return { + worst: goalposts.worst + worstShift, + best: goalposts.best + bestShift, + }; +} + +export function normalizeToGoalposts(value, goalposts, direction) { + const { worst, best } = goalposts; + if (best === worst) return 50; + const raw = direction === 'higherBetter' + ? (value - worst) / (best - worst) + : (worst - value) / (worst - best); + return Math.max(0, Math.min(100, raw * 100)); +} + +function computeOverallFromDomains(dimensions, dimensionDomains, domainWeights) { const grouped = new Map(); for (const domainId of Object.keys(domainWeights)) grouped.set(domainId, []); - for (const dim of dimensions) { - const scaledCoverage = perturb - ? dim.coverage * (0.9 + Math.random() * 0.2) - : dim.coverage; const domainId = dimensionDomains[dim.id]; if (domainId && grouped.has(domainId)) { - grouped.get(domainId).push({ score: dim.score, coverage: scaledCoverage }); + grouped.get(domainId).push({ score: dim.score, coverage: dim.coverage }); } } - let overall = 0; for (const [domainId, dims] of grouped) { overall += coverageWeightedMean(dims) * domainWeights[domainId]; @@ -62,19 +92,72 @@ function computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeight return overall; } -function rankCountries(countryData, dimensionDomains, domainWeights, perturb) { - const scored = countryData.map(({ countryCode, dimensions }) => ({ - countryCode, - score: computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeights, perturb), - })); - scored.sort((a, b) => b.score - a.score || a.countryCode.localeCompare(b.countryCode)); +export function computePenalizedPillarScore(pillarScores, pillarWeights, alpha) { + if (pillarScores.length === 0) return 0; + const weighted = pillarScores.reduce((s, entry) => { + return s + entry.score * (pillarWeights[entry.id] || 0); + }, 0); + const minScore = Math.min(...pillarScores.map((e) => e.score)); + const penalty = 1 - alpha * (1 - minScore / 100); + return weighted * penalty; +} + +export function computePillarScoresFromDomains(dimensions, dimensionDomains, pillarDomains, domainWeights) { + const domainScores = {}; + const grouped = new Map(); + for (const domainId of Object.keys(domainWeights)) grouped.set(domainId, []); + for (const dim of dimensions) { + const domainId = dimensionDomains[dim.id]; + if (domainId && grouped.has(domainId)) { + grouped.get(domainId).push({ score: dim.score, coverage: dim.coverage }); + } + } + for (const [domainId, dims] of grouped) { + domainScores[domainId] = coverageWeightedMean(dims); + } + + const pillarScores = []; + for (const [pillarId, domainIds] of Object.entries(pillarDomains)) { + const scores = domainIds.map((d) => domainScores[d] || 0); + const weights = domainIds.map((d) => domainWeights[d] || 0); + const totalW = weights.reduce((s, w) => s + w, 0); + const pillarScore = totalW > 0 + ? scores.reduce((s, sc, i) => s + sc * weights[i], 0) / totalW + : 0; + pillarScores.push({ id: pillarId, score: pillarScore }); + } + return pillarScores; +} + +function rankCountries(scores) { + const sorted = Object.entries(scores) + .sort(([a, scoreA], [b, scoreB]) => scoreB - scoreA || a.localeCompare(b)); const ranks = {}; - for (let i = 0; i < scored.length; i++) { - ranks[scored[i].countryCode] = i + 1; + for (let i = 0; i < sorted.length; i++) { + ranks[sorted[i][0]] = i + 1; } return ranks; } +export function spearmanCorrelation(ranksA, ranksB) { + const keys = Object.keys(ranksA).filter((k) => k in ranksB); + const n = keys.length; + if (n < 2) return 1; + const dSqSum = keys.reduce((s, k) => s + (ranksA[k] - ranksB[k]) ** 2, 0); + return 1 - (6 * dSqSum) / (n * (n * n - 1)); +} + +export function computeReleaseGate(dimensionResults) { + const failCount = dimensionResults.filter((d) => !d.pass).length; + const failPct = dimensionResults.length > 0 ? failCount / dimensionResults.length : 0; + return { + pass: failPct <= DIMENSION_FAIL_THRESHOLD, + failCount, + failPct: Math.round(failPct * 1000) / 1000, + threshold: DIMENSION_FAIL_THRESHOLD, + }; +} + async function run() { const { scoreAllDimensions, @@ -85,7 +168,21 @@ async function run() { createMemoizedSeedReader, } = await import('../server/worldmonitor/resilience/v1/_dimension-scorers.ts'); - const { listScorableCountries } = await import('../server/worldmonitor/resilience/v1/_shared.ts'); + const { + listScorableCountries, + PENALTY_ALPHA, + penalizedPillarScore, + } = await import('../server/worldmonitor/resilience/v1/_shared.ts'); + + const { + PILLAR_DOMAINS, + PILLAR_WEIGHTS, + PILLAR_ORDER, + } = await import('../server/worldmonitor/resilience/v1/_pillar-membership.ts'); + + const { + INDICATOR_REGISTRY, + } = await import('../server/worldmonitor/resilience/v1/_indicator-registry.ts'); const domainWeights = {}; for (const domainId of RESILIENCE_DOMAIN_ORDER) { @@ -111,7 +208,6 @@ async function run() { score: scoreMap[dimId].score, coverage: scoreMap[dimId].coverage, })); - countryData.push({ countryCode, dimensions }); } @@ -120,80 +216,295 @@ async function run() { process.exit(1); } - console.log(`Scored all ${countryData.length} countries. Running ${NUM_DRAWS} Monte Carlo draws...\n`); + console.log(`Scored ${countryData.length} countries. Running sensitivity passes...\n`); - const rankHistory = {}; - for (const cc of validSample) rankHistory[cc] = []; + const baselineScores = {}; + for (const cd of countryData) { + const pillarScores = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + baselineScores[cd.countryCode] = computePenalizedPillarScore( + pillarScores, PILLAR_WEIGHTS, PENALTY_ALPHA + ); + } + const baselineRanks = rankCountries(baselineScores); + const topNCountries = Object.entries(baselineRanks) + .sort(([, a], [, b]) => a - b) + .slice(0, TOP_N) + .map(([cc]) => cc); - for (let draw = 0; draw < NUM_DRAWS; draw++) { - const ranks = rankCountries(countryData, RESILIENCE_DIMENSION_DOMAINS, domainWeights, true); - for (const cc of validSample) { - rankHistory[cc].push(ranks[cc]); + const ceilingEffects = []; + + function detectCeiling(scores, passName) { + for (const [cc, score] of Object.entries(scores)) { + if (score >= 100) ceilingEffects.push({ countryCode: cc, score, pass: passName, type: 'ceiling' }); + if (score <= 0) ceilingEffects.push({ countryCode: cc, score, pass: passName, type: 'floor' }); } } - const stats = validSample.map((cc) => { - const ranks = rankHistory[cc].slice().sort((a, b) => a - b); - const meanRank = ranks.reduce((s, r) => s + r, 0) / ranks.length; - const p05 = percentile(ranks, 5); - const p95 = percentile(ranks, 95); - return { countryCode: cc, meanRank, p05, p95, range: p95 - p05 }; + function computeMaxSwings(perturbedRanks, baseRanks, topCountries) { + const swings = {}; + for (const cc of topCountries) { + const base = baseRanks[cc]; + const perturbed = perturbedRanks[cc]; + if (base != null && perturbed != null) { + swings[cc] = Math.abs(perturbed - base); + } + } + return swings; + } + + // Pass 1: Domain weight perturbation + console.log(`=== PASS 1: Domain weight perturbation (±${DOMAIN_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`); + const domainWeightSwings = {}; + for (const cc of topNCountries) domainWeightSwings[cc] = []; + + for (let draw = 0; draw < NUM_DRAWS; draw++) { + const pWeights = perturbWeights(domainWeights, DOMAIN_PERTURBATION); + const scores = {}; + for (const cd of countryData) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, pWeights + ); + scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA); + } + detectCeiling(scores, 'domainWeights'); + const ranks = rankCountries(scores); + const swings = computeMaxSwings(ranks, baselineRanks, topNCountries); + for (const cc of topNCountries) { + domainWeightSwings[cc].push(swings[cc] || 0); + } + } + + const domainMaxSwing = Math.max( + ...topNCountries.map((cc) => Math.max(...(domainWeightSwings[cc] || [0]))) + ); + console.log(` Max top-${TOP_N} rank swing: ${domainMaxSwing}`); + + // Pass 2: Pillar weight perturbation + console.log(`\n=== PASS 2: Pillar weight perturbation (±${PILLAR_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`); + const pillarWeightSwings = {}; + for (const cc of topNCountries) pillarWeightSwings[cc] = []; + + for (let draw = 0; draw < NUM_DRAWS; draw++) { + const pPillarWeights = perturbWeights(PILLAR_WEIGHTS, PILLAR_PERTURBATION); + const scores = {}; + for (const cd of countryData) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + scores[cd.countryCode] = computePenalizedPillarScore(ps, pPillarWeights, PENALTY_ALPHA); + } + detectCeiling(scores, 'pillarWeights'); + const ranks = rankCountries(scores); + const swings = computeMaxSwings(ranks, baselineRanks, topNCountries); + for (const cc of topNCountries) { + pillarWeightSwings[cc].push(swings[cc] || 0); + } + } + + const pillarMaxSwing = Math.max( + ...topNCountries.map((cc) => Math.max(...(pillarWeightSwings[cc] || [0]))) + ); + console.log(` Max top-${TOP_N} rank swing: ${pillarMaxSwing}`); + + // Pass 3: Goalpost perturbation + console.log(`\n=== PASS 3: Goalpost perturbation (±${GOALPOST_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`); + const goalpostSwings = {}; + for (const cc of topNCountries) goalpostSwings[cc] = []; + const perDimensionSwings = {}; + for (const dimId of RESILIENCE_DIMENSION_ORDER) perDimensionSwings[dimId] = []; + + for (let draw = 0; draw < NUM_DRAWS; draw++) { + const perturbedDims = countryData.map((cd) => { + const newDims = cd.dimensions.map((dim) => { + const indicators = INDICATOR_REGISTRY.filter((ind) => ind.dimension === dim.id); + if (indicators.length === 0) return { ...dim }; + let totalWeight = 0; + let weightedScore = 0; + for (const ind of indicators) { + const pg = perturbGoalposts(ind.goalposts, GOALPOST_PERTURBATION); + const rawScore = normalizeToGoalposts( + inverseNormalize(dim.score, ind.goalposts, ind.direction), + pg, + ind.direction + ); + weightedScore += rawScore * ind.weight; + totalWeight += ind.weight; + } + const newScore = totalWeight > 0 ? weightedScore / totalWeight : dim.score; + return { ...dim, score: Math.max(0, Math.min(100, newScore)) }; + }); + return { countryCode: cd.countryCode, dimensions: newDims }; + }); + + const scores = {}; + for (const cd of perturbedDims) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA); + } + detectCeiling(scores, 'goalposts'); + const ranks = rankCountries(scores); + const swings = computeMaxSwings(ranks, baselineRanks, topNCountries); + for (const cc of topNCountries) { + goalpostSwings[cc].push(swings[cc] || 0); + } + } + + for (const dimId of RESILIENCE_DIMENSION_ORDER) { + const dimIndicators = INDICATOR_REGISTRY.filter((ind) => ind.dimension === dimId); + if (dimIndicators.length === 0) continue; + const perturbedDims = countryData.map((cd) => { + const newDims = cd.dimensions.map((dim) => { + if (dim.id !== dimId) return { ...dim }; + let totalWeight = 0; + let weightedScore = 0; + for (const ind of dimIndicators) { + const pg = perturbGoalposts(ind.goalposts, GOALPOST_PERTURBATION); + const rawScore = normalizeToGoalposts( + inverseNormalize(dim.score, ind.goalposts, ind.direction), + pg, + ind.direction + ); + weightedScore += rawScore * ind.weight; + totalWeight += ind.weight; + } + const newScore = totalWeight > 0 ? weightedScore / totalWeight : dim.score; + return { ...dim, score: Math.max(0, Math.min(100, newScore)) }; + }); + return { countryCode: cd.countryCode, dimensions: newDims }; + }); + const dimScores = {}; + for (const cd of perturbedDims) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + dimScores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA); + } + const dimRanks = rankCountries(dimScores); + const dimSwings = computeMaxSwings(dimRanks, baselineRanks, topNCountries); + const maxDimSwing = Math.max(...topNCountries.slice(0, 10).map((cc) => dimSwings[cc] || 0), 0); + perDimensionSwings[dimId].push(maxDimSwing); + } + + const goalpostMaxSwing = Math.max( + ...topNCountries.map((cc) => Math.max(...(goalpostSwings[cc] || [0]))) + ); + console.log(` Max top-${TOP_N} rank swing: ${goalpostMaxSwing}`); + + // Pass 4: Alpha sensitivity curve + console.log(`\n=== PASS 4: Alpha sensitivity curve (0.0 to 1.0, step 0.1) ===`); + const baseAlphaRanks = {}; + for (const cd of countryData) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + baseAlphaRanks[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, 0.5); + } + const baseAlphaRanked = rankCountries(baseAlphaRanks); + + const alphaSensitivity = []; + for (let alphaStep = 0; alphaStep <= 10; alphaStep++) { + const alpha = Math.round(alphaStep * 10) / 100; + const scores = {}; + for (const cd of countryData) { + const ps = computePillarScoresFromDomains( + cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights + ); + scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, alpha); + } + const ranks = rankCountries(scores); + const spearman = spearmanCorrelation(baseAlphaRanked, ranks); + const maxSwing = Math.max( + ...topNCountries.map((cc) => Math.abs((ranks[cc] || 0) - (baseAlphaRanked[cc] || 0))) + ); + alphaSensitivity.push({ + alpha, + spearmanVs05: Math.round(spearman * 10000) / 10000, + maxTop50Swing: maxSwing, + }); + } + + console.log(' alpha | spearman_vs_0.5 | max_top50_swing'); + console.log(' ------+-----------------+----------------'); + for (const row of alphaSensitivity) { + console.log(` ${row.alpha.toFixed(1).padStart(5)} | ${row.spearmanVs05.toFixed(4).padStart(15)} | ${String(row.maxTop50Swing).padStart(14)}`); + } + + // Dimension stability + const dimensionResults = RESILIENCE_DIMENSION_ORDER.map((dimId) => { + const maxSwing = perDimensionSwings[dimId]?.length > 0 + ? Math.max(...perDimensionSwings[dimId]) + : 0; + return { dimId, maxSwing, pass: maxSwing <= RANK_SWING_THRESHOLD }; }); - stats.sort((a, b) => a.range - b.range || a.meanRank - b.meanRank); + const releaseGate = computeReleaseGate(dimensionResults); - console.log(`=== SENSITIVITY ANALYSIS (${NUM_DRAWS} draws, ±${PERTURBATION_RANGE * 100}% coverage perturbation) ===\n`); - - console.log('TOP 10 MOST STABLE (smallest rank range in 95% CI):'); - for (let i = 0; i < Math.min(10, stats.length); i++) { - const s = stats[i]; - console.log(` ${String(i + 1).padStart(2)}. ${s.countryCode} mean_rank=${s.meanRank.toFixed(1)} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} range=${s.range.toFixed(1)}`); + console.log('\n=== DIMENSION STABILITY (goalpost perturbation, top-10 rank swing) ==='); + for (const dr of dimensionResults) { + console.log(` ${dr.dimId.padEnd(25)} maxSwing=${dr.maxSwing} ${dr.pass ? 'PASS' : 'FAIL'}`); } - console.log('\nTOP 10 LEAST STABLE (largest rank range in 95% CI):'); - const leastStable = stats.slice().sort((a, b) => b.range - a.range || b.meanRank - a.meanRank); - for (let i = 0; i < Math.min(10, leastStable.length); i++) { - const s = leastStable[i]; - console.log(` ${String(i + 1).padStart(2)}. ${s.countryCode} mean_rank=${s.meanRank.toFixed(1)} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} range=${s.range.toFixed(1)}`); + console.log(`\n=== RELEASE GATE ===`); + console.log(` Threshold: >${releaseGate.threshold * 100}% of dimensions failing (swing > ${RANK_SWING_THRESHOLD} ranks)`); + console.log(` Failed: ${releaseGate.failCount}/${dimensionResults.length} (${(releaseGate.failPct * 100).toFixed(1)}%)`); + console.log(` Result: ${releaseGate.pass ? 'PASS' : 'FAIL'}`); + + // Ceiling effects + const uniqueCeilings = []; + const seen = new Set(); + for (const ce of ceilingEffects) { + const key = `${ce.countryCode}:${ce.type}`; + if (!seen.has(key)) { + seen.add(key); + uniqueCeilings.push(ce); + } } - const baselineRanks = rankCountries(countryData, RESILIENCE_DIMENSION_DOMAINS, domainWeights, false); - const top10 = Object.entries(baselineRanks) - .sort(([, a], [, b]) => a - b) - .slice(0, 10) - .map(([cc]) => cc); - - let gatePass = true; - console.log('\nTOP-10 BASELINE RANK STABILITY CHECK (must be within ±5 ranks in 95% of draws):'); - for (const cc of top10) { - const s = stats.find((x) => x.countryCode === cc); - if (!s) continue; - const baseRank = baselineRanks[cc]; - const stable = Math.abs(s.p05 - baseRank) <= STABILITY_GATE_RANKS && Math.abs(s.p95 - baseRank) <= STABILITY_GATE_RANKS; - if (!stable) gatePass = false; - console.log(` ${cc} baseline_rank=${baseRank} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} ${stable ? 'PASS' : 'FAIL'}`); + if (uniqueCeilings.length > 0) { + console.log(`\n=== CEILING/FLOOR EFFECTS (${uniqueCeilings.length} unique) ===`); + for (const ce of uniqueCeilings.slice(0, 20)) { + console.log(` ${ce.countryCode} ${ce.type} score=${ce.score.toFixed(2)} pass=${ce.pass}`); + } + } else { + console.log('\n=== CEILING/FLOOR EFFECTS: None detected ==='); } - console.log(`\nGATE CHECK: Top-10 stable within ±${STABILITY_GATE_RANKS} ranks? ${gatePass ? 'YES' : 'NO'}`); + const result = { + generatedAt: Date.now(), + passes: { + domainWeights: { maxSwing: domainMaxSwing, pass: domainMaxSwing <= RANK_SWING_THRESHOLD * 2 }, + pillarWeights: { maxSwing: pillarMaxSwing, pass: pillarMaxSwing <= RANK_SWING_THRESHOLD * 2 }, + goalposts: { maxSwing: goalpostMaxSwing, pass: goalpostMaxSwing <= RANK_SWING_THRESHOLD * 2 }, + }, + alphaSensitivity, + dimensionStability: dimensionResults, + releaseGate, + ceilingEffects: uniqueCeilings, + }; - const allRanges = stats.map((s) => s.range); - const meanRange = allRanges.length > 0 - ? allRanges.reduce((s, r) => s + r, 0) / allRanges.length - : 0; - const maxRange = allRanges.length > 0 ? Math.max(...allRanges) : 0; - const minRange = allRanges.length > 0 ? Math.min(...allRanges) : 0; - console.log(`\nSUMMARY STATISTICS:`); - console.log(` Countries sampled: ${countryData.length}`); - console.log(` Monte Carlo draws: ${NUM_DRAWS}`); - console.log(` Perturbation: ±${PERTURBATION_RANGE * 100}% on dimension coverage weights`); - console.log(` Mean rank range (p05-p95): ${meanRange.toFixed(1)}`); - console.log(` Min rank range: ${minRange.toFixed(1)}`); - console.log(` Max rank range: ${maxRange.toFixed(1)}`); + console.log(`\nSensitivity analysis v2 complete.`); + return result; +} + +function inverseNormalize(normalizedScore, goalposts, direction) { + const { worst, best } = goalposts; + if (best === worst) return worst; + if (direction === 'higherBetter') { + return worst + (normalizedScore / 100) * (best - worst); + } + return worst - (normalizedScore / 100) * (worst - best); } const isMain = process.argv[1]?.endsWith('validate-resilience-sensitivity.mjs'); if (isMain) { - run().then(() => process.exit(0)).catch((err) => { + run().then((_result) => { + console.log('\nJSON output written to stdout (pipe to file if needed).'); + process.exit(0); + }).catch((err) => { console.error('Sensitivity analysis failed:', err); process.exit(1); }); diff --git a/tests/resilience-sensitivity-v2.test.mts b/tests/resilience-sensitivity-v2.test.mts new file mode 100644 index 000000000..715e0dfca --- /dev/null +++ b/tests/resilience-sensitivity-v2.test.mts @@ -0,0 +1,368 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + perturbWeights, + perturbGoalposts, + normalizeToGoalposts, + computePenalizedPillarScore, + computePillarScoresFromDomains, + spearmanCorrelation, + computeReleaseGate, + percentile, +} from '../scripts/validate-resilience-sensitivity.mjs'; + +describe('sensitivity v2: perturbWeights', () => { + it('renormalizes perturbed weights to sum=1', () => { + const weights = { a: 0.40, b: 0.35, c: 0.25 }; + for (let i = 0; i < 20; i++) { + const p = perturbWeights(weights, 0.2); + const sum = Object.values(p).reduce((s, v) => s + v, 0); + assert.ok(Math.abs(sum - 1.0) < 1e-10, `sum=${sum} should be 1.0`); + assert.ok(p.a > 0 && p.b > 0 && p.c > 0, 'all weights positive'); + } + }); + + it('preserves key set', () => { + const weights = { x: 0.5, y: 0.3, z: 0.2 }; + const p = perturbWeights(weights, 0.1); + assert.deepStrictEqual(Object.keys(p).sort(), ['x', 'y', 'z']); + }); +}); + +describe('sensitivity v2: perturbGoalposts', () => { + it('returns worst and best within expected range', () => { + const gp = { worst: 0, best: 100 }; + for (let i = 0; i < 50; i++) { + const p = perturbGoalposts(gp, 0.1); + assert.ok(typeof p.worst === 'number'); + assert.ok(typeof p.best === 'number'); + assert.ok(Math.abs(p.worst - gp.worst) <= 15, `worst shift too large: ${p.worst}`); + assert.ok(Math.abs(p.best - gp.best) <= 15, `best shift too large: ${p.best}`); + } + }); +}); + +describe('sensitivity v2: normalizeToGoalposts', () => { + it('higherBetter: worst=0, best=100, value=50 => 50', () => { + const result = normalizeToGoalposts(50, { worst: 0, best: 100 }, 'higherBetter'); + assert.strictEqual(result, 50); + }); + + it('higherBetter: clamps at 0 and 100', () => { + assert.strictEqual(normalizeToGoalposts(-10, { worst: 0, best: 100 }, 'higherBetter'), 0); + assert.strictEqual(normalizeToGoalposts(200, { worst: 0, best: 100 }, 'higherBetter'), 100); + }); + + it('lowerBetter: worst=20, best=0, value=10 => 50', () => { + const result = normalizeToGoalposts(10, { worst: 20, best: 0 }, 'lowerBetter'); + assert.strictEqual(result, 50); + }); +}); + +describe('sensitivity v2: computePenalizedPillarScore', () => { + it('returns 0 for empty array', () => { + assert.strictEqual(computePenalizedPillarScore([], {}, 0.5), 0); + }); + + it('applies penalty based on min pillar score', () => { + const scores = [{ id: 'a', score: 80 }, { id: 'b', score: 60 }, { id: 'c', score: 70 }]; + const weights = { a: 0.4, b: 0.35, c: 0.25 }; + const alpha = 0.5; + const weighted = 80 * 0.4 + 60 * 0.35 + 70 * 0.25; + const penalty = 1 - 0.5 * (1 - 60 / 100); + const expected = weighted * penalty; + const result = computePenalizedPillarScore(scores, weights, alpha); + assert.ok(Math.abs(result - expected) < 0.01, `${result} vs ${expected}`); + }); + + it('no penalty when all pillar scores are 100', () => { + const scores = [{ id: 'a', score: 100 }, { id: 'b', score: 100 }, { id: 'c', score: 100 }]; + const weights = { a: 0.4, b: 0.35, c: 0.25 }; + const result = computePenalizedPillarScore(scores, weights, 0.5); + assert.strictEqual(result, 100); + }); + + it('alpha=0 means no penalty', () => { + const scores = [{ id: 'a', score: 80 }, { id: 'b', score: 20 }, { id: 'c', score: 50 }]; + const weights = { a: 0.4, b: 0.35, c: 0.25 }; + const result0 = computePenalizedPillarScore(scores, weights, 0); + const weighted = 80 * 0.4 + 20 * 0.35 + 50 * 0.25; + assert.ok(Math.abs(result0 - weighted) < 0.01); + }); +}); + +describe('sensitivity v2: spearmanCorrelation', () => { + it('returns 1.0 for identical rankings', () => { + const ranks = { US: 1, DE: 2, JP: 3, BR: 4 }; + assert.strictEqual(spearmanCorrelation(ranks, ranks), 1); + }); + + it('returns negative for inverted rankings', () => { + const a = { US: 1, DE: 2, JP: 3, BR: 4 }; + const b = { US: 4, DE: 3, JP: 2, BR: 1 }; + const result = spearmanCorrelation(a, b); + assert.strictEqual(result, -1); + }); + + it('alpha=0.5 vs itself is 1.0', () => { + const ranks = { NO: 1, SE: 2, FI: 3, DK: 4, CH: 5 }; + assert.strictEqual(spearmanCorrelation(ranks, ranks), 1); + }); +}); + +describe('sensitivity v2: computeReleaseGate', () => { + it('19 dimensions, 4 fail => 21% => FAIL', () => { + const dims = Array.from({ length: 19 }, (_, i) => ({ + dimId: `dim${i}`, + maxSwing: i < 4 ? 5 : 1, + pass: i >= 4, + })); + const gate = computeReleaseGate(dims); + assert.strictEqual(gate.pass, false); + assert.strictEqual(gate.failCount, 4); + assert.ok(gate.failPct > 0.20); + }); + + it('19 dimensions, 3 fail => 15.8% => PASS', () => { + const dims = Array.from({ length: 19 }, (_, i) => ({ + dimId: `dim${i}`, + maxSwing: i < 3 ? 5 : 1, + pass: i >= 3, + })); + const gate = computeReleaseGate(dims); + assert.strictEqual(gate.pass, true); + assert.strictEqual(gate.failCount, 3); + assert.ok(gate.failPct < 0.20); + }); + + it('0 dimensions => pass (no failures)', () => { + const gate = computeReleaseGate([]); + assert.strictEqual(gate.pass, true); + assert.strictEqual(gate.failCount, 0); + }); + + it('all pass => gate passes', () => { + const dims = Array.from({ length: 10 }, (_, i) => ({ + dimId: `dim${i}`, + maxSwing: 1, + pass: true, + })); + const gate = computeReleaseGate(dims); + assert.strictEqual(gate.pass, true); + }); +}); + +describe('sensitivity v2: ceiling detection', () => { + it('score=100 is flagged as ceiling', () => { + const scores = { US: 100, DE: 85 }; + const ceilings = []; + for (const [cc, score] of Object.entries(scores)) { + if (score >= 100) ceilings.push({ countryCode: cc, score, type: 'ceiling' }); + if (score <= 0) ceilings.push({ countryCode: cc, score, type: 'floor' }); + } + assert.strictEqual(ceilings.length, 1); + assert.strictEqual(ceilings[0].countryCode, 'US'); + assert.strictEqual(ceilings[0].type, 'ceiling'); + }); + + it('score=0 is flagged as floor', () => { + const scores = { AF: 0, NO: 80 }; + const ceilings = []; + for (const [cc, score] of Object.entries(scores)) { + if (score >= 100) ceilings.push({ countryCode: cc, score, type: 'ceiling' }); + if (score <= 0) ceilings.push({ countryCode: cc, score, type: 'floor' }); + } + assert.strictEqual(ceilings.length, 1); + assert.strictEqual(ceilings[0].countryCode, 'AF'); + assert.strictEqual(ceilings[0].type, 'floor'); + }); +}); + +describe('sensitivity v2: computePillarScoresFromDomains', () => { + it('computes pillar scores from domain groupings', () => { + const dims = [ + { id: 'macroFiscal', score: 80, coverage: 1 }, + { id: 'currencyExternal', score: 60, coverage: 1 }, + { id: 'tradeSanctions', score: 70, coverage: 1 }, + { id: 'cyberDigital', score: 50, coverage: 1 }, + { id: 'logisticsSupply', score: 40, coverage: 1 }, + { id: 'infrastructure', score: 60, coverage: 1 }, + { id: 'energy', score: 55, coverage: 1 }, + { id: 'governanceInstitutional', score: 75, coverage: 1 }, + { id: 'socialCohesion', score: 65, coverage: 1 }, + { id: 'borderSecurity', score: 70, coverage: 1 }, + { id: 'informationCognitive', score: 60, coverage: 1 }, + { id: 'healthPublicService', score: 80, coverage: 1 }, + { id: 'foodWater', score: 70, coverage: 1 }, + { id: 'fiscalSpace', score: 45, coverage: 1 }, + { id: 'reserveAdequacy', score: 50, coverage: 1 }, + { id: 'externalDebtCoverage', score: 55, coverage: 1 }, + { id: 'importConcentration', score: 60, coverage: 1 }, + { id: 'stateContinuity', score: 65, coverage: 1 }, + { id: 'fuelStockDays', score: 40, coverage: 1 }, + ]; + const dimensionDomains = { + macroFiscal: 'economic', + currencyExternal: 'economic', + tradeSanctions: 'economic', + cyberDigital: 'infrastructure', + logisticsSupply: 'infrastructure', + infrastructure: 'infrastructure', + energy: 'energy', + governanceInstitutional: 'social-governance', + socialCohesion: 'social-governance', + borderSecurity: 'social-governance', + informationCognitive: 'social-governance', + healthPublicService: 'health-food', + foodWater: 'health-food', + fiscalSpace: 'recovery', + reserveAdequacy: 'recovery', + externalDebtCoverage: 'recovery', + importConcentration: 'recovery', + stateContinuity: 'recovery', + fuelStockDays: 'recovery', + }; + const pillarDomains = { + 'structural-readiness': ['economic', 'social-governance'], + 'live-shock-exposure': ['infrastructure', 'energy', 'health-food'], + 'recovery-capacity': ['recovery'], + }; + const domainWeights = { + economic: 0.17, + infrastructure: 0.15, + energy: 0.11, + 'social-governance': 0.19, + 'health-food': 0.13, + recovery: 0.25, + }; + + const pillarScores = computePillarScoresFromDomains( + dims, dimensionDomains, pillarDomains, domainWeights + ); + assert.strictEqual(pillarScores.length, 3); + for (const ps of pillarScores) { + assert.ok(typeof ps.id === 'string', `pillar entry should have string id`); + assert.ok(typeof ps.score === 'number', `pillar entry should have numeric score`); + assert.ok(ps.score >= 0 && ps.score <= 100, `pillar score ${ps.score} out of range`); + } + const ids = pillarScores.map((p) => p.id); + assert.deepStrictEqual(ids, ['structural-readiness', 'live-shock-exposure', 'recovery-capacity']); + }); +}); + +describe('sensitivity v2: per-dimension goalpost perturbation', () => { + it('produces different maxSwing values for different dimensions', () => { + const dimA = { id: 'dimA', score: 50, coverage: 1 }; + const dimB = { id: 'dimB', score: 50, coverage: 1 }; + const dimensionDomains = { dimA: 'economic', dimB: 'infra' } as Record; + const pillarDomains = { p1: ['economic', 'infra'] } as Record; + const domainWeights = { economic: 0.5, infra: 0.5 }; + const pillarWeights = { p1: 1.0 }; + const alpha = 0.5; + + const indicatorRegistry = [ + { id: 'indA', dimension: 'dimA', goalposts: { worst: 0, best: 100 }, direction: 'higherBetter', weight: 1 }, + { id: 'indB', dimension: 'dimB', goalposts: { worst: 0, best: 1 }, direction: 'higherBetter', weight: 1 }, + ]; + + const countries = [ + { countryCode: 'US', dimensions: [{ ...dimA, score: 80 }, { ...dimB, score: 50 }] }, + { countryCode: 'DE', dimensions: [{ ...dimA, score: 70 }, { ...dimB, score: 60 }] }, + { countryCode: 'JP', dimensions: [{ ...dimA, score: 60 }, { ...dimB, score: 55 }] }, + { countryCode: 'BR', dimensions: [{ ...dimA, score: 50 }, { ...dimB, score: 45 }] }, + ]; + + const baseScores: Record = {}; + for (const cd of countries) { + const ps = computePillarScoresFromDomains(cd.dimensions, dimensionDomains, pillarDomains, domainWeights); + baseScores[cd.countryCode] = computePenalizedPillarScore(ps, pillarWeights, alpha); + } + const baseRanks: Record = {}; + const sorted = Object.entries(baseScores).sort(([, a], [, b]) => b - a); + sorted.forEach(([cc], i) => { baseRanks[cc] = i + 1; }); + + const topN = Object.keys(baseRanks); + const perDimSwings: Record = { dimA: [], dimB: [] }; + + for (const dimId of ['dimA', 'dimB']) { + const dimInds = indicatorRegistry.filter(ind => ind.dimension === dimId); + const perturbedCountries = countries.map(cd => { + const newDims = cd.dimensions.map(dim => { + if (dim.id !== dimId) return { ...dim }; + let tw = 0, ws = 0; + for (const ind of dimInds) { + const pg = perturbGoalposts(ind.goalposts, 0.1); + const raw = normalizeToGoalposts( + dim.score, pg, ind.direction as 'higherBetter' | 'lowerBetter' + ); + ws += raw * ind.weight; + tw += ind.weight; + } + return { ...dim, score: Math.max(0, Math.min(100, tw > 0 ? ws / tw : dim.score)) }; + }); + return { countryCode: cd.countryCode, dimensions: newDims }; + }); + const scores: Record = {}; + for (const cd of perturbedCountries) { + const ps = computePillarScoresFromDomains(cd.dimensions, dimensionDomains, pillarDomains, domainWeights); + scores[cd.countryCode] = computePenalizedPillarScore(ps, pillarWeights, alpha); + } + const ranks: Record = {}; + const s2 = Object.entries(scores).sort(([, a], [, b]) => b - a); + s2.forEach(([cc], i) => { ranks[cc] = i + 1; }); + const maxSwing = Math.max(...topN.map(cc => Math.abs((ranks[cc] || 0) - (baseRanks[cc] || 0))), 0); + perDimSwings[dimId].push(maxSwing); + } + + assert.ok( + perDimSwings.dimA.length > 0 && perDimSwings.dimB.length > 0, + 'both dimensions have swing values' + ); + }); + + it('value near edge of narrow goalposts produces higher swing than midpoint of wide goalposts', () => { + const wideGoalposts = { worst: 0, best: 100 }; + const narrowGoalposts = { worst: 48, best: 52 }; + + let wideTotal = 0; + let narrowTotal = 0; + const trials = 500; + + for (let t = 0; t < trials; t++) { + const widePg = perturbGoalposts(wideGoalposts, 0.1); + const wideScore = normalizeToGoalposts(50, widePg, 'higherBetter'); + wideTotal += Math.abs(wideScore - 50); + + const narrowPg = perturbGoalposts(narrowGoalposts, 0.1); + const narrowScore = normalizeToGoalposts(51.5, narrowPg, 'higherBetter'); + narrowTotal += Math.abs(narrowScore - normalizeToGoalposts(51.5, narrowGoalposts, 'higherBetter')); + } + + const wideAvg = wideTotal / trials; + const narrowAvg = narrowTotal / trials; + + assert.ok( + narrowAvg > wideAvg, + `narrow goalposts near edge (avg shift=${narrowAvg.toFixed(2)}) should produce higher swing than wide at midpoint (avg shift=${wideAvg.toFixed(2)})` + ); + }); +}); + +describe('sensitivity v2: percentile', () => { + it('p50 of [1,2,3,4,5] is 3', () => { + assert.strictEqual(percentile([1, 2, 3, 4, 5], 50), 3); + }); + + it('p0 returns first element', () => { + assert.strictEqual(percentile([10, 20, 30], 0), 10); + }); + + it('p100 returns last element', () => { + assert.strictEqual(percentile([10, 20, 30], 100), 30); + }); + + it('empty array returns 0', () => { + assert.strictEqual(percentile([], 50), 0); + }); +});