mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(resilience): sensitivity suite v2 + ceiling-effect detection (T2.6/T2.8) (#2991)
* feat(resilience): sensitivity suite v2 + ceiling-effect detection (Phase 2 T2.6/T2.8) Extend weight-perturbation sensitivity analysis for the three-pillar structure: - Pass 1: domain weight perturbation (+-20%, renormalized) - Pass 2: pillar weight perturbation (+-20%, renormalized) - Pass 3: goalpost perturbation (+-10% of indicator goalposts) - Pass 4: alpha-sensitivity curve (sweep 0.0-1.0 in 0.1 steps) Release gate: block if >20% of dimensions have top-10 rank swing >3 positions under perturbation. Ceiling/floor detection: flags any country whose overall score clips to 100 or 0 under standard perturbation passes. 24 unit tests covering perturbWeights, perturbGoalposts, normalizeToGoalposts, computePenalizedPillarScore, spearmanCorrelation, computeReleaseGate, ceiling detection, computePillarScoresFromDomains, and percentile. * fix(resilience): sensitivity exit code + per-dimension goalpost perturbation (#2991 P1) Two P1 findings: 1. The main catch block set process.exitCode = 0, masking fatal errors as success. Changed to exitCode = 1. 2. The goalpost perturbation pass fed the same country-level swings into every dimId, so computeReleaseGate() could only pass or fail ALL dimensions together. Restructured to perturb one dimension's goalposts at a time and record per-dimension rank swings, making the "fail >20% of dimensions" gate actually diagnostic. * fix(resilience): named pillar scores + stable rank sort (#2991 review) Return Array<{id, score}> from computePillarScoresFromDomains and look up weight by pillarWeights[entry.id] instead of positional index. Add localeCompare tiebreaker in rankCountries for deterministic sort.
This commit is contained in:
Elie Habib
@@ -1,32 +1,34 @@
|
||||
#!/usr/bin/env node
|
||||
// Coverage perturbation Monte Carlo — tests ranking stability under coverage variation.
|
||||
// Perturbs each dimension's coverage ±10% and recomputes via the production
|
||||
// sum(domainScore * domainWeight) formula.
|
||||
// Sensitivity analysis v2: weight/goalpost/alpha perturbation + ceiling-effect detection.
|
||||
// Extends the original coverage-perturbation Monte Carlo with:
|
||||
// Pass 1: Domain weight perturbation (±20%)
|
||||
// Pass 2: Pillar weight perturbation (±20%, renormalized)
|
||||
// Pass 3: Goalpost perturbation (±10%)
|
||||
// Pass 4: Alpha-sensitivity curve (0.0-1.0 in 0.1 steps)
|
||||
// Usage: node --import tsx/esm scripts/validate-resilience-sensitivity.mjs
|
||||
|
||||
import { loadEnvFile } from './_seed-utils.mjs';
|
||||
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const NUM_DRAWS = 100;
|
||||
const PERTURBATION_RANGE = 0.1; // ±10%
|
||||
const STABILITY_GATE_RANKS = 5;
|
||||
const NUM_DRAWS = 50;
|
||||
const DOMAIN_PERTURBATION = 0.2;
|
||||
const PILLAR_PERTURBATION = 0.2;
|
||||
const GOALPOST_PERTURBATION = 0.1;
|
||||
const TOP_N = 50;
|
||||
const RANK_SWING_THRESHOLD = 3;
|
||||
const DIMENSION_FAIL_THRESHOLD = 0.20;
|
||||
const MIN_SAMPLE = 20;
|
||||
|
||||
const SAMPLE = [
|
||||
// Top tier
|
||||
'NO','IS','NZ','DK','SE','FI','CH','AU','CA',
|
||||
// High
|
||||
'US','DE','GB','FR','JP','KR','IT','ES','PL',
|
||||
// Upper-mid
|
||||
'BR','MX','TR','TH','MY','CN','IN','ZA','EG',
|
||||
// Lower-mid
|
||||
'PK','NG','KE','BD','VN','PH','ID','UA','RU',
|
||||
// Fragile
|
||||
'AF','YE','SO','HT','SS','CF','SD','ML','NE','TD','SY','IQ','MM','VE','IR','ET',
|
||||
];
|
||||
|
||||
function percentile(sortedValues, p) {
|
||||
export function percentile(sortedValues, p) {
|
||||
if (sortedValues.length === 0) return 0;
|
||||
const index = (p / 100) * (sortedValues.length - 1);
|
||||
const lower = Math.floor(index);
|
||||
@@ -41,20 +43,48 @@ function coverageWeightedMean(dims) {
|
||||
return dims.reduce((s, d) => s + d.score * d.coverage, 0) / totalCoverage;
|
||||
}
|
||||
|
||||
function computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeights, perturb) {
|
||||
export function perturbWeights(weights, range) {
|
||||
const perturbed = {};
|
||||
let total = 0;
|
||||
for (const [k, v] of Object.entries(weights)) {
|
||||
const factor = 1 + (Math.random() * 2 - 1) * range;
|
||||
perturbed[k] = v * factor;
|
||||
total += perturbed[k];
|
||||
}
|
||||
for (const k of Object.keys(perturbed)) {
|
||||
perturbed[k] /= total;
|
||||
}
|
||||
return perturbed;
|
||||
}
|
||||
|
||||
export function perturbGoalposts(goalposts, range) {
|
||||
const span = Math.abs(goalposts.best - goalposts.worst) || 1;
|
||||
const worstShift = (Math.random() * 2 - 1) * range * span;
|
||||
const bestShift = (Math.random() * 2 - 1) * range * span;
|
||||
return {
|
||||
worst: goalposts.worst + worstShift,
|
||||
best: goalposts.best + bestShift,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeToGoalposts(value, goalposts, direction) {
|
||||
const { worst, best } = goalposts;
|
||||
if (best === worst) return 50;
|
||||
const raw = direction === 'higherBetter'
|
||||
? (value - worst) / (best - worst)
|
||||
: (worst - value) / (worst - best);
|
||||
return Math.max(0, Math.min(100, raw * 100));
|
||||
}
|
||||
|
||||
function computeOverallFromDomains(dimensions, dimensionDomains, domainWeights) {
|
||||
const grouped = new Map();
|
||||
for (const domainId of Object.keys(domainWeights)) grouped.set(domainId, []);
|
||||
|
||||
for (const dim of dimensions) {
|
||||
const scaledCoverage = perturb
|
||||
? dim.coverage * (0.9 + Math.random() * 0.2)
|
||||
: dim.coverage;
|
||||
const domainId = dimensionDomains[dim.id];
|
||||
if (domainId && grouped.has(domainId)) {
|
||||
grouped.get(domainId).push({ score: dim.score, coverage: scaledCoverage });
|
||||
grouped.get(domainId).push({ score: dim.score, coverage: dim.coverage });
|
||||
}
|
||||
}
|
||||
|
||||
let overall = 0;
|
||||
for (const [domainId, dims] of grouped) {
|
||||
overall += coverageWeightedMean(dims) * domainWeights[domainId];
|
||||
@@ -62,19 +92,72 @@ function computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeight
|
||||
return overall;
|
||||
}
|
||||
|
||||
function rankCountries(countryData, dimensionDomains, domainWeights, perturb) {
|
||||
const scored = countryData.map(({ countryCode, dimensions }) => ({
|
||||
countryCode,
|
||||
score: computeOverallScorePerturbed(dimensions, dimensionDomains, domainWeights, perturb),
|
||||
}));
|
||||
scored.sort((a, b) => b.score - a.score || a.countryCode.localeCompare(b.countryCode));
|
||||
export function computePenalizedPillarScore(pillarScores, pillarWeights, alpha) {
|
||||
if (pillarScores.length === 0) return 0;
|
||||
const weighted = pillarScores.reduce((s, entry) => {
|
||||
return s + entry.score * (pillarWeights[entry.id] || 0);
|
||||
}, 0);
|
||||
const minScore = Math.min(...pillarScores.map((e) => e.score));
|
||||
const penalty = 1 - alpha * (1 - minScore / 100);
|
||||
return weighted * penalty;
|
||||
}
|
||||
|
||||
export function computePillarScoresFromDomains(dimensions, dimensionDomains, pillarDomains, domainWeights) {
|
||||
const domainScores = {};
|
||||
const grouped = new Map();
|
||||
for (const domainId of Object.keys(domainWeights)) grouped.set(domainId, []);
|
||||
for (const dim of dimensions) {
|
||||
const domainId = dimensionDomains[dim.id];
|
||||
if (domainId && grouped.has(domainId)) {
|
||||
grouped.get(domainId).push({ score: dim.score, coverage: dim.coverage });
|
||||
}
|
||||
}
|
||||
for (const [domainId, dims] of grouped) {
|
||||
domainScores[domainId] = coverageWeightedMean(dims);
|
||||
}
|
||||
|
||||
const pillarScores = [];
|
||||
for (const [pillarId, domainIds] of Object.entries(pillarDomains)) {
|
||||
const scores = domainIds.map((d) => domainScores[d] || 0);
|
||||
const weights = domainIds.map((d) => domainWeights[d] || 0);
|
||||
const totalW = weights.reduce((s, w) => s + w, 0);
|
||||
const pillarScore = totalW > 0
|
||||
? scores.reduce((s, sc, i) => s + sc * weights[i], 0) / totalW
|
||||
: 0;
|
||||
pillarScores.push({ id: pillarId, score: pillarScore });
|
||||
}
|
||||
return pillarScores;
|
||||
}
|
||||
|
||||
function rankCountries(scores) {
|
||||
const sorted = Object.entries(scores)
|
||||
.sort(([a, scoreA], [b, scoreB]) => scoreB - scoreA || a.localeCompare(b));
|
||||
const ranks = {};
|
||||
for (let i = 0; i < scored.length; i++) {
|
||||
ranks[scored[i].countryCode] = i + 1;
|
||||
for (let i = 0; i < sorted.length; i++) {
|
||||
ranks[sorted[i][0]] = i + 1;
|
||||
}
|
||||
return ranks;
|
||||
}
|
||||
|
||||
export function spearmanCorrelation(ranksA, ranksB) {
|
||||
const keys = Object.keys(ranksA).filter((k) => k in ranksB);
|
||||
const n = keys.length;
|
||||
if (n < 2) return 1;
|
||||
const dSqSum = keys.reduce((s, k) => s + (ranksA[k] - ranksB[k]) ** 2, 0);
|
||||
return 1 - (6 * dSqSum) / (n * (n * n - 1));
|
||||
}
|
||||
|
||||
export function computeReleaseGate(dimensionResults) {
|
||||
const failCount = dimensionResults.filter((d) => !d.pass).length;
|
||||
const failPct = dimensionResults.length > 0 ? failCount / dimensionResults.length : 0;
|
||||
return {
|
||||
pass: failPct <= DIMENSION_FAIL_THRESHOLD,
|
||||
failCount,
|
||||
failPct: Math.round(failPct * 1000) / 1000,
|
||||
threshold: DIMENSION_FAIL_THRESHOLD,
|
||||
};
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const {
|
||||
scoreAllDimensions,
|
||||
@@ -85,7 +168,21 @@ async function run() {
|
||||
createMemoizedSeedReader,
|
||||
} = await import('../server/worldmonitor/resilience/v1/_dimension-scorers.ts');
|
||||
|
||||
const { listScorableCountries } = await import('../server/worldmonitor/resilience/v1/_shared.ts');
|
||||
const {
|
||||
listScorableCountries,
|
||||
PENALTY_ALPHA,
|
||||
penalizedPillarScore,
|
||||
} = await import('../server/worldmonitor/resilience/v1/_shared.ts');
|
||||
|
||||
const {
|
||||
PILLAR_DOMAINS,
|
||||
PILLAR_WEIGHTS,
|
||||
PILLAR_ORDER,
|
||||
} = await import('../server/worldmonitor/resilience/v1/_pillar-membership.ts');
|
||||
|
||||
const {
|
||||
INDICATOR_REGISTRY,
|
||||
} = await import('../server/worldmonitor/resilience/v1/_indicator-registry.ts');
|
||||
|
||||
const domainWeights = {};
|
||||
for (const domainId of RESILIENCE_DOMAIN_ORDER) {
|
||||
@@ -111,7 +208,6 @@ async function run() {
|
||||
score: scoreMap[dimId].score,
|
||||
coverage: scoreMap[dimId].coverage,
|
||||
}));
|
||||
|
||||
countryData.push({ countryCode, dimensions });
|
||||
}
|
||||
|
||||
@@ -120,80 +216,295 @@ async function run() {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Scored all ${countryData.length} countries. Running ${NUM_DRAWS} Monte Carlo draws...\n`);
|
||||
console.log(`Scored ${countryData.length} countries. Running sensitivity passes...\n`);
|
||||
|
||||
const rankHistory = {};
|
||||
for (const cc of validSample) rankHistory[cc] = [];
|
||||
|
||||
for (let draw = 0; draw < NUM_DRAWS; draw++) {
|
||||
const ranks = rankCountries(countryData, RESILIENCE_DIMENSION_DOMAINS, domainWeights, true);
|
||||
for (const cc of validSample) {
|
||||
rankHistory[cc].push(ranks[cc]);
|
||||
const baselineScores = {};
|
||||
for (const cd of countryData) {
|
||||
const pillarScores = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
baselineScores[cd.countryCode] = computePenalizedPillarScore(
|
||||
pillarScores, PILLAR_WEIGHTS, PENALTY_ALPHA
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = validSample.map((cc) => {
|
||||
const ranks = rankHistory[cc].slice().sort((a, b) => a - b);
|
||||
const meanRank = ranks.reduce((s, r) => s + r, 0) / ranks.length;
|
||||
const p05 = percentile(ranks, 5);
|
||||
const p95 = percentile(ranks, 95);
|
||||
return { countryCode: cc, meanRank, p05, p95, range: p95 - p05 };
|
||||
});
|
||||
|
||||
stats.sort((a, b) => a.range - b.range || a.meanRank - b.meanRank);
|
||||
|
||||
console.log(`=== SENSITIVITY ANALYSIS (${NUM_DRAWS} draws, ±${PERTURBATION_RANGE * 100}% coverage perturbation) ===\n`);
|
||||
|
||||
console.log('TOP 10 MOST STABLE (smallest rank range in 95% CI):');
|
||||
for (let i = 0; i < Math.min(10, stats.length); i++) {
|
||||
const s = stats[i];
|
||||
console.log(` ${String(i + 1).padStart(2)}. ${s.countryCode} mean_rank=${s.meanRank.toFixed(1)} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} range=${s.range.toFixed(1)}`);
|
||||
}
|
||||
|
||||
console.log('\nTOP 10 LEAST STABLE (largest rank range in 95% CI):');
|
||||
const leastStable = stats.slice().sort((a, b) => b.range - a.range || b.meanRank - a.meanRank);
|
||||
for (let i = 0; i < Math.min(10, leastStable.length); i++) {
|
||||
const s = leastStable[i];
|
||||
console.log(` ${String(i + 1).padStart(2)}. ${s.countryCode} mean_rank=${s.meanRank.toFixed(1)} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} range=${s.range.toFixed(1)}`);
|
||||
}
|
||||
|
||||
const baselineRanks = rankCountries(countryData, RESILIENCE_DIMENSION_DOMAINS, domainWeights, false);
|
||||
const top10 = Object.entries(baselineRanks)
|
||||
const baselineRanks = rankCountries(baselineScores);
|
||||
const topNCountries = Object.entries(baselineRanks)
|
||||
.sort(([, a], [, b]) => a - b)
|
||||
.slice(0, 10)
|
||||
.slice(0, TOP_N)
|
||||
.map(([cc]) => cc);
|
||||
|
||||
let gatePass = true;
|
||||
console.log('\nTOP-10 BASELINE RANK STABILITY CHECK (must be within ±5 ranks in 95% of draws):');
|
||||
for (const cc of top10) {
|
||||
const s = stats.find((x) => x.countryCode === cc);
|
||||
if (!s) continue;
|
||||
const baseRank = baselineRanks[cc];
|
||||
const stable = Math.abs(s.p05 - baseRank) <= STABILITY_GATE_RANKS && Math.abs(s.p95 - baseRank) <= STABILITY_GATE_RANKS;
|
||||
if (!stable) gatePass = false;
|
||||
console.log(` ${cc} baseline_rank=${baseRank} p05=${s.p05.toFixed(1)} p95=${s.p95.toFixed(1)} ${stable ? 'PASS' : 'FAIL'}`);
|
||||
const ceilingEffects = [];
|
||||
|
||||
function detectCeiling(scores, passName) {
|
||||
for (const [cc, score] of Object.entries(scores)) {
|
||||
if (score >= 100) ceilingEffects.push({ countryCode: cc, score, pass: passName, type: 'ceiling' });
|
||||
if (score <= 0) ceilingEffects.push({ countryCode: cc, score, pass: passName, type: 'floor' });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nGATE CHECK: Top-10 stable within ±${STABILITY_GATE_RANKS} ranks? ${gatePass ? 'YES' : 'NO'}`);
|
||||
function computeMaxSwings(perturbedRanks, baseRanks, topCountries) {
|
||||
const swings = {};
|
||||
for (const cc of topCountries) {
|
||||
const base = baseRanks[cc];
|
||||
const perturbed = perturbedRanks[cc];
|
||||
if (base != null && perturbed != null) {
|
||||
swings[cc] = Math.abs(perturbed - base);
|
||||
}
|
||||
}
|
||||
return swings;
|
||||
}
|
||||
|
||||
const allRanges = stats.map((s) => s.range);
|
||||
const meanRange = allRanges.length > 0
|
||||
? allRanges.reduce((s, r) => s + r, 0) / allRanges.length
|
||||
// Pass 1: Domain weight perturbation
|
||||
console.log(`=== PASS 1: Domain weight perturbation (±${DOMAIN_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`);
|
||||
const domainWeightSwings = {};
|
||||
for (const cc of topNCountries) domainWeightSwings[cc] = [];
|
||||
|
||||
for (let draw = 0; draw < NUM_DRAWS; draw++) {
|
||||
const pWeights = perturbWeights(domainWeights, DOMAIN_PERTURBATION);
|
||||
const scores = {};
|
||||
for (const cd of countryData) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, pWeights
|
||||
);
|
||||
scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA);
|
||||
}
|
||||
detectCeiling(scores, 'domainWeights');
|
||||
const ranks = rankCountries(scores);
|
||||
const swings = computeMaxSwings(ranks, baselineRanks, topNCountries);
|
||||
for (const cc of topNCountries) {
|
||||
domainWeightSwings[cc].push(swings[cc] || 0);
|
||||
}
|
||||
}
|
||||
|
||||
const domainMaxSwing = Math.max(
|
||||
...topNCountries.map((cc) => Math.max(...(domainWeightSwings[cc] || [0])))
|
||||
);
|
||||
console.log(` Max top-${TOP_N} rank swing: ${domainMaxSwing}`);
|
||||
|
||||
// Pass 2: Pillar weight perturbation
|
||||
console.log(`\n=== PASS 2: Pillar weight perturbation (±${PILLAR_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`);
|
||||
const pillarWeightSwings = {};
|
||||
for (const cc of topNCountries) pillarWeightSwings[cc] = [];
|
||||
|
||||
for (let draw = 0; draw < NUM_DRAWS; draw++) {
|
||||
const pPillarWeights = perturbWeights(PILLAR_WEIGHTS, PILLAR_PERTURBATION);
|
||||
const scores = {};
|
||||
for (const cd of countryData) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
scores[cd.countryCode] = computePenalizedPillarScore(ps, pPillarWeights, PENALTY_ALPHA);
|
||||
}
|
||||
detectCeiling(scores, 'pillarWeights');
|
||||
const ranks = rankCountries(scores);
|
||||
const swings = computeMaxSwings(ranks, baselineRanks, topNCountries);
|
||||
for (const cc of topNCountries) {
|
||||
pillarWeightSwings[cc].push(swings[cc] || 0);
|
||||
}
|
||||
}
|
||||
|
||||
const pillarMaxSwing = Math.max(
|
||||
...topNCountries.map((cc) => Math.max(...(pillarWeightSwings[cc] || [0])))
|
||||
);
|
||||
console.log(` Max top-${TOP_N} rank swing: ${pillarMaxSwing}`);
|
||||
|
||||
// Pass 3: Goalpost perturbation
|
||||
console.log(`\n=== PASS 3: Goalpost perturbation (±${GOALPOST_PERTURBATION * 100}%, ${NUM_DRAWS} draws) ===`);
|
||||
const goalpostSwings = {};
|
||||
for (const cc of topNCountries) goalpostSwings[cc] = [];
|
||||
const perDimensionSwings = {};
|
||||
for (const dimId of RESILIENCE_DIMENSION_ORDER) perDimensionSwings[dimId] = [];
|
||||
|
||||
for (let draw = 0; draw < NUM_DRAWS; draw++) {
|
||||
const perturbedDims = countryData.map((cd) => {
|
||||
const newDims = cd.dimensions.map((dim) => {
|
||||
const indicators = INDICATOR_REGISTRY.filter((ind) => ind.dimension === dim.id);
|
||||
if (indicators.length === 0) return { ...dim };
|
||||
let totalWeight = 0;
|
||||
let weightedScore = 0;
|
||||
for (const ind of indicators) {
|
||||
const pg = perturbGoalposts(ind.goalposts, GOALPOST_PERTURBATION);
|
||||
const rawScore = normalizeToGoalposts(
|
||||
inverseNormalize(dim.score, ind.goalposts, ind.direction),
|
||||
pg,
|
||||
ind.direction
|
||||
);
|
||||
weightedScore += rawScore * ind.weight;
|
||||
totalWeight += ind.weight;
|
||||
}
|
||||
const newScore = totalWeight > 0 ? weightedScore / totalWeight : dim.score;
|
||||
return { ...dim, score: Math.max(0, Math.min(100, newScore)) };
|
||||
});
|
||||
return { countryCode: cd.countryCode, dimensions: newDims };
|
||||
});
|
||||
|
||||
const scores = {};
|
||||
for (const cd of perturbedDims) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA);
|
||||
}
|
||||
detectCeiling(scores, 'goalposts');
|
||||
const ranks = rankCountries(scores);
|
||||
const swings = computeMaxSwings(ranks, baselineRanks, topNCountries);
|
||||
for (const cc of topNCountries) {
|
||||
goalpostSwings[cc].push(swings[cc] || 0);
|
||||
}
|
||||
}
|
||||
|
||||
for (const dimId of RESILIENCE_DIMENSION_ORDER) {
|
||||
const dimIndicators = INDICATOR_REGISTRY.filter((ind) => ind.dimension === dimId);
|
||||
if (dimIndicators.length === 0) continue;
|
||||
const perturbedDims = countryData.map((cd) => {
|
||||
const newDims = cd.dimensions.map((dim) => {
|
||||
if (dim.id !== dimId) return { ...dim };
|
||||
let totalWeight = 0;
|
||||
let weightedScore = 0;
|
||||
for (const ind of dimIndicators) {
|
||||
const pg = perturbGoalposts(ind.goalposts, GOALPOST_PERTURBATION);
|
||||
const rawScore = normalizeToGoalposts(
|
||||
inverseNormalize(dim.score, ind.goalposts, ind.direction),
|
||||
pg,
|
||||
ind.direction
|
||||
);
|
||||
weightedScore += rawScore * ind.weight;
|
||||
totalWeight += ind.weight;
|
||||
}
|
||||
const newScore = totalWeight > 0 ? weightedScore / totalWeight : dim.score;
|
||||
return { ...dim, score: Math.max(0, Math.min(100, newScore)) };
|
||||
});
|
||||
return { countryCode: cd.countryCode, dimensions: newDims };
|
||||
});
|
||||
const dimScores = {};
|
||||
for (const cd of perturbedDims) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
dimScores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, PENALTY_ALPHA);
|
||||
}
|
||||
const dimRanks = rankCountries(dimScores);
|
||||
const dimSwings = computeMaxSwings(dimRanks, baselineRanks, topNCountries);
|
||||
const maxDimSwing = Math.max(...topNCountries.slice(0, 10).map((cc) => dimSwings[cc] || 0), 0);
|
||||
perDimensionSwings[dimId].push(maxDimSwing);
|
||||
}
|
||||
|
||||
const goalpostMaxSwing = Math.max(
|
||||
...topNCountries.map((cc) => Math.max(...(goalpostSwings[cc] || [0])))
|
||||
);
|
||||
console.log(` Max top-${TOP_N} rank swing: ${goalpostMaxSwing}`);
|
||||
|
||||
// Pass 4: Alpha sensitivity curve
|
||||
console.log(`\n=== PASS 4: Alpha sensitivity curve (0.0 to 1.0, step 0.1) ===`);
|
||||
const baseAlphaRanks = {};
|
||||
for (const cd of countryData) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
baseAlphaRanks[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, 0.5);
|
||||
}
|
||||
const baseAlphaRanked = rankCountries(baseAlphaRanks);
|
||||
|
||||
const alphaSensitivity = [];
|
||||
for (let alphaStep = 0; alphaStep <= 10; alphaStep++) {
|
||||
const alpha = Math.round(alphaStep * 10) / 100;
|
||||
const scores = {};
|
||||
for (const cd of countryData) {
|
||||
const ps = computePillarScoresFromDomains(
|
||||
cd.dimensions, RESILIENCE_DIMENSION_DOMAINS, PILLAR_DOMAINS, domainWeights
|
||||
);
|
||||
scores[cd.countryCode] = computePenalizedPillarScore(ps, PILLAR_WEIGHTS, alpha);
|
||||
}
|
||||
const ranks = rankCountries(scores);
|
||||
const spearman = spearmanCorrelation(baseAlphaRanked, ranks);
|
||||
const maxSwing = Math.max(
|
||||
...topNCountries.map((cc) => Math.abs((ranks[cc] || 0) - (baseAlphaRanked[cc] || 0)))
|
||||
);
|
||||
alphaSensitivity.push({
|
||||
alpha,
|
||||
spearmanVs05: Math.round(spearman * 10000) / 10000,
|
||||
maxTop50Swing: maxSwing,
|
||||
});
|
||||
}
|
||||
|
||||
console.log(' alpha | spearman_vs_0.5 | max_top50_swing');
|
||||
console.log(' ------+-----------------+----------------');
|
||||
for (const row of alphaSensitivity) {
|
||||
console.log(` ${row.alpha.toFixed(1).padStart(5)} | ${row.spearmanVs05.toFixed(4).padStart(15)} | ${String(row.maxTop50Swing).padStart(14)}`);
|
||||
}
|
||||
|
||||
// Dimension stability
|
||||
const dimensionResults = RESILIENCE_DIMENSION_ORDER.map((dimId) => {
|
||||
const maxSwing = perDimensionSwings[dimId]?.length > 0
|
||||
? Math.max(...perDimensionSwings[dimId])
|
||||
: 0;
|
||||
const maxRange = allRanges.length > 0 ? Math.max(...allRanges) : 0;
|
||||
const minRange = allRanges.length > 0 ? Math.min(...allRanges) : 0;
|
||||
console.log(`\nSUMMARY STATISTICS:`);
|
||||
console.log(` Countries sampled: ${countryData.length}`);
|
||||
console.log(` Monte Carlo draws: ${NUM_DRAWS}`);
|
||||
console.log(` Perturbation: ±${PERTURBATION_RANGE * 100}% on dimension coverage weights`);
|
||||
console.log(` Mean rank range (p05-p95): ${meanRange.toFixed(1)}`);
|
||||
console.log(` Min rank range: ${minRange.toFixed(1)}`);
|
||||
console.log(` Max rank range: ${maxRange.toFixed(1)}`);
|
||||
return { dimId, maxSwing, pass: maxSwing <= RANK_SWING_THRESHOLD };
|
||||
});
|
||||
|
||||
const releaseGate = computeReleaseGate(dimensionResults);
|
||||
|
||||
console.log('\n=== DIMENSION STABILITY (goalpost perturbation, top-10 rank swing) ===');
|
||||
for (const dr of dimensionResults) {
|
||||
console.log(` ${dr.dimId.padEnd(25)} maxSwing=${dr.maxSwing} ${dr.pass ? 'PASS' : 'FAIL'}`);
|
||||
}
|
||||
|
||||
console.log(`\n=== RELEASE GATE ===`);
|
||||
console.log(` Threshold: >${releaseGate.threshold * 100}% of dimensions failing (swing > ${RANK_SWING_THRESHOLD} ranks)`);
|
||||
console.log(` Failed: ${releaseGate.failCount}/${dimensionResults.length} (${(releaseGate.failPct * 100).toFixed(1)}%)`);
|
||||
console.log(` Result: ${releaseGate.pass ? 'PASS' : 'FAIL'}`);
|
||||
|
||||
// Ceiling effects
|
||||
const uniqueCeilings = [];
|
||||
const seen = new Set();
|
||||
for (const ce of ceilingEffects) {
|
||||
const key = `${ce.countryCode}:${ce.type}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
uniqueCeilings.push(ce);
|
||||
}
|
||||
}
|
||||
|
||||
if (uniqueCeilings.length > 0) {
|
||||
console.log(`\n=== CEILING/FLOOR EFFECTS (${uniqueCeilings.length} unique) ===`);
|
||||
for (const ce of uniqueCeilings.slice(0, 20)) {
|
||||
console.log(` ${ce.countryCode} ${ce.type} score=${ce.score.toFixed(2)} pass=${ce.pass}`);
|
||||
}
|
||||
} else {
|
||||
console.log('\n=== CEILING/FLOOR EFFECTS: None detected ===');
|
||||
}
|
||||
|
||||
const result = {
|
||||
generatedAt: Date.now(),
|
||||
passes: {
|
||||
domainWeights: { maxSwing: domainMaxSwing, pass: domainMaxSwing <= RANK_SWING_THRESHOLD * 2 },
|
||||
pillarWeights: { maxSwing: pillarMaxSwing, pass: pillarMaxSwing <= RANK_SWING_THRESHOLD * 2 },
|
||||
goalposts: { maxSwing: goalpostMaxSwing, pass: goalpostMaxSwing <= RANK_SWING_THRESHOLD * 2 },
|
||||
},
|
||||
alphaSensitivity,
|
||||
dimensionStability: dimensionResults,
|
||||
releaseGate,
|
||||
ceilingEffects: uniqueCeilings,
|
||||
};
|
||||
|
||||
console.log(`\nSensitivity analysis v2 complete.`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function inverseNormalize(normalizedScore, goalposts, direction) {
|
||||
const { worst, best } = goalposts;
|
||||
if (best === worst) return worst;
|
||||
if (direction === 'higherBetter') {
|
||||
return worst + (normalizedScore / 100) * (best - worst);
|
||||
}
|
||||
return worst - (normalizedScore / 100) * (worst - best);
|
||||
}
|
||||
|
||||
const isMain = process.argv[1]?.endsWith('validate-resilience-sensitivity.mjs');
|
||||
if (isMain) {
|
||||
run().then(() => process.exit(0)).catch((err) => {
|
||||
run().then((_result) => {
|
||||
console.log('\nJSON output written to stdout (pipe to file if needed).');
|
||||
process.exit(0);
|
||||
}).catch((err) => {
|
||||
console.error('Sensitivity analysis failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
368
tests/resilience-sensitivity-v2.test.mts
Normal file
368
tests/resilience-sensitivity-v2.test.mts
Normal file
@@ -0,0 +1,368 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
|
||||
import {
|
||||
perturbWeights,
|
||||
perturbGoalposts,
|
||||
normalizeToGoalposts,
|
||||
computePenalizedPillarScore,
|
||||
computePillarScoresFromDomains,
|
||||
spearmanCorrelation,
|
||||
computeReleaseGate,
|
||||
percentile,
|
||||
} from '../scripts/validate-resilience-sensitivity.mjs';
|
||||
|
||||
describe('sensitivity v2: perturbWeights', () => {
|
||||
it('renormalizes perturbed weights to sum=1', () => {
|
||||
const weights = { a: 0.40, b: 0.35, c: 0.25 };
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const p = perturbWeights(weights, 0.2);
|
||||
const sum = Object.values(p).reduce((s, v) => s + v, 0);
|
||||
assert.ok(Math.abs(sum - 1.0) < 1e-10, `sum=${sum} should be 1.0`);
|
||||
assert.ok(p.a > 0 && p.b > 0 && p.c > 0, 'all weights positive');
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves key set', () => {
|
||||
const weights = { x: 0.5, y: 0.3, z: 0.2 };
|
||||
const p = perturbWeights(weights, 0.1);
|
||||
assert.deepStrictEqual(Object.keys(p).sort(), ['x', 'y', 'z']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: perturbGoalposts', () => {
|
||||
it('returns worst and best within expected range', () => {
|
||||
const gp = { worst: 0, best: 100 };
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const p = perturbGoalposts(gp, 0.1);
|
||||
assert.ok(typeof p.worst === 'number');
|
||||
assert.ok(typeof p.best === 'number');
|
||||
assert.ok(Math.abs(p.worst - gp.worst) <= 15, `worst shift too large: ${p.worst}`);
|
||||
assert.ok(Math.abs(p.best - gp.best) <= 15, `best shift too large: ${p.best}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: normalizeToGoalposts', () => {
|
||||
it('higherBetter: worst=0, best=100, value=50 => 50', () => {
|
||||
const result = normalizeToGoalposts(50, { worst: 0, best: 100 }, 'higherBetter');
|
||||
assert.strictEqual(result, 50);
|
||||
});
|
||||
|
||||
it('higherBetter: clamps at 0 and 100', () => {
|
||||
assert.strictEqual(normalizeToGoalposts(-10, { worst: 0, best: 100 }, 'higherBetter'), 0);
|
||||
assert.strictEqual(normalizeToGoalposts(200, { worst: 0, best: 100 }, 'higherBetter'), 100);
|
||||
});
|
||||
|
||||
it('lowerBetter: worst=20, best=0, value=10 => 50', () => {
|
||||
const result = normalizeToGoalposts(10, { worst: 20, best: 0 }, 'lowerBetter');
|
||||
assert.strictEqual(result, 50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: computePenalizedPillarScore', () => {
|
||||
it('returns 0 for empty array', () => {
|
||||
assert.strictEqual(computePenalizedPillarScore([], {}, 0.5), 0);
|
||||
});
|
||||
|
||||
it('applies penalty based on min pillar score', () => {
|
||||
const scores = [{ id: 'a', score: 80 }, { id: 'b', score: 60 }, { id: 'c', score: 70 }];
|
||||
const weights = { a: 0.4, b: 0.35, c: 0.25 };
|
||||
const alpha = 0.5;
|
||||
const weighted = 80 * 0.4 + 60 * 0.35 + 70 * 0.25;
|
||||
const penalty = 1 - 0.5 * (1 - 60 / 100);
|
||||
const expected = weighted * penalty;
|
||||
const result = computePenalizedPillarScore(scores, weights, alpha);
|
||||
assert.ok(Math.abs(result - expected) < 0.01, `${result} vs ${expected}`);
|
||||
});
|
||||
|
||||
it('no penalty when all pillar scores are 100', () => {
|
||||
const scores = [{ id: 'a', score: 100 }, { id: 'b', score: 100 }, { id: 'c', score: 100 }];
|
||||
const weights = { a: 0.4, b: 0.35, c: 0.25 };
|
||||
const result = computePenalizedPillarScore(scores, weights, 0.5);
|
||||
assert.strictEqual(result, 100);
|
||||
});
|
||||
|
||||
it('alpha=0 means no penalty', () => {
|
||||
const scores = [{ id: 'a', score: 80 }, { id: 'b', score: 20 }, { id: 'c', score: 50 }];
|
||||
const weights = { a: 0.4, b: 0.35, c: 0.25 };
|
||||
const result0 = computePenalizedPillarScore(scores, weights, 0);
|
||||
const weighted = 80 * 0.4 + 20 * 0.35 + 50 * 0.25;
|
||||
assert.ok(Math.abs(result0 - weighted) < 0.01);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: spearmanCorrelation', () => {
|
||||
it('returns 1.0 for identical rankings', () => {
|
||||
const ranks = { US: 1, DE: 2, JP: 3, BR: 4 };
|
||||
assert.strictEqual(spearmanCorrelation(ranks, ranks), 1);
|
||||
});
|
||||
|
||||
it('returns negative for inverted rankings', () => {
|
||||
const a = { US: 1, DE: 2, JP: 3, BR: 4 };
|
||||
const b = { US: 4, DE: 3, JP: 2, BR: 1 };
|
||||
const result = spearmanCorrelation(a, b);
|
||||
assert.strictEqual(result, -1);
|
||||
});
|
||||
|
||||
it('alpha=0.5 vs itself is 1.0', () => {
|
||||
const ranks = { NO: 1, SE: 2, FI: 3, DK: 4, CH: 5 };
|
||||
assert.strictEqual(spearmanCorrelation(ranks, ranks), 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: computeReleaseGate', () => {
|
||||
it('19 dimensions, 4 fail => 21% => FAIL', () => {
|
||||
const dims = Array.from({ length: 19 }, (_, i) => ({
|
||||
dimId: `dim${i}`,
|
||||
maxSwing: i < 4 ? 5 : 1,
|
||||
pass: i >= 4,
|
||||
}));
|
||||
const gate = computeReleaseGate(dims);
|
||||
assert.strictEqual(gate.pass, false);
|
||||
assert.strictEqual(gate.failCount, 4);
|
||||
assert.ok(gate.failPct > 0.20);
|
||||
});
|
||||
|
||||
it('19 dimensions, 3 fail => 15.8% => PASS', () => {
|
||||
const dims = Array.from({ length: 19 }, (_, i) => ({
|
||||
dimId: `dim${i}`,
|
||||
maxSwing: i < 3 ? 5 : 1,
|
||||
pass: i >= 3,
|
||||
}));
|
||||
const gate = computeReleaseGate(dims);
|
||||
assert.strictEqual(gate.pass, true);
|
||||
assert.strictEqual(gate.failCount, 3);
|
||||
assert.ok(gate.failPct < 0.20);
|
||||
});
|
||||
|
||||
it('0 dimensions => pass (no failures)', () => {
|
||||
const gate = computeReleaseGate([]);
|
||||
assert.strictEqual(gate.pass, true);
|
||||
assert.strictEqual(gate.failCount, 0);
|
||||
});
|
||||
|
||||
it('all pass => gate passes', () => {
|
||||
const dims = Array.from({ length: 10 }, (_, i) => ({
|
||||
dimId: `dim${i}`,
|
||||
maxSwing: 1,
|
||||
pass: true,
|
||||
}));
|
||||
const gate = computeReleaseGate(dims);
|
||||
assert.strictEqual(gate.pass, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: ceiling detection', () => {
|
||||
it('score=100 is flagged as ceiling', () => {
|
||||
const scores = { US: 100, DE: 85 };
|
||||
const ceilings = [];
|
||||
for (const [cc, score] of Object.entries(scores)) {
|
||||
if (score >= 100) ceilings.push({ countryCode: cc, score, type: 'ceiling' });
|
||||
if (score <= 0) ceilings.push({ countryCode: cc, score, type: 'floor' });
|
||||
}
|
||||
assert.strictEqual(ceilings.length, 1);
|
||||
assert.strictEqual(ceilings[0].countryCode, 'US');
|
||||
assert.strictEqual(ceilings[0].type, 'ceiling');
|
||||
});
|
||||
|
||||
it('score=0 is flagged as floor', () => {
|
||||
const scores = { AF: 0, NO: 80 };
|
||||
const ceilings = [];
|
||||
for (const [cc, score] of Object.entries(scores)) {
|
||||
if (score >= 100) ceilings.push({ countryCode: cc, score, type: 'ceiling' });
|
||||
if (score <= 0) ceilings.push({ countryCode: cc, score, type: 'floor' });
|
||||
}
|
||||
assert.strictEqual(ceilings.length, 1);
|
||||
assert.strictEqual(ceilings[0].countryCode, 'AF');
|
||||
assert.strictEqual(ceilings[0].type, 'floor');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: computePillarScoresFromDomains', () => {
|
||||
it('computes pillar scores from domain groupings', () => {
|
||||
const dims = [
|
||||
{ id: 'macroFiscal', score: 80, coverage: 1 },
|
||||
{ id: 'currencyExternal', score: 60, coverage: 1 },
|
||||
{ id: 'tradeSanctions', score: 70, coverage: 1 },
|
||||
{ id: 'cyberDigital', score: 50, coverage: 1 },
|
||||
{ id: 'logisticsSupply', score: 40, coverage: 1 },
|
||||
{ id: 'infrastructure', score: 60, coverage: 1 },
|
||||
{ id: 'energy', score: 55, coverage: 1 },
|
||||
{ id: 'governanceInstitutional', score: 75, coverage: 1 },
|
||||
{ id: 'socialCohesion', score: 65, coverage: 1 },
|
||||
{ id: 'borderSecurity', score: 70, coverage: 1 },
|
||||
{ id: 'informationCognitive', score: 60, coverage: 1 },
|
||||
{ id: 'healthPublicService', score: 80, coverage: 1 },
|
||||
{ id: 'foodWater', score: 70, coverage: 1 },
|
||||
{ id: 'fiscalSpace', score: 45, coverage: 1 },
|
||||
{ id: 'reserveAdequacy', score: 50, coverage: 1 },
|
||||
{ id: 'externalDebtCoverage', score: 55, coverage: 1 },
|
||||
{ id: 'importConcentration', score: 60, coverage: 1 },
|
||||
{ id: 'stateContinuity', score: 65, coverage: 1 },
|
||||
{ id: 'fuelStockDays', score: 40, coverage: 1 },
|
||||
];
|
||||
const dimensionDomains = {
|
||||
macroFiscal: 'economic',
|
||||
currencyExternal: 'economic',
|
||||
tradeSanctions: 'economic',
|
||||
cyberDigital: 'infrastructure',
|
||||
logisticsSupply: 'infrastructure',
|
||||
infrastructure: 'infrastructure',
|
||||
energy: 'energy',
|
||||
governanceInstitutional: 'social-governance',
|
||||
socialCohesion: 'social-governance',
|
||||
borderSecurity: 'social-governance',
|
||||
informationCognitive: 'social-governance',
|
||||
healthPublicService: 'health-food',
|
||||
foodWater: 'health-food',
|
||||
fiscalSpace: 'recovery',
|
||||
reserveAdequacy: 'recovery',
|
||||
externalDebtCoverage: 'recovery',
|
||||
importConcentration: 'recovery',
|
||||
stateContinuity: 'recovery',
|
||||
fuelStockDays: 'recovery',
|
||||
};
|
||||
const pillarDomains = {
|
||||
'structural-readiness': ['economic', 'social-governance'],
|
||||
'live-shock-exposure': ['infrastructure', 'energy', 'health-food'],
|
||||
'recovery-capacity': ['recovery'],
|
||||
};
|
||||
const domainWeights = {
|
||||
economic: 0.17,
|
||||
infrastructure: 0.15,
|
||||
energy: 0.11,
|
||||
'social-governance': 0.19,
|
||||
'health-food': 0.13,
|
||||
recovery: 0.25,
|
||||
};
|
||||
|
||||
const pillarScores = computePillarScoresFromDomains(
|
||||
dims, dimensionDomains, pillarDomains, domainWeights
|
||||
);
|
||||
assert.strictEqual(pillarScores.length, 3);
|
||||
for (const ps of pillarScores) {
|
||||
assert.ok(typeof ps.id === 'string', `pillar entry should have string id`);
|
||||
assert.ok(typeof ps.score === 'number', `pillar entry should have numeric score`);
|
||||
assert.ok(ps.score >= 0 && ps.score <= 100, `pillar score ${ps.score} out of range`);
|
||||
}
|
||||
const ids = pillarScores.map((p) => p.id);
|
||||
assert.deepStrictEqual(ids, ['structural-readiness', 'live-shock-exposure', 'recovery-capacity']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: per-dimension goalpost perturbation', () => {
|
||||
it('produces different maxSwing values for different dimensions', () => {
|
||||
const dimA = { id: 'dimA', score: 50, coverage: 1 };
|
||||
const dimB = { id: 'dimB', score: 50, coverage: 1 };
|
||||
const dimensionDomains = { dimA: 'economic', dimB: 'infra' } as Record<string, string>;
|
||||
const pillarDomains = { p1: ['economic', 'infra'] } as Record<string, string[]>;
|
||||
const domainWeights = { economic: 0.5, infra: 0.5 };
|
||||
const pillarWeights = { p1: 1.0 };
|
||||
const alpha = 0.5;
|
||||
|
||||
const indicatorRegistry = [
|
||||
{ id: 'indA', dimension: 'dimA', goalposts: { worst: 0, best: 100 }, direction: 'higherBetter', weight: 1 },
|
||||
{ id: 'indB', dimension: 'dimB', goalposts: { worst: 0, best: 1 }, direction: 'higherBetter', weight: 1 },
|
||||
];
|
||||
|
||||
const countries = [
|
||||
{ countryCode: 'US', dimensions: [{ ...dimA, score: 80 }, { ...dimB, score: 50 }] },
|
||||
{ countryCode: 'DE', dimensions: [{ ...dimA, score: 70 }, { ...dimB, score: 60 }] },
|
||||
{ countryCode: 'JP', dimensions: [{ ...dimA, score: 60 }, { ...dimB, score: 55 }] },
|
||||
{ countryCode: 'BR', dimensions: [{ ...dimA, score: 50 }, { ...dimB, score: 45 }] },
|
||||
];
|
||||
|
||||
const baseScores: Record<string, number> = {};
|
||||
for (const cd of countries) {
|
||||
const ps = computePillarScoresFromDomains(cd.dimensions, dimensionDomains, pillarDomains, domainWeights);
|
||||
baseScores[cd.countryCode] = computePenalizedPillarScore(ps, pillarWeights, alpha);
|
||||
}
|
||||
const baseRanks: Record<string, number> = {};
|
||||
const sorted = Object.entries(baseScores).sort(([, a], [, b]) => b - a);
|
||||
sorted.forEach(([cc], i) => { baseRanks[cc] = i + 1; });
|
||||
|
||||
const topN = Object.keys(baseRanks);
|
||||
const perDimSwings: Record<string, number[]> = { dimA: [], dimB: [] };
|
||||
|
||||
for (const dimId of ['dimA', 'dimB']) {
|
||||
const dimInds = indicatorRegistry.filter(ind => ind.dimension === dimId);
|
||||
const perturbedCountries = countries.map(cd => {
|
||||
const newDims = cd.dimensions.map(dim => {
|
||||
if (dim.id !== dimId) return { ...dim };
|
||||
let tw = 0, ws = 0;
|
||||
for (const ind of dimInds) {
|
||||
const pg = perturbGoalposts(ind.goalposts, 0.1);
|
||||
const raw = normalizeToGoalposts(
|
||||
dim.score, pg, ind.direction as 'higherBetter' | 'lowerBetter'
|
||||
);
|
||||
ws += raw * ind.weight;
|
||||
tw += ind.weight;
|
||||
}
|
||||
return { ...dim, score: Math.max(0, Math.min(100, tw > 0 ? ws / tw : dim.score)) };
|
||||
});
|
||||
return { countryCode: cd.countryCode, dimensions: newDims };
|
||||
});
|
||||
const scores: Record<string, number> = {};
|
||||
for (const cd of perturbedCountries) {
|
||||
const ps = computePillarScoresFromDomains(cd.dimensions, dimensionDomains, pillarDomains, domainWeights);
|
||||
scores[cd.countryCode] = computePenalizedPillarScore(ps, pillarWeights, alpha);
|
||||
}
|
||||
const ranks: Record<string, number> = {};
|
||||
const s2 = Object.entries(scores).sort(([, a], [, b]) => b - a);
|
||||
s2.forEach(([cc], i) => { ranks[cc] = i + 1; });
|
||||
const maxSwing = Math.max(...topN.map(cc => Math.abs((ranks[cc] || 0) - (baseRanks[cc] || 0))), 0);
|
||||
perDimSwings[dimId].push(maxSwing);
|
||||
}
|
||||
|
||||
assert.ok(
|
||||
perDimSwings.dimA.length > 0 && perDimSwings.dimB.length > 0,
|
||||
'both dimensions have swing values'
|
||||
);
|
||||
});
|
||||
|
||||
it('value near edge of narrow goalposts produces higher swing than midpoint of wide goalposts', () => {
|
||||
const wideGoalposts = { worst: 0, best: 100 };
|
||||
const narrowGoalposts = { worst: 48, best: 52 };
|
||||
|
||||
let wideTotal = 0;
|
||||
let narrowTotal = 0;
|
||||
const trials = 500;
|
||||
|
||||
for (let t = 0; t < trials; t++) {
|
||||
const widePg = perturbGoalposts(wideGoalposts, 0.1);
|
||||
const wideScore = normalizeToGoalposts(50, widePg, 'higherBetter');
|
||||
wideTotal += Math.abs(wideScore - 50);
|
||||
|
||||
const narrowPg = perturbGoalposts(narrowGoalposts, 0.1);
|
||||
const narrowScore = normalizeToGoalposts(51.5, narrowPg, 'higherBetter');
|
||||
narrowTotal += Math.abs(narrowScore - normalizeToGoalposts(51.5, narrowGoalposts, 'higherBetter'));
|
||||
}
|
||||
|
||||
const wideAvg = wideTotal / trials;
|
||||
const narrowAvg = narrowTotal / trials;
|
||||
|
||||
assert.ok(
|
||||
narrowAvg > wideAvg,
|
||||
`narrow goalposts near edge (avg shift=${narrowAvg.toFixed(2)}) should produce higher swing than wide at midpoint (avg shift=${wideAvg.toFixed(2)})`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitivity v2: percentile', () => {
|
||||
it('p50 of [1,2,3,4,5] is 3', () => {
|
||||
assert.strictEqual(percentile([1, 2, 3, 4, 5], 50), 3);
|
||||
});
|
||||
|
||||
it('p0 returns first element', () => {
|
||||
assert.strictEqual(percentile([10, 20, 30], 0), 10);
|
||||
});
|
||||
|
||||
it('p100 returns last element', () => {
|
||||
assert.strictEqual(percentile([10, 20, 30], 100), 30);
|
||||
});
|
||||
|
||||
it('empty array returns 0', () => {
|
||||
assert.strictEqual(percentile([], 50), 0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user