mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(resilience): publish resilience:static:fao aggregate from static seed (#3050)
* feat(resilience): publish resilience:static:fao aggregate from static seed
Weekly validation cron Outcome-Backtest reads resilience:static:fao for
the Food Crisis Escalation family, but nothing wrote that key — dangling
reference, Food Crisis stuck at AUC=0.5.
IPC Phase 3+ data is already fetched by fetchFsinDataset (HDX global IPC
CSV) and stored per-country. This PR reshapes the same in-memory map into
an aggregate view and writes it in the existing Redis pipeline — no extra
fetch, no new cron service.
Output shape matches what detectFoodCrisis already walks:
{ countries: { [iso2]: { ipcPhase, phase, peopleInCrisis, year, source } },
count, fetchedAt, seedYear, source: 'hdx-ipc' }
Only Phase 3+ countries are included, matching IPC's own publish rule.
Absence = not-monitored-crisis, consistent with scoreFoodWater()'s
stable-absence semantics.
Tests: 5 unit tests for buildFaoAggregate (incl. contract test against
detectFoodCrisis) + 1 health.js registration test. No cron/Railway
changes needed — seed-bundle-static-ref picks it up on its next October
window; restart to backfill sooner.
FX Stress / Power Outages / Refugees / Conflict also fail today but for
different reasons (detector shape mismatches) — out of scope here.
* fix(resilience): wire resilienceStaticFao into SEED_META to unmask empty-state
Reviewer catch on #3050: adding resilienceStaticFao to STANDALONE_KEYS
and EMPTY_DATA_OK_KEYS without a matching SEED_META entry leaves
seedStale=null in the standalone-key health branch, so an empty or
missing resilience:static:fao key resolves to plain OK instead of
STALE_SEED — silently masking the exact bug this PR is meant to
surface.
Adds SEED_META.resilienceStaticFao pointing at seed-meta:resilience:static
(same heartbeat as resilienceStaticIndex, since the aggregate is written
in the same Redis pipeline by the same seeder). Now: missing data with
stale heartbeat -> STALE_SEED (warn); with fresh heartbeat and no
countries in Phase 3+ -> OK (still valid per EMPTY_DATA_OK_KEYS).
Same trap documented in feedback_empty_data_ok_keys_bootstrap_blind_spot.md
but in the STANDALONE_KEYS path, not BOOTSTRAP_KEYS.
Test locks it in with a source-string regex assertion.
This commit is contained in:
@@ -149,6 +149,7 @@ const STANDALONE_KEYS = {
|
||||
climateNews: 'climate:news-intelligence:v1',
|
||||
pizzint: 'intelligence:pizzint:seed:v1',
|
||||
resilienceStaticIndex: 'resilience:static:index:v1',
|
||||
resilienceStaticFao: 'resilience:static:fao',
|
||||
resilienceRanking: 'resilience:ranking:v9',
|
||||
productCatalog: 'product-catalog:v2',
|
||||
energySpineCountries: 'energy:spine:v1:_countries',
|
||||
@@ -308,6 +309,7 @@ const SEED_META = {
|
||||
vpdTrackerRealtime: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // daily seed (0 2 * * *); 2880min = 48h = 2x interval
|
||||
vpdTrackerHistorical: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // shares seed-meta key with vpdTrackerRealtime (same run)
|
||||
resilienceStaticIndex: { key: 'seed-meta:resilience:static', maxStaleMin: 576000 }, // annual October snapshot; 400d threshold matches TTL and preserves prior-year data on source outages
|
||||
resilienceStaticFao: { key: 'seed-meta:resilience:static', maxStaleMin: 576000 }, // same seeder + same heartbeat as resilienceStaticIndex; required so EMPTY_DATA_OK + missing data degrades to STALE_SEED instead of silent OK
|
||||
resilienceRanking: { key: 'seed-meta:resilience:ranking', maxStaleMin: 720 }, // on-demand RPC cache (6h TTL); 12h threshold catches stale rankings without paging on cold start
|
||||
resilienceIntervals: { key: 'seed-meta:resilience:intervals', maxStaleMin: 20160 }, // weekly cron; 20160min = 14d = 2x interval
|
||||
energyExposure: { key: 'seed-meta:economic:owid-energy-mix', maxStaleMin: 50400 }, // monthly cron on 1st; 50400min = 35d = TTL matches cron cadence + 5d buffer
|
||||
@@ -373,6 +375,7 @@ const EMPTY_DATA_OK_KEYS = new Set([
|
||||
'recoveryFiscalSpace',
|
||||
'recoveryImportHhi', 'recoveryFuelStocks', // recovery pillar seeds: stub seeders write empty payloads until real sources are wired
|
||||
'ddosAttacks', 'trafficAnomalies', // zero events during quiet periods is valid, not critical
|
||||
'resilienceStaticFao', // empty aggregate = no IPC Phase 3+ countries this year (possible in theory); the key must exist but count=0 is fine
|
||||
]);
|
||||
|
||||
// Cascade groups: if any key in the group has data, all empty siblings are OK.
|
||||
|
||||
@@ -28,6 +28,11 @@ loadEnvFile(import.meta.url);
|
||||
export const RESILIENCE_STATIC_INDEX_KEY = 'resilience:static:index:v1';
|
||||
export const RESILIENCE_STATIC_META_KEY = 'seed-meta:resilience:static';
|
||||
export const RESILIENCE_STATIC_PREFIX = 'resilience:static:';
|
||||
// Aggregated IPC Phase 3+ view — readers that want "which countries are in a
|
||||
// food crisis this year" without fanning out to 222 per-country keys. Shape is
|
||||
// compatible with scripts/backtest-resilience-outcomes.mjs::detectFoodCrisis:
|
||||
// { countries: { ISO2: { ipcPhase, phase, peopleInCrisis, year } } }.
|
||||
export const RESILIENCE_STATIC_FAO_KEY = 'resilience:static:fao';
|
||||
export const RESILIENCE_STATIC_TTL_SECONDS = 400 * 24 * 60 * 60;
|
||||
export const RESILIENCE_STATIC_SOURCE_VERSION = 'resilience-static-v7';
|
||||
export const RESILIENCE_STATIC_WINDOW_CRON = '0 */4 1-3 10 *';
|
||||
@@ -825,13 +830,53 @@ async function readJsonKey(key) {
|
||||
return verifySeedKey(key);
|
||||
}
|
||||
|
||||
async function publishSuccess(countryPayloads, manifest, meta) {
|
||||
/**
|
||||
* Build the aggregated `resilience:static:fao` payload from the per-country
|
||||
* FAO dataset map. Only includes countries that IPC lists as Phase 3+ — by
|
||||
* design, IPC's "global latest" CSV only publishes crisis cases, so absence
|
||||
* from this aggregate means "not an IPC-monitored crisis country" (consistent
|
||||
* with how scoreFoodWater() treats missing per-country fao data).
|
||||
*
|
||||
* Output shape matches backtest-resilience-outcomes.mjs::detectFoodCrisis:
|
||||
* { countries: { [iso2]: { ipcPhase, phase, peopleInCrisis, year, source } },
|
||||
* fetchedAt, source, count, seedYear }
|
||||
*/
|
||||
export function buildFaoAggregate(faoMap, seedYear, seededAt) {
|
||||
const countries = {};
|
||||
let count = 0;
|
||||
for (const [iso2, entry] of faoMap.entries()) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const phaseMatch = typeof entry.phase === 'string' ? entry.phase.match(/\d+/) : null;
|
||||
const ipcPhase = phaseMatch ? Number(phaseMatch[0]) : null;
|
||||
if (ipcPhase == null || ipcPhase < 3) continue;
|
||||
countries[iso2] = {
|
||||
ipcPhase,
|
||||
phase: entry.phase,
|
||||
peopleInCrisis: entry.peopleInCrisis ?? null,
|
||||
year: entry.year ?? null,
|
||||
source: entry.source ?? 'hdx-ipc',
|
||||
};
|
||||
count += 1;
|
||||
}
|
||||
return {
|
||||
countries,
|
||||
count,
|
||||
fetchedAt: seededAt,
|
||||
seedYear,
|
||||
source: 'hdx-ipc',
|
||||
};
|
||||
}
|
||||
|
||||
async function publishSuccess(countryPayloads, manifest, meta, { faoAggregate } = {}) {
|
||||
const commands = [];
|
||||
for (const [iso2, payload] of countryPayloads.entries()) {
|
||||
commands.push(['SET', countryRedisKey(iso2), JSON.stringify(payload), 'EX', RESILIENCE_STATIC_TTL_SECONDS]);
|
||||
}
|
||||
commands.push(['SET', RESILIENCE_STATIC_INDEX_KEY, JSON.stringify(manifest), 'EX', RESILIENCE_STATIC_TTL_SECONDS]);
|
||||
commands.push(['SET', RESILIENCE_STATIC_META_KEY, JSON.stringify(meta), 'EX', RESILIENCE_STATIC_TTL_SECONDS]);
|
||||
if (faoAggregate) {
|
||||
commands.push(['SET', RESILIENCE_STATIC_FAO_KEY, JSON.stringify(faoAggregate), 'EX', RESILIENCE_STATIC_TTL_SECONDS]);
|
||||
}
|
||||
const results = await redisPipeline(commands);
|
||||
const failures = results.filter(r => r?.error || r?.result === 'ERR');
|
||||
if (failures.length > 0) {
|
||||
@@ -988,7 +1033,15 @@ export async function seedResilienceStatic() {
|
||||
failedDatasets,
|
||||
});
|
||||
|
||||
await publishSuccess(countryPayloads, manifest, meta);
|
||||
// Piggyback on the same fetch: the FAO dataset map is already in memory,
|
||||
// just reshape and publish as an aggregate readable by the weekly
|
||||
// validation cron's Outcome-Backtest (resilience:static:fao). Skip when
|
||||
// the FAO fetch itself failed — the rest of the snapshot is still valid.
|
||||
const faoAggregate = failedDatasets.includes('fao')
|
||||
? null
|
||||
: buildFaoAggregate(datasetMaps.fao ?? new Map(), seedYear, seededAt);
|
||||
|
||||
await publishSuccess(countryPayloads, manifest, meta, { faoAggregate });
|
||||
|
||||
return {
|
||||
skipped: false,
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
RESILIENCE_STATIC_META_KEY,
|
||||
RESILIENCE_STATIC_SOURCE_VERSION,
|
||||
buildFailureRefreshKeys,
|
||||
buildFaoAggregate,
|
||||
buildManifest,
|
||||
buildTradeToGdpMap,
|
||||
countryRedisKey,
|
||||
@@ -186,6 +187,77 @@ describe('resilience static seed CSV parsers', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFaoAggregate', () => {
|
||||
const seededAt = '2026-04-13T08:00:00.000Z';
|
||||
const seedYear = 2026;
|
||||
|
||||
it('returns a detectFoodCrisis-compatible shape with countries keyed by ISO2', () => {
|
||||
const faoMap = new Map([
|
||||
['SS', { source: 'hdx-ipc', year: 2025, peopleInCrisis: 7700000, phase: 'IPC Phase 4' }],
|
||||
['YE', { source: 'hdx-ipc', year: 2024, peopleInCrisis: 17000000, phase: 'IPC Phase 3' }],
|
||||
]);
|
||||
const aggregate = buildFaoAggregate(faoMap, seedYear, seededAt);
|
||||
|
||||
assert.equal(aggregate.source, 'hdx-ipc');
|
||||
assert.equal(aggregate.seedYear, 2026);
|
||||
assert.equal(aggregate.fetchedAt, seededAt);
|
||||
assert.equal(aggregate.count, 2);
|
||||
assert.deepEqual(Object.keys(aggregate.countries).sort(), ['SS', 'YE']);
|
||||
assert.equal(aggregate.countries.SS.ipcPhase, 4);
|
||||
assert.equal(aggregate.countries.SS.phase, 'IPC Phase 4');
|
||||
assert.equal(aggregate.countries.SS.peopleInCrisis, 7700000);
|
||||
assert.equal(aggregate.countries.YE.ipcPhase, 3);
|
||||
});
|
||||
|
||||
it('includes only Phase 3+ countries (IPC crisis threshold)', () => {
|
||||
const faoMap = new Map([
|
||||
['SS', { source: 'hdx-ipc', peopleInCrisis: 7700000, phase: 'IPC Phase 4' }],
|
||||
['KE', { source: 'hdx-ipc', peopleInCrisis: 500000, phase: 'IPC Phase 2' }],
|
||||
]);
|
||||
const aggregate = buildFaoAggregate(faoMap, seedYear, seededAt);
|
||||
assert.equal(aggregate.count, 1);
|
||||
assert.ok('SS' in aggregate.countries);
|
||||
assert.ok(!('KE' in aggregate.countries), 'Phase 2 country must be excluded');
|
||||
});
|
||||
|
||||
it('skips entries with unparseable phase strings', () => {
|
||||
const faoMap = new Map([
|
||||
['SS', { source: 'hdx-ipc', phase: 'IPC Phase 3' }],
|
||||
['AA', { source: 'hdx-ipc', phase: null }],
|
||||
['BB', { source: 'hdx-ipc', phase: 'Unknown' }],
|
||||
]);
|
||||
const aggregate = buildFaoAggregate(faoMap, seedYear, seededAt);
|
||||
assert.equal(aggregate.count, 1);
|
||||
assert.deepEqual(Object.keys(aggregate.countries), ['SS']);
|
||||
});
|
||||
|
||||
it('returns an empty aggregate when the input map is empty', () => {
|
||||
const aggregate = buildFaoAggregate(new Map(), seedYear, seededAt);
|
||||
assert.equal(aggregate.count, 0);
|
||||
assert.deepEqual(aggregate.countries, {});
|
||||
assert.equal(aggregate.seedYear, 2026);
|
||||
});
|
||||
|
||||
it('is readable by backtest-resilience-outcomes.mjs::detectFoodCrisis (contract)', async () => {
|
||||
// Locks the contract between this seeder and the downstream validator.
|
||||
// If detectFoodCrisis is refactored, or the aggregate shape drifts, this
|
||||
// fails loudly instead of silently returning 0 positive events in the
|
||||
// weekly validation cron.
|
||||
const { detectFoodCrisis } = await import('../scripts/backtest-resilience-outcomes.mjs');
|
||||
const faoMap = new Map([
|
||||
['SS', { source: 'hdx-ipc', year: 2025, peopleInCrisis: 7700000, phase: 'IPC Phase 4' }],
|
||||
['YE', { source: 'hdx-ipc', year: 2024, peopleInCrisis: 17000000, phase: 'IPC Phase 3' }],
|
||||
['KE', { source: 'hdx-ipc', peopleInCrisis: 500000, phase: 'IPC Phase 2' }],
|
||||
]);
|
||||
const aggregate = buildFaoAggregate(faoMap, seedYear, seededAt);
|
||||
const labels = detectFoodCrisis(aggregate, ['SS', 'YE', 'KE', 'NO']);
|
||||
assert.equal(labels.get('SS'), true);
|
||||
assert.equal(labels.get('YE'), true);
|
||||
assert.equal(labels.get('KE'), undefined, 'Phase 2 must not be labeled crisis');
|
||||
assert.equal(labels.get('NO'), undefined, 'non-IPC country must not be labeled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildAquastatWbMap', () => {
|
||||
it('produces the { source, value, indicator, year } shape scoreAquastatValue() reads', () => {
|
||||
const input = new Map([
|
||||
@@ -537,6 +609,29 @@ describe('resilience static health registrations', () => {
|
||||
assert.match(healthSrc, /seed-meta:resilience:static/);
|
||||
});
|
||||
|
||||
it('registers the FAO aggregate key with empty-data tolerance in health.js', () => {
|
||||
// buildFaoAggregate writes `resilience:static:fao` during the annual
|
||||
// static seed. Health must know about the key (STANDALONE_KEYS) AND
|
||||
// tolerate count=0 (EMPTY_DATA_OK_KEYS) — a year with no countries in
|
||||
// IPC Phase 3+ is theoretically valid, not a paging event.
|
||||
assert.match(healthSrc, /resilienceStaticFao:\s+'resilience:static:fao'/);
|
||||
assert.match(healthSrc, /'resilienceStaticFao'/);
|
||||
});
|
||||
|
||||
it('registers SEED_META for resilienceStaticFao so empty data degrades to STALE_SEED, not silent OK', () => {
|
||||
// Without a SEED_META entry, the STANDALONE_KEYS health branch leaves
|
||||
// seedStale=null and treats an empty/missing key in EMPTY_DATA_OK_KEYS
|
||||
// as plain OK — which would mask the exact "nothing wrote the key"
|
||||
// state this seeder is designed to fix. Must share the static seeder's
|
||||
// heartbeat (seed-meta:resilience:static) since the aggregate is
|
||||
// written in the same Redis pipeline.
|
||||
assert.match(
|
||||
healthSrc,
|
||||
/resilienceStaticFao:\s*\{\s*key:\s*'seed-meta:resilience:static'/,
|
||||
'resilienceStaticFao must appear in SEED_META pointing at seed-meta:resilience:static',
|
||||
);
|
||||
});
|
||||
|
||||
it('registers annual seed-health monitoring for resilience static', () => {
|
||||
assert.match(seedHealthSrc, /'resilience:static':\s+\{ key: 'seed-meta:resilience:static',\s+intervalMin: 288000 \}/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user