mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
9d27ff0d6a
* fix(seeds): strict-floor validators must not poison seed-meta on empty When `runSeed`'s validateFn rejected (empty/short data), seed-meta was refreshed with `fetchedAt=now, recordCount=0`. Bundle runners read `fetchedAt` to decide skip — so one transient empty fetch locked the IMF-extended bundle (30-day cadence) out for a full month. Adds opt-in `emptyDataIsFailure` flag that skips the meta refresh on validation failure, letting the bundle retry next cron fire and health flip to STALE_SEED. Wires it on all four IMF/WEO seeders (floor 150-190 countries), which structurally can't have legitimate empty results. Default behavior unchanged for quiet-period feeds (news, events) where empty is normal. Observed: Railway log 2026-04-13 18:58 — imf-external validation fail; next fire 8h later skipped "483min ago / interval 43200min". * test(seeds): regression coverage for emptyDataIsFailure branch Static-analysis guard against the PR #3078 regression reintroducing itself: - Asserts runSeed gates writeFreshnessMetadata on opts.emptyDataIsFailure and that extendExistingTtl still runs in both branches (cache preserved). - Asserts the four strict-floor IMF seeders (external/growth/labor/macro) pass emptyDataIsFailure: true. Prevents silent bundle-lockout if someone removes the gate or adds a new strict-floor seeder without the flag. * fix(seeds): strict-floor failure must exit(1) + behavioral test P2 (surfacing upstream failures in bundle summary): Strict-floor seeders with emptyDataIsFailure:true now process.exit(1) after logging FAILURE. _bundle-runner's spawnSeed wraps execFile, so non-zero exit rejects → failed++ increments → bundle itself exits 1. Before: bundle logged 'Done' and ran++ on a poisoned upstream, hiding 30-day outages from Railway monitoring. P3 (behavioral regression coverage, replacing static source-shape test): Stubs globalThis.fetch (Upstash REST) + process.exit to drive runSeed through both branches. Asserts on actual Redis commands: - strict path: zero seed-meta SET, pipeline EXPIRE still called, exit(1) - default path: exactly one seed-meta SET, exit(0) Catches future regressions where writeFreshnessMetadata is reintroduced indirectly, and is immune to cosmetic refactors of _seed-utils.mjs. * test(seeds): regression for emptyDataIsFailure meta-refresh gate Proves that validation failure with opts.emptyDataIsFailure:true does NOT write seed-meta (strict-floor seeders) while the default behavior DOES write count=0 meta (quiet-period feeds). Addresses PR #3078 review.
103 lines
3.2 KiB
JavaScript
103 lines
3.2 KiB
JavaScript
#!/usr/bin/env node
|
|
//
|
|
// IMF WEO — labor & demographics
|
|
// Canonical key: economic:imf:labor:v1
|
|
//
|
|
// Indicators:
|
|
// LUR — Unemployment rate, %
|
|
// LP — Population, persons (millions)
|
|
//
|
|
// Per WorldMonitor #3027 — feeds resilience macroFiscal scoring (LUR
|
|
// sub-metric) and CountryDeepDivePanel demographic tiles (LP).
|
|
|
|
import { loadEnvFile, runSeed, loadSharedConfig, imfSdmxFetchIndicator } from './_seed-utils.mjs';
|
|
|
|
loadEnvFile(import.meta.url);
|
|
|
|
const CANONICAL_KEY = 'economic:imf:labor:v1';
|
|
const CACHE_TTL = 35 * 24 * 3600;
|
|
|
|
const ISO2_TO_ISO3 = loadSharedConfig('iso2-to-iso3.json');
|
|
const ISO3_TO_ISO2 = Object.fromEntries(Object.entries(ISO2_TO_ISO3).map(([k, v]) => [v, k]));
|
|
|
|
const AGGREGATE_CODES = new Set([
|
|
'ADVEC', 'EMEDE', 'EURO', 'MECA', 'OEMDC', 'WEOWORLD', 'EU',
|
|
'AS5', 'DA', 'EDE', 'MAE', 'OAE', 'SSA', 'WE', 'EMDE', 'G20',
|
|
]);
|
|
|
|
export function isAggregate(code) {
|
|
if (!code || code.length !== 3) return true;
|
|
return AGGREGATE_CODES.has(code) || code.endsWith('Q');
|
|
}
|
|
|
|
export function weoYears() {
|
|
const y = new Date().getFullYear();
|
|
return [`${y}`, `${y - 1}`, `${y - 2}`];
|
|
}
|
|
|
|
export function latestValue(byYear) {
|
|
for (const year of weoYears()) {
|
|
const v = Number(byYear?.[year]);
|
|
if (Number.isFinite(v)) return { value: v, year: Number(year) };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function buildLaborCountries({ unemployment = {}, population = {} }) {
|
|
const countries = {};
|
|
const allIso3 = new Set([...Object.keys(unemployment), ...Object.keys(population)]);
|
|
for (const iso3 of allIso3) {
|
|
if (isAggregate(iso3)) continue;
|
|
const iso2 = ISO3_TO_ISO2[iso3];
|
|
if (!iso2) continue;
|
|
|
|
const lur = latestValue(unemployment[iso3]);
|
|
const lp = latestValue(population[iso3]);
|
|
|
|
if (!lur && !lp) continue;
|
|
|
|
countries[iso2] = {
|
|
unemploymentPct: lur?.value ?? null,
|
|
populationMillions: lp?.value ?? null,
|
|
year: lur?.year ?? lp?.year ?? null,
|
|
};
|
|
}
|
|
return countries;
|
|
}
|
|
|
|
export async function fetchImfLabor() {
|
|
const years = weoYears();
|
|
const [unemployment, population] = await Promise.all([
|
|
imfSdmxFetchIndicator('LUR', { years }),
|
|
imfSdmxFetchIndicator('LP', { years }),
|
|
]);
|
|
return {
|
|
countries: buildLaborCountries({ unemployment, population }),
|
|
seededAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
// LUR (unemployment) is reported for ~100 countries while population (LP) is
|
|
// reported for ~210. Since buildLaborCountries unions the two, healthy runs
|
|
// yield ~210 countries. Require >=190 to reject partial snapshots; this still
|
|
// accommodates indicators that have slightly narrower reporting.
|
|
export function validate(data) {
|
|
return typeof data?.countries === 'object' && Object.keys(data.countries).length >= 190;
|
|
}
|
|
|
|
export { CANONICAL_KEY, CACHE_TTL };
|
|
|
|
if (process.argv[1]?.endsWith('seed-imf-labor.mjs')) {
|
|
runSeed('economic', 'imf-labor', CANONICAL_KEY, fetchImfLabor, {
|
|
validateFn: validate,
|
|
ttlSeconds: CACHE_TTL,
|
|
sourceVersion: `imf-sdmx-weo-${new Date().getFullYear()}`,
|
|
recordCount: (data) => Object.keys(data?.countries ?? {}).length,
|
|
emptyDataIsFailure: true,
|
|
}).catch((err) => {
|
|
const _cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : '';
|
|
console.error('FATAL:', (err.message || err) + _cause);
|
|
process.exit(1);
|
|
});
|
|
}
|