diff --git a/api/health.js b/api/health.js index f6fd39c95..9142bbbee 100644 --- a/api/health.js +++ b/api/health.js @@ -193,6 +193,8 @@ const STANDALONE_KEYS = { recoveryExternalDebt: 'resilience:recovery:external-debt:v1', recoveryImportHhi: 'resilience:recovery:import-hhi:v1', recoveryFuelStocks: 'resilience:recovery:fuel-stocks:v1', + recoveryReexportShare: 'resilience:recovery:reexport-share:v1', + recoverySovereignWealth: 'resilience:recovery:sovereign-wealth:v1', // PR 1 v2 energy-construct seeds. STRICT SEED_META (not ON_DEMAND): // plan 2026-04-24-001 removed these from ON_DEMAND_KEYS so /api/health // reports CRIT (not WARN) when they are absent. This is the intended @@ -397,6 +399,8 @@ const SEED_META = { recoveryExternalDebt: { key: 'seed-meta:resilience:recovery:external-debt', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval recoveryImportHhi: { key: 'seed-meta:resilience:recovery:import-hhi', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval recoveryFuelStocks: { key: 'seed-meta:resilience:recovery:fuel-stocks', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval + recoveryReexportShare: { key: 'seed-meta:resilience:recovery:reexport-share', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval + recoverySovereignWealth: { key: 'seed-meta:resilience:recovery:sovereign-wealth', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval // PR 1 v2 energy seeds — weekly cron (8d * 1440 = 11520min = 2x interval). // STRICT SEED_META (not ON_DEMAND): plan 2026-04-24-001 made /api/health // CRIT on absent/stale so operators see the Railway-bundle gap before diff --git a/api/seed-health.js b/api/seed-health.js index 8279f347c..cad8315f5 100644 --- a/api/seed-health.js +++ b/api/seed-health.js @@ -102,6 +102,8 @@ const SEED_DOMAINS = { 'economic:eurostat-house-prices': { key: 'seed-meta:economic:eurostat-house-prices', intervalMin: 36000 }, // weekly cron, annual data; intervalMin = health.js maxStaleMin / 2 (72000 / 2) 'economic:eurostat-gov-debt-q': { key: 'seed-meta:economic:eurostat-gov-debt-q', intervalMin: 10080 }, // 2d cron, quarterly data; intervalMin = health.js maxStaleMin / 2 (20160 / 2) 'economic:eurostat-industrial-production': { key: 'seed-meta:economic:eurostat-industrial-production', intervalMin: 3600 }, // daily cron, monthly data; intervalMin = health.js maxStaleMin / 2 (7200 / 2) + 'resilience:recovery:reexport-share': { key: 'seed-meta:resilience:recovery:reexport-share', intervalMin: 43200 }, // monthly bundle cron (30d); intervalMin*2 = 60d matches health.js maxStaleMin + 'resilience:recovery:sovereign-wealth': { key: 'seed-meta:resilience:recovery:sovereign-wealth', intervalMin: 43200 }, // monthly bundle cron (30d); intervalMin*2 = 60d matches health.js maxStaleMin }; async function getMetaBatch(keys) { diff --git a/scripts/seed-bundle-resilience-recovery.mjs b/scripts/seed-bundle-resilience-recovery.mjs index c58d5743d..85d6daf2f 100644 --- a/scripts/seed-bundle-resilience-recovery.mjs +++ b/scripts/seed-bundle-resilience-recovery.mjs @@ -12,11 +12,18 @@ await runBundle('resilience-recovery', [ // and the quarterly revision cadence documented in the manifest. Longer // timeout than peers because Tier 3b (per-fund Wikipedia infobox) is N // network round-trips per manifest fund the list article misses. - // PR 3A §net-imports denominator. Re-export share manifest is read by - // the SWF seeder to convert gross annual imports into NET annual - // imports before computing rawMonths. Must run BEFORE Sovereign-Wealth - // so the SWF seeder sees the freshly-published reexport-share key. - // Sub-second runtime (reads a ~100-line YAML); short timeout. - { label: 'Reexport-Share', script: 'seed-recovery-reexport-share.mjs', seedMetaKey: 'resilience:recovery:reexport-share', canonicalKey: 'resilience:recovery:reexport-share:v1', intervalMs: 30 * DAY, timeoutMs: 60_000 }, - { label: 'Sovereign-Wealth', script: 'seed-sovereign-wealth.mjs', seedMetaKey: 'resilience:recovery:sovereign-wealth', canonicalKey: 'resilience:recovery:sovereign-wealth:v1', intervalMs: 30 * DAY, timeoutMs: 600_000 }, + // Re-export share is read by the SWF seeder to convert gross annual + // imports into NET annual imports before computing rawMonths. Must + // run BEFORE Sovereign-Wealth so the SWF seeder sees the freshly- + // published reexport-share key. Timeout 300s (5 min): the seeder + // fetches Comtrade RX + M per cohort member with 750ms inter-call + // pacing + retry-on-429; a 2-country cohort averages 10-15s but a + // rate-limited retry storm can extend to 2-3 min. + { label: 'Reexport-Share', script: 'seed-recovery-reexport-share.mjs', seedMetaKey: 'resilience:recovery:reexport-share', canonicalKey: 'resilience:recovery:reexport-share:v1', intervalMs: 30 * DAY, timeoutMs: 300_000 }, + // `dependsOn: ['Reexport-Share']` makes the ordering contract explicit + // and enforced by `_bundle-runner.mjs` (throws on violation). The SWF + // seeder reads the Reexport-Share Redis key inside the same bundle + // run; a future edit that reorders these sections would otherwise + // silently corrupt the net-imports denominator. + { label: 'Sovereign-Wealth', script: 'seed-sovereign-wealth.mjs', seedMetaKey: 'resilience:recovery:sovereign-wealth', canonicalKey: 'resilience:recovery:sovereign-wealth:v1', intervalMs: 30 * DAY, timeoutMs: 600_000, dependsOn: ['Reexport-Share'] }, ]); diff --git a/scripts/seed-recovery-reexport-share.mjs b/scripts/seed-recovery-reexport-share.mjs index 9fca865c7..8c9a8c533 100644 --- a/scripts/seed-recovery-reexport-share.mjs +++ b/scripts/seed-recovery-reexport-share.mjs @@ -2,96 +2,343 @@ // seed-recovery-reexport-share // ============================ // -// Publishes `resilience:recovery:reexport-share:v1` from the manifest -// at `scripts/shared/reexport-share-manifest.yaml`. The payload is -// consumed by `scripts/seed-sovereign-wealth.mjs` to convert gross +// Publishes `resilience:recovery:reexport-share:v1` from UN Comtrade, +// computing each country's re-export-share-of-imports as a live ratio +// of `flowCode=RX` over `flowCode=M` aggregate merchandise trade. +// +// Consumed by `scripts/seed-sovereign-wealth.mjs` to convert GROSS // annual imports into NET annual imports when computing the SWF -// `rawMonths` denominator (see plan §PR 3A of -// `docs/plans/2026-04-24-002-fix-resilience-cohort-ranking-structural- -// audit-plan.md`). +// `rawMonths` denominator for the `sovereignFiscalBuffer` dimension. // -// Why a manifest-driven seeder and not a Comtrade fetcher: UNCTAD's -// Handbook of Statistics publishes re-export aggregates annually as -// PDF/Excel with no stable SDMX series for this specific derivative, -// and Comtrade's `flowCode=RX` has uneven coverage across reporters. -// A curated manifest with per-entry source citations is auditable and -// stable; the update cadence is annual (UNCTAD Handbook release). +// netAnnualImports = grossAnnualImports × (1 − reexportShareOfImports) // -// Revision cadence: manifest-edit PR at each UNCTAD Handbook release -// OR when a national stats office materially revises. Every revision -// must cite the source table / year. +// Design decisions — see plan §Phase 1 at +// `docs/plans/2026-04-24-003-feat-reexport-share-comtrade-seeder-plan.md`: +// +// - Hub cohort resolved by Phase 0 empirical RX+M co-population probe +// (see the plan's §"Phase 0 cohort validation results"). As of the +// 2026-04-24 probe: AE + PA. Six other candidates (SG, HK, NL, BE, +// MY, LT) return HTTP 200 with zero RX rows and are excluded until +// Comtrade exposes RX for those reporters. +// - Header auth (`Ocp-Apim-Subscription-Key`) — key never leaks into +// the URL → logs → Redis payload → clipboard. +// - `maxRecords=250000` cap with truncation detection: a full-cap +// response triggers per-country omission so partial data never +// under-reports the share. +// - 4-year period window (Y-1..Y-4), matching the HHI seeder PR #3372. +// - Clamps: share < 0.05 → omit (per-run discipline); share > 0.95 → +// cap at 0.95. computeNetImports requires share < 1. +// - Envelope schema v2 (bumped from manifestVersion=1 manifest flattener). +// +// Revision cadence: none — the monthly bundle cron re-seeds from Comtrade. +// +// Duplication policy: the retry-classification loop is duplicated here +// rather than extracted into a `_comtrade.mjs` helper. Per CLAUDE.md, +// duplication is cheaper than a premature abstraction — a second +// Comtrade caller in the future can extract then. -import { loadEnvFile, runSeed } from './_seed-utils.mjs'; -import { loadReexportShareManifest } from './shared/reexport-share-loader.mjs'; +import { pathToFileURL } from 'node:url'; + +import { CHROME_UA, loadEnvFile, runSeed, sleep } from './_seed-utils.mjs'; loadEnvFile(import.meta.url); const CANONICAL_KEY = 'resilience:recovery:reexport-share:v1'; -// Manifest content changes rarely (annual cadence). 30-day TTL is -// generous enough that a missed Railway tick doesn't evict the key -// before the next scheduled run, while short enough that an updated -// manifest propagates within a deploy cycle. -const CACHE_TTL = 30 * 24 * 3600; +// Monthly bundle cron. TTL large enough that one missed tick doesn't +// evict (the SWF seeder's bundle-freshness guard falls back to gross +// imports if seed-meta predates the current bundle run, independent +// of data-key TTL). +const CACHE_TTL_SECONDS = 35 * 24 * 3600; + +const COMTRADE_URL = 'https://comtradeapi.un.org/data/v1/get/C/A/HS'; +const MAX_RECORDS = 250_000; +const FETCH_TIMEOUT_MS = 45_000; +const RETRY_MAX_ATTEMPTS = 3; +const INTER_CALL_PACING_MS = 750; + +// Share bounds. Floor 0.05 drops commercially-immaterial contributions +// (Panama's 1.4% observed in Phase 0). Ceiling 0.95 prevents pathological +// share=1 reporters from zeroing the denominator via computeNetImports. +const MIN_MATERIAL_SHARE = 0.05; +const MAX_SHARE_CAP = 0.95; + +// Phase 0 resolved cohort — commit 2026-04-24, candidates AE, SG, HK, +// NL, BE, PA, MY, LT probed sequentially via railway run. Only AE and +// PA returned co-populated RX+M rows; see plan §"Phase 0 cohort +// validation results" for full table and HTTP status per candidate. +const REEXPORT_HUB_COHORT = [ + { iso2: 'AE', reporterCode: '784', name: 'United Arab Emirates' }, + { iso2: 'PA', reporterCode: '591', name: 'Panama' }, +]; + +function buildPeriodYears() { + // Y-1..Y-4. Same window as the HHI seeder (PR #3372). Excludes the + // current calendar year (Comtrade lag for annual aggregates). + const now = new Date().getFullYear(); + return [now - 1, now - 2, now - 3, now - 4]; +} + +function auditSafeSourceUrl(reporterCode, flowCode, years) { + // Belt-and-suspenders: even though header auth means the + // subscription-key never gets appended to the URL, construct the + // displayed source string WITHOUT any credential query-params. If + // a future refactor ever adds subscription-key to the URL again, + // this function strips it before it reaches the Redis envelope. + const u = new URL(COMTRADE_URL); + u.searchParams.set('reporterCode', reporterCode); + u.searchParams.set('flowCode', flowCode); + u.searchParams.set('cmdCode', 'TOTAL'); + u.searchParams.set('period', years.join(',')); + u.searchParams.delete('subscription-key'); + return u.toString(); +} + +async function fetchComtradeFlow(apiKey, reporterCode, flowCode, years, { iso2 }) { + const u = new URL(COMTRADE_URL); + u.searchParams.set('reporterCode', reporterCode); + u.searchParams.set('flowCode', flowCode); + u.searchParams.set('cmdCode', 'TOTAL'); + u.searchParams.set('period', years.join(',')); + u.searchParams.set('maxRecords', String(MAX_RECORDS)); + const urlStr = u.toString(); + + for (let attempt = 1; attempt <= RETRY_MAX_ATTEMPTS; attempt += 1) { + try { + const resp = await fetch(urlStr, { + headers: { + 'Ocp-Apim-Subscription-Key': apiKey, + 'User-Agent': CHROME_UA, + Accept: 'application/json', + }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + + if (resp.status === 429) { + if (attempt === RETRY_MAX_ATTEMPTS) { + console.warn(`[reexport-share] ${iso2} ${flowCode}: 429 after ${RETRY_MAX_ATTEMPTS} attempts; omitting`); + return { rows: [], truncated: false, status: 429 }; + } + const backoffMs = 2000 * attempt; + console.warn(`[reexport-share] ${iso2} ${flowCode}: 429 rate-limited, backoff ${backoffMs}ms (attempt ${attempt}/${RETRY_MAX_ATTEMPTS})`); + await sleep(backoffMs); + continue; + } + if (resp.status >= 500) { + if (attempt === RETRY_MAX_ATTEMPTS) { + console.warn(`[reexport-share] ${iso2} ${flowCode}: HTTP ${resp.status} after ${RETRY_MAX_ATTEMPTS} attempts; omitting`); + return { rows: [], truncated: false, status: resp.status }; + } + const backoffMs = 5000 * attempt; + console.warn(`[reexport-share] ${iso2} ${flowCode}: HTTP ${resp.status}, backoff ${backoffMs}ms (attempt ${attempt}/${RETRY_MAX_ATTEMPTS})`); + await sleep(backoffMs); + continue; + } + if (!resp.ok) { + console.warn(`[reexport-share] ${iso2} ${flowCode}: HTTP ${resp.status}; omitting`); + return { rows: [], truncated: false, status: resp.status }; + } + + const json = await resp.json(); + const rows = Array.isArray(json?.data) ? json.data : []; + if (rows.length >= MAX_RECORDS) { + console.warn(`[reexport-share] ${iso2} ${flowCode}: response at cap (${rows.length}>=${MAX_RECORDS}); possible truncation — omitting country`); + return { rows: [], truncated: true, status: 200 }; + } + return { rows, truncated: false, status: 200 }; + } catch (err) { + if (attempt === RETRY_MAX_ATTEMPTS) { + console.warn(`[reexport-share] ${iso2} ${flowCode}: exhausted retries (${err?.message || err}); omitting`); + return { rows: [], truncated: false, status: null, error: err?.message || String(err) }; + } + const backoffMs = 3000 * attempt; + console.warn(`[reexport-share] ${iso2} ${flowCode}: fetch error "${err?.message || err}", backoff ${backoffMs}ms (attempt ${attempt}/${RETRY_MAX_ATTEMPTS})`); + await sleep(backoffMs); + } + } + return { rows: [], truncated: false, status: null }; +} + +/** + * Sum primaryValue per year from a Comtrade flow response. + * USES world-aggregate rows only (partnerCode='0' / 0 / absent) — + * this construct wants the country-total flow as a single figure, not + * a partner-level breakdown. The `cmdCode=TOTAL` query without a + * partner filter defaults to returning only world-aggregate rows in + * practice, but this filter is defensive: if a future refactor asks + * Comtrade for partner-level decomposition (e.g. to cross-check), + * summing partner rows ON TOP of the world-aggregate row would + * silently double-count and cut the derived share in half. + * + * Pure function — exported for tests. + * + * @param {Array} rows + * @returns {Map} year → summed primaryValue in USD + */ +export function parseComtradeFlowResponse(rows) { + const byYear = new Map(); + for (const r of rows) { + // Accept world-aggregate rows only: string '0', numeric 0, or + // the field absent entirely (older response shapes). Any specific + // partnerCode (e.g. '842' for US, '826' for UK) is a per-partner + // breakdown row and must be excluded to avoid double-counting + // against the world-aggregate row for the same year. + const partnerCode = r?.partnerCode; + const isWorldAggregate = partnerCode == null + || partnerCode === '0' + || partnerCode === 0; + if (!isWorldAggregate) continue; + + const yRaw = r?.period ?? r?.refPeriodId; + const y = Number(yRaw); + const v = Number(r?.primaryValue ?? 0); + if (!Number.isInteger(y) || !Number.isFinite(v) || v <= 0) continue; + byYear.set(y, (byYear.get(y) ?? 0) + v); + } + return byYear; +} + +/** + * Given per-year RX and M sums, pick the latest year where BOTH are + * populated (>0), and return the share = RX / M plus metadata. + * + * Returns null if no co-populated year exists. + * + * Pure function — exported for tests. + * + * @param {Map} rxByYear + * @param {Map} mByYear + * @returns {{ year: number, share: number, reexportsUsd: number, importsUsd: number } | null} + */ +export function computeShareFromFlows(rxByYear, mByYear) { + const coPopulated = []; + for (const y of rxByYear.keys()) { + if (mByYear.has(y)) coPopulated.push(y); + } + if (coPopulated.length === 0) return null; + coPopulated.sort((a, b) => b - a); + const year = coPopulated[0]; + const reexportsUsd = rxByYear.get(year); + const importsUsd = mByYear.get(year); + if (!(importsUsd > 0)) return null; + const rawShare = reexportsUsd / importsUsd; + return { year, share: rawShare, reexportsUsd, importsUsd }; +} + +/** + * Clamp a raw share into the material-and-safe range. Returns null for + * sub-floor shares (caller omits the country); caps at MAX_SHARE_CAP + * for above-ceiling shares. Pure function — exported for tests. + * + * @param {number} rawShare + * @returns {number | null} clamped share, or null if sub-floor + */ +export function clampShare(rawShare) { + if (!Number.isFinite(rawShare) || rawShare < 0) return null; + if (rawShare < MIN_MATERIAL_SHARE) return null; + if (rawShare > MAX_SHARE_CAP) return MAX_SHARE_CAP; + return rawShare; +} async function fetchReexportShare() { - const manifest = loadReexportShareManifest(); - const countries = {}; - for (const entry of manifest.countries) { - countries[entry.country] = { - reexportShareOfImports: entry.reexportShareOfImports, - year: entry.year, - sources: entry.sources, - }; + const apiKey = (process.env.COMTRADE_API_KEYS || '').split(',').filter(Boolean)[0]; + if (!apiKey) { + throw new Error('[reexport-share] COMTRADE_API_KEYS not set — cannot fetch'); } - return { - manifestVersion: manifest.manifestVersion, - lastReviewed: manifest.lastReviewed, - externalReviewStatus: manifest.externalReviewStatus, + + const years = buildPeriodYears(); + const countries = {}; + + for (const { iso2, reporterCode } of REEXPORT_HUB_COHORT) { + const mResult = await fetchComtradeFlow(apiKey, reporterCode, 'M', years, { iso2 }); + await sleep(INTER_CALL_PACING_MS); + const rxResult = await fetchComtradeFlow(apiKey, reporterCode, 'RX', years, { iso2 }); + await sleep(INTER_CALL_PACING_MS); + + if (mResult.truncated || rxResult.truncated) { + console.warn(`[reexport-share] ${iso2}: skipping due to truncation`); + continue; + } + + const mByYear = parseComtradeFlowResponse(mResult.rows); + const rxByYear = parseComtradeFlowResponse(rxResult.rows); + const picked = computeShareFromFlows(rxByYear, mByYear); + if (!picked) { + console.warn(`[reexport-share] ${iso2}: no co-populated RX+M year in window ${years.join(',')}; omitting`); + continue; + } + + const clamped = clampShare(picked.share); + if (clamped == null) { + console.log(`[reexport-share] ${iso2}: raw share ${(picked.share * 100).toFixed(2)}% below floor (${MIN_MATERIAL_SHARE * 100}%) at Y=${picked.year}; omitting`); + continue; + } + + countries[iso2] = { + reexportShareOfImports: clamped, + year: picked.year, + reexportsUsd: picked.reexportsUsd, + grossImportsUsd: picked.importsUsd, + source: 'comtrade', + sources: [ + auditSafeSourceUrl(reporterCode, 'RX', years), + auditSafeSourceUrl(reporterCode, 'M', years), + ], + }; + console.log(`[reexport-share] ${iso2}: share=${(clamped * 100).toFixed(1)}% at Y=${picked.year} (RX $${(picked.reexportsUsd / 1e9).toFixed(1)}B / M $${(picked.importsUsd / 1e9).toFixed(1)}B)`); + } + + const payload = { + manifestVersion: 2, + lastReviewed: new Date().toISOString().slice(0, 10), + externalReviewStatus: 'REVIEWED', countries, seededAt: new Date().toISOString(), }; + + // Hard guarantee: no serialized field may contain the subscription- + // key query param. If any future refactor leaks it into the sources + // array or anywhere else in the envelope, fail the run loudly + // instead of publishing the credential. + const serialized = JSON.stringify(payload); + if (/subscription-key=/i.test(serialized)) { + throw new Error('[reexport-share] serialized payload contains subscription-key — refusing to publish'); + } + + return payload; } -// Manifest may legitimately be empty (this PR ships empty + infrastructure; -// follow-up PRs populate entries with citations). `validateFn` thus accepts -// both the empty and populated cases — the goal is schema soundness, not a -// minimum-coverage gate. The SWF seeder treats absence as "use gross -// imports" so an empty manifest is a safe no-op. function validate(data) { - return ( - data != null - && typeof data === 'object' - && typeof data.countries === 'object' - && data.countries !== null - && typeof data.manifestVersion === 'number' - ); + if (!data || typeof data !== 'object') return false; + if (data.manifestVersion !== 2) return false; + if (!data.countries || typeof data.countries !== 'object') return false; + return true; } export function declareRecords(data) { return Object.keys(data?.countries ?? {}).length; } -runSeed('resilience', 'recovery:reexport-share', CANONICAL_KEY, fetchReexportShare, { - validateFn: validate, - ttlSeconds: CACHE_TTL, - sourceVersion: 'unctad-manifest-v1', - declareRecords, - schemaVersion: 1, - // Note on empty-manifest behaviour. Our validate() returns true for - // an empty manifest ({countries: {}}) — the schema is sound, the - // content is just empty. runSeed therefore publishes the payload - // normally. We intentionally do NOT pass emptyDataIsFailure (which - // is strict-mode); an empty manifest on this PR's landing is the - // legitimate shape pending follow-up PRs that add entries with - // UNCTAD citations. - // - // Manifest cadence is weekly at most (bundle cron is weekly). Allow - // generous staleness before health flags it — a manifest that's a - // week old is perfectly fine because the underlying UNCTAD data - // revision cadence is ANNUAL. - maxStaleMin: 10080, -}).catch((err) => { - const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; - console.error('FATAL:', (err.message || err) + cause); - process.exit(1); -}); +// Guard top-level runSeed so the module can be imported by tests without +// triggering the full fetch/publish flow. Uses the canonical +// `pathToFileURL` comparison — unambiguous across path forms (symlink, +// case-different on macOS HFS+, Windows backslash vs slash) — rather +// than the basename-suffix matching pattern used by some older seeders. +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isMain) { + runSeed('resilience', 'recovery:reexport-share', CANONICAL_KEY, fetchReexportShare, { + validateFn: validate, + ttlSeconds: CACHE_TTL_SECONDS, + sourceVersion: 'comtrade-rx-m-ratio-v2', + declareRecords, + schemaVersion: 2, + // Empty-countries is ACCEPTABLE if every cohort member omits (Phase 0 + // may prune all; per-country floor may omit all). Downstream SWF + // seeder handles an empty map as "all gross imports". Not strict. + zeroIsValid: true, + maxStaleMin: 10080, + }).catch((err) => { + const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; + console.error('FATAL:', (err.message || err) + cause); + process.exit(1); + }); +} diff --git a/scripts/seed-sovereign-wealth.mjs b/scripts/seed-sovereign-wealth.mjs index 9e7b43acf..1042d363f 100644 --- a/scripts/seed-sovereign-wealth.mjs +++ b/scripts/seed-sovereign-wealth.mjs @@ -79,10 +79,79 @@ // §3.4 "What happens to no-SWF countries"). This is substantively // different from IMPUTE fallback (which is "data-source-failed"). -import { loadEnvFile, CHROME_UA, runSeed, SHARED_FX_FALLBACKS, getSharedFxRates } from './_seed-utils.mjs'; +import { loadEnvFile, CHROME_UA, runSeed, readSeedSnapshot, SHARED_FX_FALLBACKS, getSharedFxRates, getBundleRunStartedAtMs } from './_seed-utils.mjs'; import iso3ToIso2 from './shared/iso3-to-iso2.json' with { type: 'json' }; import { groupFundsByCountry, loadSwfManifest } from './shared/swf-manifest-loader.mjs'; -import { loadReexportShareByCountry } from './shared/reexport-share-loader.mjs'; + +const REEXPORT_SHARE_CANONICAL_KEY = 'resilience:recovery:reexport-share:v1'; +const REEXPORT_SHARE_META_KEY = 'seed-meta:resilience:recovery:reexport-share'; + +/** + * Read the Comtrade-seeded re-export-share map from Redis, guarded by + * bundle-run freshness. Returns an empty Map on any failure signal — + * missing key, malformed payload, or seed-meta older than this bundle + * run. The caller treats an empty map as "use gross imports for all + * countries" (status-quo fallback). + * + * Why bundle-run freshness matters: the Reexport-Share seeder runs + * immediately before this SWF seeder inside the resilience-recovery + * bundle. If that seeder fails (Comtrade outage, 429 storm, timeout), + * its Redis key still holds LAST MONTH's envelope — reading that + * would silently apply stale shares to the current month's SWF data. + * The bundle-freshness guard rejects any meta predating the current + * bundle run, forcing a hard fallback to gross imports. + * + * @returns {Promise>} + */ +export async function loadReexportShareFromRedis() { + const map = new Map(); + const raw = await readSeedSnapshot(REEXPORT_SHARE_CANONICAL_KEY); + if (!raw || typeof raw !== 'object') { + console.warn('[seed-sovereign-wealth] reexport-share Redis key empty/malformed; falling back to gross-imports denominator for all countries'); + return map; + } + + const metaRaw = await readSeedSnapshot(REEXPORT_SHARE_META_KEY); + const fetchedAtMs = Number(metaRaw?.fetchedAt ?? 0); + if (!fetchedAtMs) { + // Meta absent or malformed — can't tell whether the peer seeder ran. + // Safer to treat as outage than to trust the data key alone. + console.warn('[seed-sovereign-wealth] reexport-share seed-meta absent/malformed; falling back to gross-imports denominator for all countries'); + return map; + } + const bundleStartMs = getBundleRunStartedAtMs(); + // Freshness gate applies ONLY when spawned by _bundle-runner.mjs (i.e. + // `getBundleRunStartedAtMs()` returns a timestamp). Standalone runs + // (manual invocation, operator debugging) return null and skip the + // gate: the operator is responsible for running the peer seeder + // first, and we trust any `fetchedAt` in that context. The gate's + // purpose is protecting against across-bundle-tick staleness inside + // a cron run, which has no analog outside a bundle. + if (bundleStartMs != null && fetchedAtMs < bundleStartMs) { + const ageMin = ((Date.now() - fetchedAtMs) / 60_000).toFixed(0); + console.warn(`[seed-sovereign-wealth] reexport-share seed-meta NOT from this bundle run (age=${ageMin}min, bundleStart=${new Date(bundleStartMs).toISOString()}). Falling back to gross imports for all countries.`); + return map; + } + + const countries = raw.countries ?? {}; + for (const [iso2, entry] of Object.entries(countries)) { + const share = entry?.reexportShareOfImports; + // Numeric bounds check — NaN / Infinity / negative / ≥ 1 all pass + // `typeof === 'number'`. computeNetImports requires share ∈ [0, 1). + // The Comtrade seeder caps at 0.95 but this guard protects against + // a rogue payload (e.g. a manual redis-cli write mid-migration). + if (!Number.isFinite(share) || share < 0 || share > 0.95) { + console.warn(`[seed-sovereign-wealth] ${iso2} share ${share} fails bounds check [0, 0.95]; skipping`); + continue; + } + map.set(iso2, { + reexportShareOfImports: share, + year: entry?.year ?? null, + sources: Array.isArray(entry?.sources) ? entry.sources : [], + }); + } + return map; +} loadEnvFile(import.meta.url); @@ -653,15 +722,16 @@ export function computeNetImports(grossImportsUsd, reexportShareOfImports) { export async function fetchSovereignWealth() { const manifest = loadSwfManifest(); - // PR 3A §net-imports. Re-export share manifest: per-country fraction - // of gross imports that flow through as re-exports without settling - // as domestic consumption. Loaded from - // `scripts/shared/reexport-share-manifest.yaml`. Countries NOT in the - // manifest get `netImports = grossImports` (status-quo behaviour) — - // absence MUST NOT throw or zero the denominator. Load is local - // (YAML parse), not a Redis read: the manifest is code-adjacent and - // always available in the seeder's working directory. - const reexportShareByCountry = loadReexportShareByCountry(); + // Re-export share: per-country fraction of gross imports that flow + // through as re-exports without settling as domestic consumption. + // Sourced from Comtrade via the sibling Reexport-Share seeder that + // runs immediately before this one inside the resilience-recovery + // bundle. loadReexportShareFromRedis() enforces bundle-run freshness + // — if the sibling's seed-meta predates this bundle's start, all + // countries fall back to gross imports (hard fail-safe). Countries + // not in the returned map get netImports = grossImports (status-quo + // behaviour). Absence MUST NOT throw or zero the denominator. + const reexportShareByCountry = await loadReexportShareFromRedis(); const [imports, wikipediaCache, fxRates] = await Promise.all([ fetchAnnualImportsUsd(), loadWikipediaRankingsCache(), diff --git a/scripts/shared/reexport-share-loader.mjs b/scripts/shared/reexport-share-loader.mjs deleted file mode 100644 index 07fdf77f3..000000000 --- a/scripts/shared/reexport-share-loader.mjs +++ /dev/null @@ -1,166 +0,0 @@ -// Loader + validator for the re-export share manifest at -// scripts/shared/reexport-share-manifest.yaml. -// -// Mirrors the swf-manifest-loader.mjs pattern: -// - Co-located with the YAML so the Railway recovery-bundle container -// (rootDirectory=scripts/) ships both together under a single COPY. -// - Pure JS (no Redis, no env mutations) so the SWF seeder can import -// it at top-level without touching the I/O layer. -// - Strict schema validation at load time so a malformed manifest -// fails the seeder cold, not silently. -// -// See plan `docs/plans/2026-04-24-002-fix-resilience-cohort-ranking- -// structural-audit-plan.md` §PR 3A for the construct rationale. - -import { readFileSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import { dirname, resolve } from 'node:path'; -import { parse as parseYaml } from 'yaml'; - -const here = dirname(fileURLToPath(import.meta.url)); -const MANIFEST_PATH = resolve(here, './reexport-share-manifest.yaml'); - -/** - * @typedef {Object} ReexportShareEntry - * @property {string} country ISO-3166-1 alpha-2 - * @property {number} reexportShareOfImports 0..1 inclusive - * @property {number} year reference year (e.g. 2023) - * @property {string} rationale one-line summary of the cited source - * @property {string[]} sources list of URLs / citations - */ - -/** - * @typedef {Object} ReexportShareManifest - * @property {number} manifestVersion - * @property {string} lastReviewed - * @property {'PENDING'|'REVIEWED'} externalReviewStatus - * @property {ReexportShareEntry[]} countries - */ - -function fail(msg) { - throw new Error(`[reexport-manifest] ${msg}`); -} - -function assertZeroToOne(value, path) { - if (typeof value !== 'number' || Number.isNaN(value) || value < 0 || value > 1) { - fail(`${path}: expected number in [0, 1], got ${JSON.stringify(value)}`); - } -} - -function assertIso2(value, path) { - if (typeof value !== 'string' || !/^[A-Z]{2}$/.test(value)) { - fail(`${path}: expected ISO-3166-1 alpha-2 country code, got ${JSON.stringify(value)}`); - } -} - -function assertNonEmptyString(value, path) { - if (typeof value !== 'string' || value.trim().length === 0) { - fail(`${path}: expected non-empty string, got ${JSON.stringify(value)}`); - } -} - -function assertYear(value, path) { - if (typeof value !== 'number' || !Number.isInteger(value) || value < 2000 || value > 2100) { - fail(`${path}: expected integer year in [2000, 2100], got ${JSON.stringify(value)}`); - } -} - -function validateSources(sources, path) { - if (!Array.isArray(sources) || sources.length === 0) { - fail(`${path}: expected non-empty array`); - } - for (const [srcIdx, src] of sources.entries()) { - assertNonEmptyString(src, `${path}[${srcIdx}]`); - } - return sources.slice(); -} - -function validateCountryEntry(raw, idx, seenCountries) { - const path = `countries[${idx}]`; - if (!raw || typeof raw !== 'object') fail(`${path}: expected object`); - const c = /** @type {Record} */ (raw); - - assertIso2(c.country, `${path}.country`); - assertZeroToOne(c.reexport_share_of_imports, `${path}.reexport_share_of_imports`); - assertYear(c.year, `${path}.year`); - assertNonEmptyString(c.rationale, `${path}.rationale`); - const sources = validateSources(c.sources, `${path}.sources`); - - const countryCode = /** @type {string} */ (c.country); - if (seenCountries.has(countryCode)) { - fail(`${path}.country: duplicate entry for ${countryCode}`); - } - seenCountries.add(countryCode); - - return { - country: countryCode, - reexportShareOfImports: /** @type {number} */ (c.reexport_share_of_imports), - year: /** @type {number} */ (c.year), - rationale: /** @type {string} */ (c.rationale), - sources, - }; -} - -/** - * Load and validate the re-export share manifest. - * Throws with a detailed path-prefixed error on schema violation; a - * broken manifest MUST fail the seeder cold — silently proceeding with - * a partial read would leave some countries' net-imports denominator - * wrong without signal. - * - * @returns {ReexportShareManifest} - */ -export function loadReexportShareManifest() { - const raw = readFileSync(MANIFEST_PATH, 'utf8'); - const doc = parseYaml(raw); - if (!doc || typeof doc !== 'object') { - fail(`root: expected object, got ${typeof doc}`); - } - - const version = doc.manifest_version; - if (version !== 1) fail(`manifest_version: expected 1, got ${JSON.stringify(version)}`); - - const lastReviewed = doc.last_reviewed; - if (typeof lastReviewed !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(lastReviewed)) { - fail(`last_reviewed: expected YYYY-MM-DD, got ${JSON.stringify(lastReviewed)}`); - } - - const status = doc.external_review_status; - if (status !== 'PENDING' && status !== 'REVIEWED') { - fail(`external_review_status: expected 'PENDING'|'REVIEWED', got ${JSON.stringify(status)}`); - } - - const rawCountries = doc.countries; - if (!Array.isArray(rawCountries)) { - fail(`countries: expected array, got ${typeof rawCountries}`); - } - const seen = new Set(); - const countries = rawCountries.map((r, i) => validateCountryEntry(r, i, seen)); - - return { - manifestVersion: 1, - lastReviewed, - externalReviewStatus: /** @type {'PENDING'|'REVIEWED'} */ (status), - countries, - }; -} - -/** - * Read the manifest and return an ISO2 → reexportShareOfImports lookup. - * Countries missing from the manifest return undefined — the SWF seeder - * MUST treat undefined as "no adjustment, use gross imports." - * - * @returns {Map} - */ -export function loadReexportShareByCountry() { - const manifest = loadReexportShareManifest(); - const map = new Map(); - for (const entry of manifest.countries) { - map.set(entry.country, { - reexportShareOfImports: entry.reexportShareOfImports, - year: entry.year, - sources: entry.sources, - }); - } - return map; -} diff --git a/scripts/shared/reexport-share-manifest.yaml b/scripts/shared/reexport-share-manifest.yaml deleted file mode 100644 index 0810773d0..000000000 --- a/scripts/shared/reexport-share-manifest.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# Re-export share manifest -# ========================= -# -# Per-country published re-export share — used by the SWF seeder -# (`scripts/seed-sovereign-wealth.mjs`) to convert GROSS annual imports -# into NET annual imports when computing `rawMonths = SWF / imports × 12` -# for the `sovereignFiscalBuffer` dimension. See plan reference at -# `docs/plans/2026-04-24-002-fix-resilience-cohort-ranking-structural-audit-plan.md` -# §PR 3A. -# -# Why this matters. For countries that function as re-export hubs (goods -# imported for onward re-shipment without substantial transformation), -# the "months of imports" denominator is structurally wrong: the gross -# import figure double-counts flow-through trade that never represents -# domestic consumption. The correction: -# -# netAnnualImports = grossAnnualImports × (1 − reexport_share_of_imports) -# -# where `reexport_share_of_imports` is the UNCTAD-published fraction of -# that country's merchandise imports re-exported without transformation. -# -# Why manifest + citation instead of a live API. UNCTAD's Handbook of -# Statistics publishes re-export aggregates annually as PDF/Excel; there -# is no stable SDMX series for this specific derivative. National stats -# offices publish the underlying data with varying schemas. Manifest- -# with-citation mirrors the `scripts/shared/swf-classification-manifest.yaml` -# pattern: each entry is a committed audit trail (source URL, table -# reference, year) that any reader can verify. -# -# Revision policy. Update when UNCTAD releases a new Handbook (annual) -# OR a national stats office publishes a materially different figure. -# Every revision: coefficient change + source citation update in the -# same PR. -# -# Schema (per entry): -# country: ISO-3166-1 alpha-2 country code -# reexport_share_of_imports: 0..1 — fraction of gross imports that -# flow through as re-exports -# year: reference year of the cited figure -# rationale: what the cited source says, in one line -# sources: list of URLs / citations -# -# Scoring impact (applied in `scripts/seed-sovereign-wealth.mjs`): -# When a country entry is PRESENT: -# netAnnualImports = grossAnnualImports × (1 − reexport_share_of_imports) -# rawMonths = aum / netAnnualImports × 12 -# When a country entry is ABSENT: -# rawMonths = aum / grossAnnualImports × 12 (status quo behaviour) -# -# Fallback semantics are CRITICAL: absence MUST NOT throw or produce a -# silent zero. Absent = "no published UNCTAD figure, use gross imports -# as a conservative approximation." The SWF seeder's per-country loop -# must tolerate a missing entry. -# -# Candidates deferred for follow-up verification (NOT in this manifest): -# -# HK (Hong Kong SAR): well-documented re-export-dominated trade. -# Hong Kong C&SD publishes a clean domestic-export -# vs re-export split. Likely 0.80-0.95 after the -# imports/exports parity adjustment. Verify -# against HK C&SD monthly trade release. -# SG (Singapore): ~45-50% of exports are re-exports (Singapore -# Statistics). Convert to imports-share via -# imports ≈ exports steady-state assumption. -# NL (Netherlands): Rotterdam + Europoort transhipment hub. -# CBS publishes a clean re-export series. -# BE (Belgium): Antwerp + Zeebrugge. Modest re-export share. -# PA (Panama): Colón Free Zone. High share. -# AE (United Arab Emirates): Dubai Jebel Ali + Sharjah. The plan's -# primary motivating case — this correction is -# exactly what unblocks UAE's sovereignFiscalBuffer -# denominator. Verify against UAE Federal -# Competitiveness and Statistics Authority. -# MY (Malaysia): Port Klang transhipment. Moderate. -# LT (Lithuania): EU-to-CIS transit, pre-2022. Post-2022 sanctions -# have reshaped this — figure needs current-year -# verification. -# -# Each deferred candidate gets its own manifest-edit PR with: -# (a) specific UNCTAD / national-stats source URL -# (b) table / section number + year -# (c) the exact value copied from the source, not interpolated -# (d) re-run of the cohort-sanity audit -# (e) contribution-decomposition table showing the effMo shift - -manifest_version: 1 -last_reviewed: 2026-04-24 -# REVIEWED means: the schema is committed and the loader validates the -# file. Entries themselves are added country-by-country in follow-up -# PRs per the "one entry per PR with citation" discipline. -external_review_status: REVIEWED - -# Countries with UNCTAD-verified re-export shares. Intentionally empty -# in this landing PR — populated entry-by-entry in follow-up PRs with -# explicit source citations. Empty + loader-validated is the safe -# default: the SWF seeder falls back to gross imports for every country -# until a specific entry lands. -countries: [] diff --git a/tests/reexport-share-loader.test.mts b/tests/reexport-share-loader.test.mts deleted file mode 100644 index b9b95c06f..000000000 --- a/tests/reexport-share-loader.test.mts +++ /dev/null @@ -1,178 +0,0 @@ -// Schema-validation tests for the re-export share manifest loader -// (`scripts/shared/reexport-share-loader.mjs`). Mirrors the validation -// discipline applied to scripts/shared/swf-manifest-loader.mjs. -// -// The loader MUST fail-closed on any schema violation: a malformed -// manifest propagates as a silent zero denominator via the SWF seeder -// and poisons every re-export-hub country's sovereignFiscalBuffer -// score. Strict validation at load time catches the drift before it -// reaches Redis. - -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { writeFileSync, unlinkSync, existsSync } from 'node:fs'; -import { join } from 'node:path'; -import os from 'node:os'; -import { fileURLToPath } from 'node:url'; - -import { loadReexportShareManifest } from '../scripts/shared/reexport-share-loader.mjs'; - -describe('reexport-share manifest loader — committed manifest shape', () => { - it('loads the repo-committed manifest without error (empty countries array is valid)', () => { - const manifest = loadReexportShareManifest(); - assert.equal(manifest.manifestVersion, 1); - assert.ok(/^\d{4}-\d{2}-\d{2}$/.test(manifest.lastReviewed)); - assert.ok(manifest.externalReviewStatus === 'REVIEWED' || manifest.externalReviewStatus === 'PENDING'); - assert.ok(Array.isArray(manifest.countries)); - }); -}); - -// Build a temp manifest file + a local loader for schema-violation -// tests. We cannot use the shared loader directly because it reads the -// repo-committed path. Instead we call the same validator functions -// via a re-import against a synthetic file. -function writeTempManifest(content: string): string { - const tmp = join(os.tmpdir(), `reexport-test-${process.pid}-${Date.now()}.yaml`); - writeFileSync(tmp, content); - return tmp; -} - -// Reuse the production loader by pointing at a different file via -// dynamic import + readFileSync path override. Since the loader has a -// hardcoded path, we invoke the schema validation indirectly through -// writeTempManifest + a small local clone that mirrors the schema -// checks. This keeps the schema-violation tests hermetic while -// preserving the invariant that the validator is the single source of -// truth. Below is a minimal re-implementation of the validator that -// the production loader uses — any divergence in validation logic -// will break this test first. -async function loadManifestFromPath(path: string) { - // Fresh import each call avoids any module-level caching. - const { readFileSync: rfs } = await import('node:fs'); - const { parse: parseYaml } = await import('yaml'); - const raw = rfs(path, 'utf8'); - const doc = parseYaml(raw); - // Validate — mirror the production validator's sequence so test - // failures point at the same rules the production loader enforces. - if (!doc || typeof doc !== 'object') throw new Error('root: expected object'); - if (doc.manifest_version !== 1) throw new Error(`manifest_version: expected 1, got ${JSON.stringify(doc.manifest_version)}`); - if (typeof doc.last_reviewed !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(doc.last_reviewed)) { - throw new Error(`last_reviewed: expected YYYY-MM-DD, got ${JSON.stringify(doc.last_reviewed)}`); - } - if (doc.external_review_status !== 'PENDING' && doc.external_review_status !== 'REVIEWED') { - throw new Error(`external_review_status: expected 'PENDING'|'REVIEWED', got ${JSON.stringify(doc.external_review_status)}`); - } - if (!Array.isArray(doc.countries)) throw new Error('countries: expected array'); - const seen = new Set(); - for (const [i, entry] of doc.countries.entries()) { - if (!entry || typeof entry !== 'object') throw new Error(`countries[${i}]: expected object`); - if (!/^[A-Z]{2}$/.test(String(entry.country ?? ''))) { - throw new Error(`countries[${i}].country: expected ISO-3166-1 alpha-2`); - } - const share = entry.reexport_share_of_imports; - if (typeof share !== 'number' || Number.isNaN(share) || share < 0 || share > 1) { - throw new Error(`countries[${i}].reexport_share_of_imports: expected number in [0, 1]`); - } - if (seen.has(entry.country)) throw new Error(`countries[${i}].country: duplicate entry`); - seen.add(entry.country); - } - return doc; -} - -describe('reexport-share manifest loader — schema violations fail-closed', () => { - const cleanup: string[] = []; - after(() => { - for (const p of cleanup) if (existsSync(p)) unlinkSync(p); - }); - - function temp(content: string) { - const path = writeTempManifest(content); - cleanup.push(path); - return path; - } - - it('rejects share > 1', async () => { - const path = temp(`manifest_version: 1 -last_reviewed: 2026-04-24 -external_review_status: REVIEWED -countries: - - country: XX - reexport_share_of_imports: 1.5 - year: 2023 - rationale: test - sources: - - https://example.org -`); - await assert.rejects(loadManifestFromPath(path), /reexport_share_of_imports: expected number in \[0, 1\]/); - }); - - it('rejects negative share', async () => { - const path = temp(`manifest_version: 1 -last_reviewed: 2026-04-24 -external_review_status: REVIEWED -countries: - - country: XX - reexport_share_of_imports: -0.1 - year: 2023 - rationale: test - sources: ['https://example.org'] -`); - await assert.rejects(loadManifestFromPath(path), /reexport_share_of_imports: expected number in \[0, 1\]/); - }); - - it('rejects invalid ISO-2 country code', async () => { - const path = temp(`manifest_version: 1 -last_reviewed: 2026-04-24 -external_review_status: REVIEWED -countries: - - country: USA - reexport_share_of_imports: 0.2 - year: 2023 - rationale: test - sources: ['https://example.org'] -`); - await assert.rejects(loadManifestFromPath(path), /country: expected ISO-3166-1 alpha-2/); - }); - - it('rejects duplicate country entries', async () => { - const path = temp(`manifest_version: 1 -last_reviewed: 2026-04-24 -external_review_status: REVIEWED -countries: - - country: SG - reexport_share_of_imports: 0.4 - year: 2023 - rationale: first - sources: ['https://example.org'] - - country: SG - reexport_share_of_imports: 0.5 - year: 2023 - rationale: second - sources: ['https://example.org'] -`); - await assert.rejects(loadManifestFromPath(path), /duplicate entry/); - }); - - it('rejects bad manifest_version', async () => { - const path = temp(`manifest_version: 99 -last_reviewed: 2026-04-24 -external_review_status: REVIEWED -countries: [] -`); - await assert.rejects(loadManifestFromPath(path), /manifest_version: expected 1/); - }); - - it('rejects malformed last_reviewed', async () => { - const path = temp(`manifest_version: 1 -last_reviewed: not-a-date -external_review_status: REVIEWED -countries: [] -`); - await assert.rejects(loadManifestFromPath(path), /last_reviewed: expected YYYY-MM-DD/); - }); -}); - -// Minimal after() helper compatible with node:test harness. -function after(fn: () => void) { - process.on('exit', fn); -} diff --git a/tests/resilience-cache-keys-health-sync.test.mts b/tests/resilience-cache-keys-health-sync.test.mts index 0f2bbe80e..0ff12e5ee 100644 --- a/tests/resilience-cache-keys-health-sync.test.mts +++ b/tests/resilience-cache-keys-health-sync.test.mts @@ -138,4 +138,39 @@ describe('resilience cache-key health-registry sync (T1.9)', () => { } }); }); + + // Plan 2026-04-24-003 dual-registry drift guard. `api/health.js` and + // `api/seed-health.js` maintain INDEPENDENT registries (see + // `feedback_two_health_endpoints_must_match`). They are NOT globally + // identical — health.js watches keys seed-health.js doesn't, and vice + // versa. Only the keys explicitly added by this PR are required in + // BOTH registries; pre-existing recovery entries (fiscal-space, + // reserve-adequacy, external-debt, import-hhi, fuel-stocks) live only + // in api/health.js by design and are NOT asserted here. + describe('resilience-recovery dual-registry parity (this PR only)', () => { + const SHARED_RESILIENCE_KEYS = [ + 'resilience:recovery:reexport-share', + 'resilience:recovery:sovereign-wealth', + ] as const; + + const healthJsText = readFileSync(join(repoRoot, 'api/health.js'), 'utf-8'); + const seedHealthJsText = readFileSync(join(repoRoot, 'api/seed-health.js'), 'utf-8'); + + for (const key of SHARED_RESILIENCE_KEYS) { + it(`'${key}' is registered in api/health.js SEED_META`, () => { + const metaKey = `seed-meta:${key}`; + assert.ok( + healthJsText.includes(`'${metaKey}'`) || healthJsText.includes(`"${metaKey}"`), + `api/health.js must register '${metaKey}' in SEED_META`, + ); + }); + + it(`'${key}' is registered in api/seed-health.js SEED_DOMAINS`, () => { + assert.ok( + seedHealthJsText.includes(`'${key}'`) || seedHealthJsText.includes(`"${key}"`), + `api/seed-health.js must register '${key}' in SEED_DOMAINS`, + ); + }); + } + }); }); diff --git a/tests/seed-bundle-resilience-recovery.test.mjs b/tests/seed-bundle-resilience-recovery.test.mjs index 43ca5e941..696838494 100644 --- a/tests/seed-bundle-resilience-recovery.test.mjs +++ b/tests/seed-bundle-resilience-recovery.test.mjs @@ -55,4 +55,28 @@ describe('seed-bundle-resilience-recovery', () => { assert.ok(bundleSource.includes('runBundle'), 'Missing runBundle import'); assert.ok(bundleSource.includes('DAY'), 'Missing DAY import'); }); + + // Plan 2026-04-24-003: Reexport-Share became Comtrade-backed; 60s + // timeout is no longer enough. Guard against a revert / accidental + // restore of the pre-Comtrade timeout. + it('Reexport-Share entry has timeoutMs >= 180_000', () => { + // Match only the Reexport-Share entry's object body, not the full + // file, to avoid cross-entry timeout leakage. + const entryMatch = bundleSource.match(/\{[^}]*label:\s*'Reexport-Share'[^}]*\}/); + assert.ok(entryMatch, 'Could not locate Reexport-Share entry'); + const timeoutMatch = entryMatch[0].match(/timeoutMs:\s*([\d_]+)/); + assert.ok(timeoutMatch, 'Reexport-Share entry missing timeoutMs'); + const timeoutMs = Number(timeoutMatch[1].replace(/_/g, '')); + assert.ok(timeoutMs >= 180_000, + `Reexport-Share timeoutMs must be >= 180_000 (Comtrade + retry can take 2-3min); got ${timeoutMs}`); + }); + + it('Reexport-Share runs BEFORE Sovereign-Wealth in bundle ordering', () => { + const reexportIdx = bundleSource.indexOf("label: 'Reexport-Share'"); + const swfIdx = bundleSource.indexOf("label: 'Sovereign-Wealth'"); + assert.ok(reexportIdx >= 0, 'Reexport-Share not in bundle'); + assert.ok(swfIdx >= 0, 'Sovereign-Wealth not in bundle'); + assert.ok(reexportIdx < swfIdx, + `Reexport-Share must run before Sovereign-Wealth (so SWF seeder reads a freshly-written re-export share key)`); + }); }); diff --git a/tests/seed-recovery-reexport-share-comtrade.test.mts b/tests/seed-recovery-reexport-share-comtrade.test.mts new file mode 100644 index 000000000..0913ee615 --- /dev/null +++ b/tests/seed-recovery-reexport-share-comtrade.test.mts @@ -0,0 +1,183 @@ +// Pure-math tests for the Comtrade-backed re-export-share seeder. +// Verifies the three extracted helpers (`parseComtradeFlowResponse`, +// `computeShareFromFlows`, `clampShare`) behave correctly in isolation, +// and that no subscription-key query param ever appears in the +// serialized envelope (belt-and-suspenders even with header auth). +// +// Context: plan 2026-04-24-003 §Phase 3 tests 1-6. These replace the +// 7 obsolete reexport-share-loader tests (YAML flattener deleted in +// this same PR). + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + clampShare, + computeShareFromFlows, + declareRecords, + parseComtradeFlowResponse, +} from '../scripts/seed-recovery-reexport-share.mjs'; + +describe('parseComtradeFlowResponse', () => { + it('sums primaryValue per year, skipping zero/negative/non-numeric', () => { + const rows = [ + { period: 2023, primaryValue: 100_000 }, + { period: 2023, primaryValue: 50_000 }, + { period: 2022, primaryValue: 30_000 }, + { period: 2021, primaryValue: 0 }, // skipped (zero) + { period: 2021, primaryValue: -5 }, // skipped (negative) + { period: 2021, primaryValue: 'x' }, // skipped (non-numeric) + ]; + const out = parseComtradeFlowResponse(rows); + assert.equal(out.get(2023), 150_000); + assert.equal(out.get(2022), 30_000); + assert.equal(out.has(2021), false); + }); + + it('sums ONLY world-aggregate rows (partnerCode=0), excludes partner-level rows', () => { + // Defensive filter: if Comtrade returns BOTH a world-aggregate + // row (partner=0) AND per-partner breakdown rows for the same + // year, summing all would silently double-count and cut any + // derived share in half. We sum only partnerCode='0' / 0 / null. + const rows = [ + { period: 2023, partnerCode: '0', primaryValue: 1_000_000 }, // world aggregate — include + { period: 2023, partnerCode: '842', primaryValue: 200_000 }, // per-partner (US) — EXCLUDE + { period: 2023, partnerCode: '826', primaryValue: 150_000 }, // per-partner (UK) — EXCLUDE + ]; + const out = parseComtradeFlowResponse(rows); + assert.equal(out.get(2023), 1_000_000, + 'per-partner rows must not add to the world-aggregate total'); + }); + + it('accepts numeric 0 partnerCode (shape variant)', () => { + // Comtrade has occasionally emitted numeric 0 vs string '0' depending + // on response shape; both must be treated as world-aggregate. + const rows = [ + { period: 2023, partnerCode: 0, primaryValue: 500 }, + { period: 2023, partnerCode: '0', primaryValue: 500 }, + ]; + const out = parseComtradeFlowResponse(rows); + assert.equal(out.get(2023), 1_000); + }); + + it('accepts rows with no partnerCode field (older response shape)', () => { + // Defensive: if a response shape omits partnerCode entirely, + // treat the row as world-aggregate rather than silently dropping it. + const rows = [{ period: 2024, primaryValue: 42 }]; + const out = parseComtradeFlowResponse(rows); + assert.equal(out.get(2024), 42); + }); + + it('handles refPeriodId fallback when period is absent', () => { + const rows = [{ refPeriodId: 2024, primaryValue: 42 }]; + const out = parseComtradeFlowResponse(rows); + assert.equal(out.get(2024), 42); + }); + + it('returns empty map on empty input', () => { + assert.equal(parseComtradeFlowResponse([]).size, 0); + }); +}); + +describe('computeShareFromFlows', () => { + it('picks the latest co-populated year and returns share = RX / M', () => { + const rx = new Map([[2023, 300], [2022, 200], [2021, 100]]); + const m = new Map([[2023, 1000], [2022, 500], [2021, 400]]); + const picked = computeShareFromFlows(rx, m); + assert.equal(picked?.year, 2023); + assert.equal(picked?.share, 0.3); + assert.equal(picked?.reexportsUsd, 300); + assert.equal(picked?.importsUsd, 1000); + }); + + it('ignores years where RX or M is missing', () => { + const rx = new Map([[2024, 500], [2022, 200]]); // 2024 is RX-only + const m = new Map([[2023, 1000], [2022, 500]]); // 2023 is M-only + const picked = computeShareFromFlows(rx, m); + // Only 2022 is co-populated; even though 2024 is newer, it's not in M. + assert.equal(picked?.year, 2022); + assert.equal(picked?.share, 0.4); + }); + + it('returns null when no year is co-populated', () => { + const rx = new Map([[2024, 500]]); + const m = new Map([[2022, 500]]); + assert.equal(computeShareFromFlows(rx, m), null); + }); + + it('returns null when imports at picked year is zero (guards division)', () => { + // This can only happen if parseComtradeFlowResponse changes behavior; + // test the branch anyway since computeShareFromFlows is exported for + // tests and could be called with hand-crafted maps. + const rx = new Map([[2023, 300]]); + const m = new Map([[2023, 0]]); + assert.equal(computeShareFromFlows(rx, m), null); + }); +}); + +describe('clampShare', () => { + it('returns null for sub-floor shares (< 0.05)', () => { + assert.equal(clampShare(0.03), null); + assert.equal(clampShare(0.049999), null); + assert.equal(clampShare(0), null); + }); + + it('caps above-ceiling shares at 0.95 (< 1 guard for computeNetImports)', () => { + assert.equal(clampShare(1.2), 0.95); + assert.equal(clampShare(0.99), 0.95); + assert.equal(clampShare(0.951), 0.95); + }); + + it('passes through in-range shares unchanged', () => { + assert.equal(clampShare(0.05), 0.05); + assert.equal(clampShare(0.355), 0.355); + assert.equal(clampShare(0.5), 0.5); + assert.equal(clampShare(0.95), 0.95); + }); + + it('returns null for NaN, Infinity, and negative', () => { + assert.equal(clampShare(NaN), null); + assert.equal(clampShare(Infinity), null); + assert.equal(clampShare(-0.1), null); + }); +}); + +describe('declareRecords', () => { + it('counts material entries in the published payload', () => { + const payload = { countries: { AE: {}, PA: {} } }; + assert.equal(declareRecords(payload), 2); + }); + + it('returns 0 for empty countries map (valid zero state)', () => { + assert.equal(declareRecords({ countries: {} }), 0); + assert.equal(declareRecords(null), 0); + assert.equal(declareRecords({}), 0); + }); +}); + +describe('credential-leak regression guard', () => { + it('module source must not embed subscription-key in any URL literal', async () => { + // Read the seeder source file and assert no literal `subscription-key=` + // appears anywhere. Belt-and-suspenders even though fetchComtradeFlow + // uses header auth — if any future refactor adds `subscription-key=` + // to a URL builder, this test fails before it leaks to prod Redis. + const { readFile } = await import('node:fs/promises'); + const { fileURLToPath } = await import('node:url'); + const here = fileURLToPath(import.meta.url); + const seederPath = here.replace(/\/tests\/.*$/, '/scripts/seed-recovery-reexport-share.mjs'); + const src = await readFile(seederPath, 'utf8'); + // Flag only string-literal embeddings inside '...', "...", or `...`; + // regex literals (/subscription-key=/i used by the defensive serialize + // check) are intentional safeguards, not leaks. + // [^'\n] variant prevents the regex from spanning across multiple + // lines, which would falsely match any two unrelated quotes that + // happen to sandwich a `subscription-key=` reference elsewhere. + const stringLitMatches = [ + ...src.matchAll(/'[^'\n]*subscription-key=[^'\n]*'/g), + ...src.matchAll(/"[^"\n]*subscription-key=[^"\n]*"/g), + ...src.matchAll(/`[^`\n]*subscription-key=[^`\n]*`/g), + ]; + assert.equal(stringLitMatches.length, 0, + `found hardcoded subscription-key in string literal: ${stringLitMatches.map(m => m[0]).join(', ')}`); + }); +}); diff --git a/tests/seed-sovereign-wealth-reads-redis-reexport-share.test.mts b/tests/seed-sovereign-wealth-reads-redis-reexport-share.test.mts new file mode 100644 index 000000000..e26a9bf6e --- /dev/null +++ b/tests/seed-sovereign-wealth-reads-redis-reexport-share.test.mts @@ -0,0 +1,205 @@ +// Regression guards for Gap #2: the SWF seeder MUST read the re-export +// share map from Redis (populated by the Comtrade seeder that runs +// immediately before it in the resilience-recovery bundle), NOT from +// the static YAML that was deleted in this PR. +// +// These four tests defend against the exact failure mode that surfaced +// in the 2026-04-24 cohort audit: SWF scores didn't move after the +// Comtrade work shipped because the SWF seeder was still reading a +// (now-absent) YAML. See plan 2026-04-24-003 §Phase 3 tests 7-10. + +import assert from 'node:assert/strict'; +import { beforeEach, afterEach, describe, it } from 'node:test'; + +import { loadReexportShareFromRedis } from '../scripts/seed-sovereign-wealth.mjs'; + +const REEXPORT_SHARE_KEY = 'resilience:recovery:reexport-share:v1'; +const REEXPORT_SHARE_META_KEY = 'seed-meta:resilience:recovery:reexport-share'; + +const ORIGINAL_FETCH = globalThis.fetch; +const ORIGINAL_ENV = { + UPSTASH_REDIS_REST_URL: process.env.UPSTASH_REDIS_REST_URL, + UPSTASH_REDIS_REST_TOKEN: process.env.UPSTASH_REDIS_REST_TOKEN, + BUNDLE_RUN_STARTED_AT_MS: process.env.BUNDLE_RUN_STARTED_AT_MS, +}; + +let keyStore: Record; + +beforeEach(() => { + process.env.UPSTASH_REDIS_REST_URL = 'https://fake-upstash.example.com'; + process.env.UPSTASH_REDIS_REST_TOKEN = 'fake-token'; + keyStore = {}; + + // readSeedSnapshot issues `GET /get/`. + // Stub: look up keyStore, return `{ result: JSON.stringify(value) }` + // or `{ result: null }` for absent keys. + globalThis.fetch = async (url) => { + const s = String(url); + const match = s.match(/\/get\/(.+)$/); + if (!match) { + return new Response(JSON.stringify({ result: null }), { status: 200 }); + } + const key = decodeURIComponent(match[1]); + const value = keyStore[key]; + const body = value !== undefined + ? JSON.stringify({ result: JSON.stringify(value) }) + : JSON.stringify({ result: null }); + return new Response(body, { status: 200 }); + }; +}); + +afterEach(() => { + globalThis.fetch = ORIGINAL_FETCH; + for (const k of Object.keys(ORIGINAL_ENV) as Array) { + const v = ORIGINAL_ENV[k]; + if (v == null) delete process.env[k]; + else process.env[k] = v; + } +}); + +describe('loadReexportShareFromRedis — Gap #2 regression guards', () => { + it('reads the Redis key and returns a Map of ISO2 → share when bundle-fresh', async () => { + const bundleStart = 1_700_000_000_000; + process.env.BUNDLE_RUN_STARTED_AT_MS = String(bundleStart); + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { + AE: { reexportShareOfImports: 0.4, year: 2023, sources: ['https://comtrade.example/AE'] }, + PA: { reexportShareOfImports: 0.07, year: 2024, sources: ['https://comtrade.example/PA'] }, + }, + }; + keyStore[REEXPORT_SHARE_META_KEY] = { + fetchedAt: bundleStart + 1000, // 1s AFTER bundle start — fresh + }; + + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 2); + assert.equal(map.get('AE')?.reexportShareOfImports, 0.4); + assert.equal(map.get('PA')?.reexportShareOfImports, 0.07); + assert.equal(map.get('AE')?.year, 2023); + assert.deepEqual(map.get('AE')?.sources, ['https://comtrade.example/AE']); + }); + + it('absent canonical key → empty map (status-quo gross-imports fallback)', async () => { + process.env.BUNDLE_RUN_STARTED_AT_MS = String(1_700_000_000_000); + // keyStore is empty — readSeedSnapshot returns null. + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 0); + }); + + it('malformed entry (share is string) → skip that country, others unaffected', async () => { + const bundleStart = 1_700_000_000_000; + process.env.BUNDLE_RUN_STARTED_AT_MS = String(bundleStart); + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { + AE: { reexportShareOfImports: 0.4, year: 2023 }, + XX: { reexportShareOfImports: 'not-a-number' }, // type-wrong + YY: { reexportShareOfImports: 1.5 }, // > 0.95 cap + ZZ: { reexportShareOfImports: -0.1 }, // negative + AA: { reexportShareOfImports: NaN }, // NaN + }, + }; + keyStore[REEXPORT_SHARE_META_KEY] = { fetchedAt: bundleStart + 1000 }; + + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 1); + assert.equal(map.get('AE')?.reexportShareOfImports, 0.4); + assert.equal(map.has('XX'), false); + assert.equal(map.has('YY'), false); + assert.equal(map.has('ZZ'), false); + assert.equal(map.has('AA'), false); + }); + + it('stale seed-meta (fetchedAt < bundle start) → empty map (hard fail-safe)', async () => { + const bundleStart = 1_700_000_000_000; + process.env.BUNDLE_RUN_STARTED_AT_MS = String(bundleStart); + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { + AE: { reexportShareOfImports: 0.4, year: 2023 }, + }, + }; + // fetchedAt is 1 hour BEFORE bundle start — previous bundle tick. + // The SWF seeder MUST NOT apply last-month's share to this month's + // data. Hard fallback: return empty, everyone uses gross imports. + keyStore[REEXPORT_SHARE_META_KEY] = { fetchedAt: bundleStart - 3_600_000 }; + + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 0, + 'stale seed-meta must produce empty map, NOT pass stale shares through'); + }); + + it('missing seed-meta key → empty map (outage fail-safe)', async () => { + const bundleStart = 1_700_000_000_000; + process.env.BUNDLE_RUN_STARTED_AT_MS = String(bundleStart); + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { AE: { reexportShareOfImports: 0.4, year: 2023 } }, + }; + // Meta is absent — seeder produced a data envelope but seed-meta + // write failed or races. Safer to treat as "did not run this + // bundle" than to trust the data-key alone. + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 0); + }); + + it('standalone mode (BUNDLE_RUN_STARTED_AT_MS unset) skips the freshness gate', async () => { + // Regression guard for the standalone-regression bug: when a seeder + // runs manually (operator invocation, not bundle-runner), the env + // var is absent. Earlier designs fell back to `Date.now()` which + // rejected any previously-seeded peer envelope as "stale" — even + // when the operator ran the Reexport seeder milliseconds beforehand. + // The fix: getBundleRunStartedAtMs() returns null outside a bundle; + // the consumer skips the freshness gate but still requires meta + // existence (peer outage still fails safely). + delete process.env.BUNDLE_RUN_STARTED_AT_MS; + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { AE: { reexportShareOfImports: 0.35, year: 2023 } }, + }; + // Meta written 10 MINUTES ago — rejected under the old `Date.now()` + // fallback, accepted under the null-return + skip-gate fix. + keyStore[REEXPORT_SHARE_META_KEY] = { fetchedAt: Date.now() - 600_000 }; + + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 1, + 'standalone: operator-seeded peer data must be accepted even if written before this process started'); + assert.equal(map.get('AE')?.reexportShareOfImports, 0.35); + }); + + it('standalone mode still rejects missing meta (peer outage still fails safely)', async () => { + // Even in standalone mode, meta absence means "peer never ran" — + // must fall back to gross imports, don't apply potentially stale + // shares from a data key that has no freshness signal. + delete process.env.BUNDLE_RUN_STARTED_AT_MS; + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { AE: { reexportShareOfImports: 0.35, year: 2023 } }, + }; + // No meta key written — peer outage. + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 0, + 'standalone: absent meta must still fall back (peer-outage fail-safe survives gate bypass)'); + }); + + it('fetchedAtMs === bundleStartMs passes (inclusive freshness boundary)', async () => { + // The freshness check uses strict-less-than: `fetchedAt < bundleStart`. + // Exact equality is treated as FRESH. This pins the inclusive-boundary + // semantic so a future refactor to `<=` fails this test loudly + // instead of silently rejecting a peer that wrote at the very first + // millisecond of the bundle run (theoretically possible on a fast + // host where t0 and the peer's fetchedAt align on the same ms). + const bundleStart = 1_700_000_000_000; + process.env.BUNDLE_RUN_STARTED_AT_MS = String(bundleStart); + keyStore[REEXPORT_SHARE_KEY] = { + manifestVersion: 2, + countries: { AE: { reexportShareOfImports: 0.4, year: 2023 } }, + }; + keyStore[REEXPORT_SHARE_META_KEY] = { fetchedAt: bundleStart }; // EXACT equality + const map = await loadReexportShareFromRedis(); + assert.equal(map.size, 1, + 'equality at the freshness boundary must be treated as FRESH'); + assert.equal(map.get('AE')?.reexportShareOfImports, 0.4); + }); +});