From 38218db7cdf07728466b834bf20e0557b24b3c89 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 23 Apr 2026 20:43:27 +0400 Subject: [PATCH] =?UTF-8?q?fix(energy):=20strict=20validation=20=E2=80=94?= =?UTF-8?q?=20emptyDataIsFailure=20on=20Atlas=20seeders=20(#3350)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `emptyDataIsFailure: true` to all 5 curated-registry seeders in the `seed-bundle-energy-sources` Railway service. File-read-and-validate seeders whose validateFn returns false (stale container, missing data file, shape regression, etc.) MUST leave seed-meta stale rather than stamping fresh `recordCount: 0` via the default `publishResult.skipped` branch in `_seed-utils.mjs:906-917`. Why this matters — observed production incident on 2026-04-23 (post PR #3337 merge): - Subset of Atlas seeders hit the validation-skip path (for reasons involving a Railway container stale vs the merged code + a local Option A run during an intermediate-file-state window). - `_seed-utils.mjs:910` `writeFreshnessMetadata(..., 0, ...)` stamped `seed-meta:energy:pipelines-oil` and `seed-meta:energy:storage-facilities` with fresh `fetchedAt + recordCount: 0`. - Bundle runner's interval gate at `_bundle-runner.mjs:210` reads `fetchedAt` only, not `recordCount`. With `elapsed < 0.8 × 10080min = 8064min`, the gate skipped these 2 sections for ~5.5 days. No canonical data was written; health reported EMPTY; bundle never self-healed. With `emptyDataIsFailure: true`, the strict branch at `_seed-utils.mjs:897-905` fires instead: FAILURE: validation failed (empty data) — seed-meta NOT refreshed; bundle will retry next cycle Seed-meta stays stale, bundle counts it as `failed++`, next cron tick retries. Health flips STALE_SEED within max-stale-min. Operator sees it. Loud-failure instead of silent-skip-with-meta-refresh. Pattern previously documented for strict-floor validators (IMF/WEO 180+ country seeders in `feedback_strict_floor_validate_fail_poisons_seed_meta.md`) — now applied to all 5 Energy Atlas curated registries for the same reasons. No functional change in the healthy path — validation-passing runs still publish canonical + fresh seed-meta as before. Verification: typecheck clean, 6618/6618 data tests pass. --- scripts/seed-energy-disruptions.mjs | 3 +++ scripts/seed-fuel-shortages.mjs | 3 +++ scripts/seed-pipelines-gas.mjs | 6 ++++++ scripts/seed-pipelines-oil.mjs | 3 +++ scripts/seed-storage-facilities.mjs | 6 ++++++ 5 files changed, 21 insertions(+) diff --git a/scripts/seed-energy-disruptions.mjs b/scripts/seed-energy-disruptions.mjs index daf7157b1..322c9f740 100644 --- a/scripts/seed-energy-disruptions.mjs +++ b/scripts/seed-energy-disruptions.mjs @@ -25,6 +25,9 @@ if (isMain) { declareRecords, schemaVersion: 1, maxStaleMin: MAX_STALE_MIN, + // See seed-pipelines-gas.mjs for rationale — strict validation failure + // must leave seed-meta stale so the bundle retries every tick. + emptyDataIsFailure: true, }).catch((err) => { const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + cause); diff --git a/scripts/seed-fuel-shortages.mjs b/scripts/seed-fuel-shortages.mjs index 421733768..d0d0760c8 100644 --- a/scripts/seed-fuel-shortages.mjs +++ b/scripts/seed-fuel-shortages.mjs @@ -28,6 +28,9 @@ if (isMain) { declareRecords, schemaVersion: 1, maxStaleMin: MAX_STALE_MIN, + // See seed-pipelines-gas.mjs for rationale — strict validation failure + // must leave seed-meta stale so the bundle retries every tick. + emptyDataIsFailure: true, }).catch((err) => { const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + cause); diff --git a/scripts/seed-pipelines-gas.mjs b/scripts/seed-pipelines-gas.mjs index f2fb0d2bb..04f0867e9 100644 --- a/scripts/seed-pipelines-gas.mjs +++ b/scripts/seed-pipelines-gas.mjs @@ -31,6 +31,12 @@ if (isMain) { declareRecords, schemaVersion: 1, maxStaleMin: MAX_STALE_MIN, + // File-read-and-validate seeder: if the container can't load/validate the + // registry (stale image, missing data file, shape regression), fail LOUDLY + // rather than refreshing seed-meta with recordCount=0. Without this, the + // bundle's interval gate silently locks the seeder out for ~7 days after + // a single transient validation failure. + emptyDataIsFailure: true, }).catch((err) => { const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + cause); diff --git a/scripts/seed-pipelines-oil.mjs b/scripts/seed-pipelines-oil.mjs index 2fd5bfbfd..176245b65 100644 --- a/scripts/seed-pipelines-oil.mjs +++ b/scripts/seed-pipelines-oil.mjs @@ -30,6 +30,9 @@ if (isMain) { declareRecords, schemaVersion: 1, maxStaleMin: MAX_STALE_MIN, + // See seed-pipelines-gas.mjs for rationale — strict validation failure + // must leave seed-meta stale so the bundle retries every tick. + emptyDataIsFailure: true, }).catch((err) => { const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + cause); diff --git a/scripts/seed-storage-facilities.mjs b/scripts/seed-storage-facilities.mjs index 0d7679035..f7e671e23 100644 --- a/scripts/seed-storage-facilities.mjs +++ b/scripts/seed-storage-facilities.mjs @@ -29,6 +29,12 @@ if (isMain) { declareRecords, schemaVersion: 1, maxStaleMin: MAX_STALE_MIN, + // File-read-and-validate seeder: if the container can't load/validate the + // registry (stale image, missing data file, shape regression), fail LOUDLY + // rather than refreshing seed-meta with recordCount=0. Without this, the + // bundle gate silently locks the seeder out for ~5.5 days after a single + // validation hiccup. See seed-pipelines-gas.mjs for the canonical incident. + emptyDataIsFailure: true, }).catch((err) => { const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + cause);