mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
38218db7cd
Adds `emptyDataIsFailure: true` to all 5 curated-registry seeders in the `seed-bundle-energy-sources` Railway service. File-read-and-validate seeders whose validateFn returns false (stale container, missing data file, shape regression, etc.) MUST leave seed-meta stale rather than stamping fresh `recordCount: 0` via the default `publishResult.skipped` branch in `_seed-utils.mjs:906-917`. Why this matters — observed production incident on 2026-04-23 (post PR #3337 merge): - Subset of Atlas seeders hit the validation-skip path (for reasons involving a Railway container stale vs the merged code + a local Option A run during an intermediate-file-state window). - `_seed-utils.mjs:910` `writeFreshnessMetadata(..., 0, ...)` stamped `seed-meta:energy:pipelines-oil` and `seed-meta:energy:storage-facilities` with fresh `fetchedAt + recordCount: 0`. - Bundle runner's interval gate at `_bundle-runner.mjs:210` reads `fetchedAt` only, not `recordCount`. With `elapsed < 0.8 × 10080min = 8064min`, the gate skipped these 2 sections for ~5.5 days. No canonical data was written; health reported EMPTY; bundle never self-healed. With `emptyDataIsFailure: true`, the strict branch at `_seed-utils.mjs:897-905` fires instead: FAILURE: validation failed (empty data) — seed-meta NOT refreshed; bundle will retry next cycle Seed-meta stays stale, bundle counts it as `failed++`, next cron tick retries. Health flips STALE_SEED within max-stale-min. Operator sees it. Loud-failure instead of silent-skip-with-meta-refresh. Pattern previously documented for strict-floor validators (IMF/WEO 180+ country seeders in `feedback_strict_floor_validate_fail_poisons_seed_meta.md`) — now applied to all 5 Energy Atlas curated registries for the same reasons. No functional change in the healthy path — validation-passing runs still publish canonical + fresh seed-meta as before. Verification: typecheck clean, 6618/6618 data tests pass.
42 lines
1.3 KiB
JavaScript
42 lines
1.3 KiB
JavaScript
#!/usr/bin/env node
|
|
// @ts-check
|
|
//
|
|
// Publishes energy:pipelines:oil:v1 from the curated registry in
|
|
// scripts/data/pipelines-oil.json. See _pipeline-registry.mjs for the shared
|
|
// helpers + validation. See seed-pipelines-gas.mjs for the gas sibling —
|
|
// separate entry points because runSeed() hard-exits its terminal paths.
|
|
|
|
import { loadEnvFile, runSeed } from './_seed-utils.mjs';
|
|
import {
|
|
OIL_CANONICAL_KEY,
|
|
PIPELINES_TTL_SECONDS,
|
|
MAX_STALE_MIN,
|
|
buildOilPayload,
|
|
validateRegistry,
|
|
recordCount,
|
|
declareRecords,
|
|
} from './_pipeline-registry.mjs';
|
|
|
|
loadEnvFile(import.meta.url);
|
|
|
|
const isMain = process.argv[1]?.endsWith('seed-pipelines-oil.mjs');
|
|
|
|
if (isMain) {
|
|
runSeed('energy', 'pipelines-oil', OIL_CANONICAL_KEY, buildOilPayload, {
|
|
validateFn: validateRegistry,
|
|
ttlSeconds: PIPELINES_TTL_SECONDS,
|
|
sourceVersion: 'pipelines-oil-registry-v1',
|
|
recordCount,
|
|
declareRecords,
|
|
schemaVersion: 1,
|
|
maxStaleMin: MAX_STALE_MIN,
|
|
// See seed-pipelines-gas.mjs for rationale — strict validation failure
|
|
// must leave seed-meta stale so the bundle retries every tick.
|
|
emptyDataIsFailure: true,
|
|
}).catch((err) => {
|
|
const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : '';
|
|
console.error('FATAL:', (err.message || err) + cause);
|
|
process.exit(1);
|
|
});
|
|
}
|