fix(seeds): add empty-data guards and fix health semantics (#1767)

Health semantics:
- Add faaDelays + gpsjam to EMPTY_DATA_OK_KEYS (0 records = calm, not error)
- Fix EMPTY_DATA_OK_KEYS branch to still check seed-meta freshness
  (prevents stale empty caches from staying green indefinitely)

Seed guards:
- seed-airport-delays: fix meta key in fetch-failure path
  (seed-meta:aviation:delays -> seed-meta:aviation:faa + seed-meta:aviation:notam)
- seed-military-flights: add full TTL extension on zero-flights branch
  (was exiting without preserving any derived data TTLs)
- seed-wb-indicators: add percentage-drop guard (new count < 50% of cached
  = likely partial API failure, extend TTL instead of overwriting)
- ais-relay.cjs: same percentage-drop guard for WB dual writer

Codex-reviewed plan (5 rounds, approved).
This commit is contained in:
Elie Habib
2026-03-17 16:12:05 +04:00
committed by GitHub
parent 0d4519c324
commit 83fe44afa3
5 changed files with 63 additions and 8 deletions

View File

@@ -151,7 +151,7 @@ const ON_DEMAND_KEYS = new Set([
// Keys where 0 records is a valid healthy state (e.g. no airports closed).
// The key must still exist in Redis; only the record count can be 0.
const EMPTY_DATA_OK_KEYS = new Set(['notamClosures']);
const EMPTY_DATA_OK_KEYS = new Set(['notamClosures', 'faaDelays', 'gpsjam']);
// Cascade groups: if any key in the group has data, all empty siblings are OK.
// Theater posture uses live → stale → backup fallback chain.
@@ -333,9 +333,14 @@ export default async function handler(req) {
if (cascadeCovered) {
status = 'OK_CASCADE';
okCount++;
} else if (EMPTY_DATA_OK_KEYS.has(name) && seedStale === false) {
status = 'OK';
okCount++;
} else if (EMPTY_DATA_OK_KEYS.has(name)) {
if (seedStale === true) {
status = 'STALE_SEED';
warnCount++;
} else {
status = 'OK';
okCount++;
}
} else if (isOnDemand) {
status = 'EMPTY_ON_DEMAND';
warnCount++;
@@ -348,8 +353,13 @@ export default async function handler(req) {
status = 'OK_CASCADE';
okCount++;
} else if (EMPTY_DATA_OK_KEYS.has(name)) {
status = 'OK';
okCount++;
if (seedStale === true) {
status = 'STALE_SEED';
warnCount++;
} else {
status = 'OK';
okCount++;
}
} else if (isOnDemand) {
status = 'EMPTY_ON_DEMAND';
warnCount++;

View File

@@ -3692,6 +3692,19 @@ async function seedWorldBank() {
return;
}
// Percentage-drop guard: if new count < 50% of prior count, skip overwrite
try {
const priorMeta = await upstashGet(`seed-meta:${WB_BOOTSTRAP_KEY}`);
if (priorMeta && typeof priorMeta.recordCount === 'number' && priorMeta.recordCount > 0) {
if (rankings.length < priorMeta.recordCount * 0.5) {
console.warn(`[WB] Rankings dropped >50%: ${rankings.length} vs prior ${priorMeta.recordCount} — skipping overwrite`);
return;
}
}
} catch (e) {
console.warn('[WB] Percentage-drop guard failed (proceeding):', e?.message);
}
const metaTtl = WB_TTL_SECONDS + 3600;
let ok = await upstashSet(WB_BOOTSTRAP_KEY, rankings, WB_TTL_SECONDS);
console.log(`[WB] techReadiness: ${rankings.length} rankings (redis: ${ok ? 'OK' : 'FAIL'})`);

View File

@@ -274,7 +274,7 @@ async function main() {
} catch (err) {
await releaseLock('aviation:delays', runId);
console.error(` FETCH FAILED: ${err.message || err}`);
await extendExistingTtl([FAA_CACHE_KEY, NOTAM_CACHE_KEY, 'seed-meta:aviation:delays'], CACHE_TTL);
await extendExistingTtl([FAA_CACHE_KEY, NOTAM_CACHE_KEY, 'seed-meta:aviation:faa', 'seed-meta:aviation:notam'], CACHE_TTL);
console.log(`\n=== Failed gracefully (${Math.round(Date.now() - startMs)}ms) ===`);
process.exit(0);
}

View File

@@ -1319,7 +1319,13 @@ async function main() {
}
if (flights.length === 0) {
console.log(' SKIPPED: 0 military flights — preserving stale data');
console.log(' SKIPPED: 0 military flights — extending existing TTLs');
await extendExistingTtl([LIVE_KEY, 'seed-meta:military:flights'], LIVE_TTL);
await extendExistingTtl([STALE_KEY, THEATER_POSTURE_STALE_KEY, MILITARY_SURGES_STALE_KEY, MILITARY_FORECAST_INPUTS_STALE_KEY, MILITARY_CLASSIFICATION_AUDIT_STALE_KEY], STALE_TTL);
await extendExistingTtl([THEATER_POSTURE_LIVE_KEY, MILITARY_FORECAST_INPUTS_LIVE_KEY, MILITARY_CLASSIFICATION_AUDIT_LIVE_KEY], THEATER_POSTURE_LIVE_TTL);
await extendExistingTtl([THEATER_POSTURE_BACKUP_KEY], THEATER_POSTURE_BACKUP_TTL);
await extendExistingTtl([MILITARY_SURGES_LIVE_KEY], MILITARY_SURGES_LIVE_TTL);
await extendExistingTtl(['seed-meta:theater-posture', 'seed-meta:military-forecast-inputs', 'seed-meta:military-surges'], STALE_TTL);
await releaseLock('military:flights', runId);
lockReleased = true;
process.exit(0);

View File

@@ -441,6 +441,32 @@ async function main() {
process.exit(1);
}
// Percentage-drop guard: if new count < 50% of prior count, extend TTLs instead of overwriting
try {
const priorMetaResp = await redisPipeline(redisUrl, redisToken, [
['GET', `seed-meta:${BOOTSTRAP_KEY}`],
]);
const priorMeta = priorMetaResp[0]?.result ? JSON.parse(priorMetaResp[0].result) : null;
if (priorMeta && typeof priorMeta.recordCount === 'number' && priorMeta.recordCount > 0) {
if (rankings.length < priorMeta.recordCount * 0.5) {
console.warn(`Rankings dropped >50%: ${rankings.length} vs prior ${priorMeta.recordCount} — extending TTLs instead of overwriting.`);
const extendPipeline = [
['EXPIRE', fullKey, String(TTL_SECONDS)],
['EXPIRE', `seed-meta:${BOOTSTRAP_KEY}`, String(TTL_SECONDS + 3600)],
['EXPIRE', progressKey, String(TTL_SECONDS)],
['EXPIRE', `seed-meta:${PROGRESS_KEY}`, String(TTL_SECONDS + 3600)],
['EXPIRE', renewableKey, String(TTL_SECONDS)],
['EXPIRE', `seed-meta:${RENEWABLE_KEY}`, String(TTL_SECONDS + 3600)],
];
await redisPipeline(redisUrl, redisToken, extendPipeline);
console.log('TTLs extended. Exiting without overwriting.');
process.exit(0);
}
}
} catch (err) {
console.warn(`Percentage-drop guard failed (proceeding with write): ${err.message}`);
}
// Write all keys + seed-meta to Redis in one pipeline
const metaTtl = String(TTL_SECONDS + 3600); // seed-meta outlives data by 1h
const pipeline = [