mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
fix(seeds): add empty-data guards and fix health semantics (#1767)
Health semantics: - Add faaDelays + gpsjam to EMPTY_DATA_OK_KEYS (0 records = calm, not error) - Fix EMPTY_DATA_OK_KEYS branch to still check seed-meta freshness (prevents stale empty caches from staying green indefinitely) Seed guards: - seed-airport-delays: fix meta key in fetch-failure path (seed-meta:aviation:delays -> seed-meta:aviation:faa + seed-meta:aviation:notam) - seed-military-flights: add full TTL extension on zero-flights branch (was exiting without preserving any derived data TTLs) - seed-wb-indicators: add percentage-drop guard (new count < 50% of cached = likely partial API failure, extend TTL instead of overwriting) - ais-relay.cjs: same percentage-drop guard for WB dual writer Codex-reviewed plan (5 rounds, approved).
This commit is contained in:
@@ -151,7 +151,7 @@ const ON_DEMAND_KEYS = new Set([
|
||||
|
||||
// Keys where 0 records is a valid healthy state (e.g. no airports closed).
|
||||
// The key must still exist in Redis; only the record count can be 0.
|
||||
const EMPTY_DATA_OK_KEYS = new Set(['notamClosures']);
|
||||
const EMPTY_DATA_OK_KEYS = new Set(['notamClosures', 'faaDelays', 'gpsjam']);
|
||||
|
||||
// Cascade groups: if any key in the group has data, all empty siblings are OK.
|
||||
// Theater posture uses live → stale → backup fallback chain.
|
||||
@@ -333,9 +333,14 @@ export default async function handler(req) {
|
||||
if (cascadeCovered) {
|
||||
status = 'OK_CASCADE';
|
||||
okCount++;
|
||||
} else if (EMPTY_DATA_OK_KEYS.has(name) && seedStale === false) {
|
||||
status = 'OK';
|
||||
okCount++;
|
||||
} else if (EMPTY_DATA_OK_KEYS.has(name)) {
|
||||
if (seedStale === true) {
|
||||
status = 'STALE_SEED';
|
||||
warnCount++;
|
||||
} else {
|
||||
status = 'OK';
|
||||
okCount++;
|
||||
}
|
||||
} else if (isOnDemand) {
|
||||
status = 'EMPTY_ON_DEMAND';
|
||||
warnCount++;
|
||||
@@ -348,8 +353,13 @@ export default async function handler(req) {
|
||||
status = 'OK_CASCADE';
|
||||
okCount++;
|
||||
} else if (EMPTY_DATA_OK_KEYS.has(name)) {
|
||||
status = 'OK';
|
||||
okCount++;
|
||||
if (seedStale === true) {
|
||||
status = 'STALE_SEED';
|
||||
warnCount++;
|
||||
} else {
|
||||
status = 'OK';
|
||||
okCount++;
|
||||
}
|
||||
} else if (isOnDemand) {
|
||||
status = 'EMPTY_ON_DEMAND';
|
||||
warnCount++;
|
||||
|
||||
@@ -3692,6 +3692,19 @@ async function seedWorldBank() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Percentage-drop guard: if new count < 50% of prior count, skip overwrite
|
||||
try {
|
||||
const priorMeta = await upstashGet(`seed-meta:${WB_BOOTSTRAP_KEY}`);
|
||||
if (priorMeta && typeof priorMeta.recordCount === 'number' && priorMeta.recordCount > 0) {
|
||||
if (rankings.length < priorMeta.recordCount * 0.5) {
|
||||
console.warn(`[WB] Rankings dropped >50%: ${rankings.length} vs prior ${priorMeta.recordCount} — skipping overwrite`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[WB] Percentage-drop guard failed (proceeding):', e?.message);
|
||||
}
|
||||
|
||||
const metaTtl = WB_TTL_SECONDS + 3600;
|
||||
let ok = await upstashSet(WB_BOOTSTRAP_KEY, rankings, WB_TTL_SECONDS);
|
||||
console.log(`[WB] techReadiness: ${rankings.length} rankings (redis: ${ok ? 'OK' : 'FAIL'})`);
|
||||
|
||||
@@ -274,7 +274,7 @@ async function main() {
|
||||
} catch (err) {
|
||||
await releaseLock('aviation:delays', runId);
|
||||
console.error(` FETCH FAILED: ${err.message || err}`);
|
||||
await extendExistingTtl([FAA_CACHE_KEY, NOTAM_CACHE_KEY, 'seed-meta:aviation:delays'], CACHE_TTL);
|
||||
await extendExistingTtl([FAA_CACHE_KEY, NOTAM_CACHE_KEY, 'seed-meta:aviation:faa', 'seed-meta:aviation:notam'], CACHE_TTL);
|
||||
console.log(`\n=== Failed gracefully (${Math.round(Date.now() - startMs)}ms) ===`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -1319,7 +1319,13 @@ async function main() {
|
||||
}
|
||||
|
||||
if (flights.length === 0) {
|
||||
console.log(' SKIPPED: 0 military flights — preserving stale data');
|
||||
console.log(' SKIPPED: 0 military flights — extending existing TTLs');
|
||||
await extendExistingTtl([LIVE_KEY, 'seed-meta:military:flights'], LIVE_TTL);
|
||||
await extendExistingTtl([STALE_KEY, THEATER_POSTURE_STALE_KEY, MILITARY_SURGES_STALE_KEY, MILITARY_FORECAST_INPUTS_STALE_KEY, MILITARY_CLASSIFICATION_AUDIT_STALE_KEY], STALE_TTL);
|
||||
await extendExistingTtl([THEATER_POSTURE_LIVE_KEY, MILITARY_FORECAST_INPUTS_LIVE_KEY, MILITARY_CLASSIFICATION_AUDIT_LIVE_KEY], THEATER_POSTURE_LIVE_TTL);
|
||||
await extendExistingTtl([THEATER_POSTURE_BACKUP_KEY], THEATER_POSTURE_BACKUP_TTL);
|
||||
await extendExistingTtl([MILITARY_SURGES_LIVE_KEY], MILITARY_SURGES_LIVE_TTL);
|
||||
await extendExistingTtl(['seed-meta:theater-posture', 'seed-meta:military-forecast-inputs', 'seed-meta:military-surges'], STALE_TTL);
|
||||
await releaseLock('military:flights', runId);
|
||||
lockReleased = true;
|
||||
process.exit(0);
|
||||
|
||||
@@ -441,6 +441,32 @@ async function main() {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Percentage-drop guard: if new count < 50% of prior count, extend TTLs instead of overwriting
|
||||
try {
|
||||
const priorMetaResp = await redisPipeline(redisUrl, redisToken, [
|
||||
['GET', `seed-meta:${BOOTSTRAP_KEY}`],
|
||||
]);
|
||||
const priorMeta = priorMetaResp[0]?.result ? JSON.parse(priorMetaResp[0].result) : null;
|
||||
if (priorMeta && typeof priorMeta.recordCount === 'number' && priorMeta.recordCount > 0) {
|
||||
if (rankings.length < priorMeta.recordCount * 0.5) {
|
||||
console.warn(`Rankings dropped >50%: ${rankings.length} vs prior ${priorMeta.recordCount} — extending TTLs instead of overwriting.`);
|
||||
const extendPipeline = [
|
||||
['EXPIRE', fullKey, String(TTL_SECONDS)],
|
||||
['EXPIRE', `seed-meta:${BOOTSTRAP_KEY}`, String(TTL_SECONDS + 3600)],
|
||||
['EXPIRE', progressKey, String(TTL_SECONDS)],
|
||||
['EXPIRE', `seed-meta:${PROGRESS_KEY}`, String(TTL_SECONDS + 3600)],
|
||||
['EXPIRE', renewableKey, String(TTL_SECONDS)],
|
||||
['EXPIRE', `seed-meta:${RENEWABLE_KEY}`, String(TTL_SECONDS + 3600)],
|
||||
];
|
||||
await redisPipeline(redisUrl, redisToken, extendPipeline);
|
||||
console.log('TTLs extended. Exiting without overwriting.');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`Percentage-drop guard failed (proceeding with write): ${err.message}`);
|
||||
}
|
||||
|
||||
// Write all keys + seed-meta to Redis in one pipeline
|
||||
const metaTtl = String(TTL_SECONDS + 3600); // seed-meta outlives data by 1h
|
||||
const pipeline = [
|
||||
|
||||
Reference in New Issue
Block a user