mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(economic): WoW price tracking + weekly cadence for BigMac & Grocery panels (#1974)
* feat(economic): add WoW tracking and fix plumbing for bigmac/grocery-basket panels Phase 1 — Fix Plumbing: - Adjust CACHE_TTL to 10 days (864000s) for bigmac and grocery-basket seeds - Align health.js SEED_META maxStaleMin to 10080 (7 days) for both - Add grocery-basket and bigmac to seed-health.js SEED_DOMAINS with intervalMin: 5040 - Refactor publish.ts writeSnapshot to accept advanceSeedMeta param; only advance seed-meta when fresh data exists (overallFreshnessMin < 120) - Add manual-fallback-only comment to seed-consumer-prices.mjs Phase 2 — Week-over-Week Tracking: - Add wow_pct field to BigMacCountryPrice and CountryBasket proto messages - Add wow_avg_pct, wow_available, prev_fetched_at to both response protos - Regenerate client/server TypeScript from updated protos - Add readCurrentSnapshot() helper + WoW computation to seed-bigmac.mjs and seed-grocery-basket.mjs; write :prev key via extraKeys - Update BigMacPanel.ts to show per-country WoW column and global avg summary - Update GroceryBasketPanel.ts to show WoW badge on total row and basket avg summary - Add .bm-wow-up, .bm-wow-down, .bm-wow-summary, .gb-wow CSS classes - Fix server handlers to include new WoW fields in fallback responses * fix(economic): guard :prev extraKey against null on first seed run; eliminate double freshness query in publish.ts * refactor(economic): address code review findings from PR #1974 - Extract readSeedSnapshot() into _seed-utils.mjs (DRY: was duplicated verbatim in seed-bigmac and seed-grocery-basket) - Add FRESH_DATA_THRESHOLD_MIN constant in publish.ts (replace magic 120) - Fix seed-consumer-prices.mjs contradictory JSDoc (remove stale "Deployed as: Railway cron service" line that contradicted manual-only warning) - Add i18n keys panels.bigmacWow / panels.bigmacCountry to en.json - Replace hardcoded "WoW" / "Country" with t() calls in BigMacPanel - Replace IIFE-in-ternary pattern with plain if blocks in BigMacPanel and GroceryBasketPanel (P2/P3 from code review) * fix(publish): gate advanceSeedMeta on any-retailer freshness, not average overallFreshnessMin is the arithmetic mean across all retailers, so with 1 fresh + 2 stale retailers the average can exceed 120 min and suppress seed-meta advancement even while fresh data is being published. Use retailers.some(r => r.freshnessMin < 120) to correctly implement "at least one retailer scraped within the last 2 hours."
This commit is contained in:
@@ -44,21 +44,26 @@ async function writeSnapshot(
|
||||
key: string,
|
||||
data: unknown,
|
||||
ttlSeconds: number,
|
||||
) {
|
||||
advanceSeedMeta = true,
|
||||
): Promise<void> {
|
||||
const json = JSON.stringify(data);
|
||||
await upstashCommand(url, token, ['SET', key, json, 'EX', ttlSeconds]);
|
||||
await upstashCommand(url, token, [
|
||||
'SET',
|
||||
makeKey(['seed-meta', key]),
|
||||
JSON.stringify({ fetchedAt: Date.now(), recordCount: recordCount(data) }),
|
||||
'EX',
|
||||
ttlSeconds * 2,
|
||||
]);
|
||||
logger.info(` wrote ${key} (${json.length} bytes, ttl=${ttlSeconds}s)`);
|
||||
if (advanceSeedMeta) {
|
||||
await upstashCommand(url, token, [
|
||||
'SET',
|
||||
makeKey(['seed-meta', key]),
|
||||
JSON.stringify({ fetchedAt: Date.now(), recordCount: recordCount(data) }),
|
||||
'EX',
|
||||
ttlSeconds * 2,
|
||||
]);
|
||||
}
|
||||
logger.info(` wrote ${key} (${json.length} bytes, ttl=${ttlSeconds}s, meta=${advanceSeedMeta})`);
|
||||
}
|
||||
|
||||
// 26h TTL — longer than the 24h cron cadence to survive scheduling drift
|
||||
const TTL = 93600;
|
||||
// Freshness gate: at least one retailer scraped within this window advances seed-meta
|
||||
const FRESH_DATA_THRESHOLD_MIN = 120;
|
||||
|
||||
export async function publishAll() {
|
||||
const url = process.env.UPSTASH_REDIS_REST_URL;
|
||||
@@ -69,12 +74,30 @@ export async function publishAll() {
|
||||
const markets = [...new Set(retailers.map((r) => r.marketCode))];
|
||||
const baskets = loadAllBasketConfigs();
|
||||
|
||||
// Build freshness snapshots first — used both to gate seed-meta and as the written payload
|
||||
const marketFreshnessSnapshots: Record<string, Awaited<ReturnType<typeof buildFreshnessSnapshot>> | null> = {};
|
||||
for (const marketCode of markets) {
|
||||
logger.info(`Publishing snapshots for market: ${marketCode}`);
|
||||
try {
|
||||
marketFreshnessSnapshots[marketCode] = await buildFreshnessSnapshot(marketCode);
|
||||
} catch {
|
||||
marketFreshnessSnapshots[marketCode] = null;
|
||||
}
|
||||
}
|
||||
|
||||
for (const marketCode of markets) {
|
||||
const freshnessSnapshot = marketFreshnessSnapshots[marketCode];
|
||||
// hasFreshData = at least one retailer scraped within last 2 hours
|
||||
// NOTE: overallFreshnessMin is an average — using .some() to correctly check "any retailer is fresh"
|
||||
const advanceSeedMeta =
|
||||
freshnessSnapshot != null &&
|
||||
freshnessSnapshot.retailers.some(
|
||||
(r) => r.freshnessMin > 0 && r.freshnessMin < FRESH_DATA_THRESHOLD_MIN,
|
||||
);
|
||||
logger.info(`Publishing snapshots for market: ${marketCode} (freshData=${advanceSeedMeta})`);
|
||||
|
||||
try {
|
||||
const overview = await buildOverviewSnapshot(marketCode);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'overview', marketCode]), overview, TTL);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'overview', marketCode]), overview, TTL, advanceSeedMeta);
|
||||
} catch (err) {
|
||||
logger.error(`overview:${marketCode} failed: ${err}`);
|
||||
}
|
||||
@@ -82,15 +105,17 @@ export async function publishAll() {
|
||||
for (const days of [7, 30]) {
|
||||
try {
|
||||
const movers = await buildMoversSnapshot(marketCode, days);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'movers', marketCode, `${days}d`]), movers, TTL);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'movers', marketCode, `${days}d`]), movers, TTL, advanceSeedMeta);
|
||||
} catch (err) {
|
||||
logger.error(`movers:${marketCode}:${days}d failed: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const freshness = await buildFreshnessSnapshot(marketCode);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'freshness', marketCode]), freshness, TTL);
|
||||
// Reuse already-built freshness snapshot — no second DB query
|
||||
if (freshnessSnapshot != null) {
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'freshness', marketCode]), freshnessSnapshot, TTL, advanceSeedMeta);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`freshness:${marketCode} failed: ${err}`);
|
||||
}
|
||||
@@ -98,7 +123,7 @@ export async function publishAll() {
|
||||
for (const range of ['7d', '30d', '90d']) {
|
||||
try {
|
||||
const categories = await buildCategoriesSnapshot(marketCode, range);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'categories', marketCode, range]), categories, TTL);
|
||||
await writeSnapshot(url, token, makeKey(['consumer-prices', 'categories', marketCode, range]), categories, TTL, advanceSeedMeta);
|
||||
} catch (err) {
|
||||
logger.error(`categories:${marketCode}:${range} failed: ${err}`);
|
||||
}
|
||||
@@ -112,6 +137,7 @@ export async function publishAll() {
|
||||
makeKey(['consumer-prices', 'retailer-spread', marketCode, basket.slug]),
|
||||
spread,
|
||||
TTL,
|
||||
advanceSeedMeta,
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(`spread:${marketCode}:${basket.slug} failed: ${err}`);
|
||||
@@ -125,6 +151,7 @@ export async function publishAll() {
|
||||
makeKey(['consumer-prices', 'basket-series', marketCode, basket.slug, range]),
|
||||
series,
|
||||
TTL,
|
||||
advanceSeedMeta,
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(`basket-series:${marketCode}:${basket.slug}:${range} failed: ${err}`);
|
||||
|
||||
Reference in New Issue
Block a user