feat(economic): WoW price tracking + weekly cadence for BigMac & Grocery panels (#1974)

* feat(economic): add WoW tracking and fix plumbing for bigmac/grocery-basket panels

Phase 1 — Fix Plumbing:
- Adjust CACHE_TTL to 10 days (864000s) for bigmac and grocery-basket seeds
- Align health.js SEED_META maxStaleMin to 10080 (7 days) for both
- Add grocery-basket and bigmac to seed-health.js SEED_DOMAINS with intervalMin: 5040
- Refactor publish.ts writeSnapshot to accept advanceSeedMeta param; only
  advance seed-meta when fresh data exists (overallFreshnessMin < 120)
- Add manual-fallback-only comment to seed-consumer-prices.mjs

Phase 2 — Week-over-Week Tracking:
- Add wow_pct field to BigMacCountryPrice and CountryBasket proto messages
- Add wow_avg_pct, wow_available, prev_fetched_at to both response protos
- Regenerate client/server TypeScript from updated protos
- Add readCurrentSnapshot() helper + WoW computation to seed-bigmac.mjs
  and seed-grocery-basket.mjs; write :prev key via extraKeys
- Update BigMacPanel.ts to show per-country WoW column and global avg summary
- Update GroceryBasketPanel.ts to show WoW badge on total row and basket avg summary
- Add .bm-wow-up, .bm-wow-down, .bm-wow-summary, .gb-wow CSS classes
- Fix server handlers to include new WoW fields in fallback responses

* fix(economic): guard :prev extraKey against null on first seed run; eliminate double freshness query in publish.ts

* refactor(economic): address code review findings from PR #1974

- Extract readSeedSnapshot() into _seed-utils.mjs (DRY: was duplicated
  verbatim in seed-bigmac and seed-grocery-basket)
- Add FRESH_DATA_THRESHOLD_MIN constant in publish.ts (replace magic 120)
- Fix seed-consumer-prices.mjs contradictory JSDoc (remove stale
  "Deployed as: Railway cron service" line that contradicted manual-only warning)
- Add i18n keys panels.bigmacWow / panels.bigmacCountry to en.json
- Replace hardcoded "WoW" / "Country" with t() calls in BigMacPanel
- Replace IIFE-in-ternary pattern with plain if blocks in BigMacPanel
  and GroceryBasketPanel (P2/P3 from code review)

* fix(publish): gate advanceSeedMeta on any-retailer freshness, not average

overallFreshnessMin is the arithmetic mean across all retailers, so with
1 fresh + 2 stale retailers the average can exceed 120 min and suppress
seed-meta advancement even while fresh data is being published.

Use retailers.some(r => r.freshnessMin < 120) to correctly implement
"at least one retailer scraped within the last 2 hours."
This commit is contained in:
Elie Habib
2026-03-21 10:56:48 +04:00
committed by GitHub
parent 3aa8e627fc
commit 2e16159bb6
19 changed files with 244 additions and 34 deletions

View File

@@ -44,21 +44,26 @@ async function writeSnapshot(
key: string,
data: unknown,
ttlSeconds: number,
) {
advanceSeedMeta = true,
): Promise<void> {
const json = JSON.stringify(data);
await upstashCommand(url, token, ['SET', key, json, 'EX', ttlSeconds]);
await upstashCommand(url, token, [
'SET',
makeKey(['seed-meta', key]),
JSON.stringify({ fetchedAt: Date.now(), recordCount: recordCount(data) }),
'EX',
ttlSeconds * 2,
]);
logger.info(` wrote ${key} (${json.length} bytes, ttl=${ttlSeconds}s)`);
if (advanceSeedMeta) {
await upstashCommand(url, token, [
'SET',
makeKey(['seed-meta', key]),
JSON.stringify({ fetchedAt: Date.now(), recordCount: recordCount(data) }),
'EX',
ttlSeconds * 2,
]);
}
logger.info(` wrote ${key} (${json.length} bytes, ttl=${ttlSeconds}s, meta=${advanceSeedMeta})`);
}
// 26h TTL — longer than the 24h cron cadence to survive scheduling drift
const TTL = 93600;
// Freshness gate: at least one retailer scraped within this window advances seed-meta
const FRESH_DATA_THRESHOLD_MIN = 120;
export async function publishAll() {
const url = process.env.UPSTASH_REDIS_REST_URL;
@@ -69,12 +74,30 @@ export async function publishAll() {
const markets = [...new Set(retailers.map((r) => r.marketCode))];
const baskets = loadAllBasketConfigs();
// Build freshness snapshots first — used both to gate seed-meta and as the written payload
const marketFreshnessSnapshots: Record<string, Awaited<ReturnType<typeof buildFreshnessSnapshot>> | null> = {};
for (const marketCode of markets) {
logger.info(`Publishing snapshots for market: ${marketCode}`);
try {
marketFreshnessSnapshots[marketCode] = await buildFreshnessSnapshot(marketCode);
} catch {
marketFreshnessSnapshots[marketCode] = null;
}
}
for (const marketCode of markets) {
const freshnessSnapshot = marketFreshnessSnapshots[marketCode];
// hasFreshData = at least one retailer scraped within last 2 hours
// NOTE: overallFreshnessMin is an average — using .some() to correctly check "any retailer is fresh"
const advanceSeedMeta =
freshnessSnapshot != null &&
freshnessSnapshot.retailers.some(
(r) => r.freshnessMin > 0 && r.freshnessMin < FRESH_DATA_THRESHOLD_MIN,
);
logger.info(`Publishing snapshots for market: ${marketCode} (freshData=${advanceSeedMeta})`);
try {
const overview = await buildOverviewSnapshot(marketCode);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'overview', marketCode]), overview, TTL);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'overview', marketCode]), overview, TTL, advanceSeedMeta);
} catch (err) {
logger.error(`overview:${marketCode} failed: ${err}`);
}
@@ -82,15 +105,17 @@ export async function publishAll() {
for (const days of [7, 30]) {
try {
const movers = await buildMoversSnapshot(marketCode, days);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'movers', marketCode, `${days}d`]), movers, TTL);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'movers', marketCode, `${days}d`]), movers, TTL, advanceSeedMeta);
} catch (err) {
logger.error(`movers:${marketCode}:${days}d failed: ${err}`);
}
}
try {
const freshness = await buildFreshnessSnapshot(marketCode);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'freshness', marketCode]), freshness, TTL);
// Reuse already-built freshness snapshot — no second DB query
if (freshnessSnapshot != null) {
await writeSnapshot(url, token, makeKey(['consumer-prices', 'freshness', marketCode]), freshnessSnapshot, TTL, advanceSeedMeta);
}
} catch (err) {
logger.error(`freshness:${marketCode} failed: ${err}`);
}
@@ -98,7 +123,7 @@ export async function publishAll() {
for (const range of ['7d', '30d', '90d']) {
try {
const categories = await buildCategoriesSnapshot(marketCode, range);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'categories', marketCode, range]), categories, TTL);
await writeSnapshot(url, token, makeKey(['consumer-prices', 'categories', marketCode, range]), categories, TTL, advanceSeedMeta);
} catch (err) {
logger.error(`categories:${marketCode}:${range} failed: ${err}`);
}
@@ -112,6 +137,7 @@ export async function publishAll() {
makeKey(['consumer-prices', 'retailer-spread', marketCode, basket.slug]),
spread,
TTL,
advanceSeedMeta,
);
} catch (err) {
logger.error(`spread:${marketCode}:${basket.slug} failed: ${err}`);
@@ -125,6 +151,7 @@ export async function publishAll() {
makeKey(['consumer-prices', 'basket-series', marketCode, basket.slug, range]),
series,
TTL,
advanceSeedMeta,
);
} catch (err) {
logger.error(`basket-series:${marketCode}:${basket.slug}:${range} failed: ${err}`);