mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(seeds): Eurostat house prices + quarterly debt + industrial production (#3047)
* feat(seeds): Eurostat house prices + quarterly debt + industrial production Adds three new Eurostat overlay seeders covering all 27 EU members plus EA20 and EU27_2020 aggregates (issue #3028): - prc_hpi_a (annual house price index, 10y sparkline, TTL 35d) key: economic:eurostat:house-prices:v1 complements BIS WS_SPP (#3026) for the Housing cycle tile - gov_10q_ggdebt (quarterly gov debt %GDP, 8q sparkline, TTL 14d) key: economic:eurostat:gov-debt-q:v1 upgrades National Debt card cadence from annual IMF to quarterly for EU - sts_inpr_m (monthly industrial production, 12m sparkline, TTL 5d) key: economic:eurostat:industrial-production:v1 feeds "Real economy pulse" sparkline on Economic Indicators card Shared JSON-stat parser in scripts/_eurostat-utils.mjs handles the EL/GR and EA20 geo quirks and returns full time series for sparklines. Wires each seeder into bootstrap (SLOW_KEYS), health registries (keys + seed-meta thresholds matched to cadence), macro seed bundle, cache-keys shared module, and the MCP tool registry (get_eu_housing_cycle, get_eu_quarterly_gov_debt, get_eu_industrial_production). MCP tool count updated to 31. Tests cover JSON-stat parsing, sparkline ordering, EU-only coverage gating (non-EU geos return null so panels never render blank tiles), validator thresholds, and registry wiring across all surfaces. https://claude.ai/code/session_01Tgm6gG5yUMRoc2LRAKvmza * fix(bootstrap): register new Eurostat keys in tiers, defer consumers Adds eurostatHousePrices/GovDebtQ/IndProd to BOOTSTRAP_TIERS ('slow') to match SLOW_KEYS in api/bootstrap.js, and lists them as PENDING_CONSUMERS in the hydration coverage test (panel wiring lands in follow-up). * fix(eurostat): raise seeder coverage thresholds to catch partial publishes The three Eurostat overlay seeders (house prices, quarterly gov debt, monthly industrial production) all validated with makeValidator(10) against a fixed 29-geo universe (EU27 + EA20 + EU27_2020). A bad run returning only 10-15 geos would pass validation and silently publish a snapshot missing most of the EU. Raise thresholds to near-complete coverage, with a small margin for geos with patchy reporting: - house prices (annual): 10 -> 24 - gov debt (quarterly): 10 -> 24 - industrial prod (monthly): 10 -> 22 (monthly is slightly patchier) Add a guard test that asserts every overlay seeder keeps its threshold >=22 so this regression can't reappear. * fix(seed-health): register 3 Eurostat seed-meta entries house-prices, gov-debt-q, industrial-production were wired in api/health.js SEED_META but missing from api/seed-health.js SEED_DOMAINS, so /api/seed-health would not surface their freshness. intervalMin = health.js maxStaleMin / 2 per convention. --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
6
api/bootstrap.js
vendored
6
api/bootstrap.js
vendored
@@ -79,6 +79,9 @@ const BOOTSTRAP_CACHE_KEYS = {
|
||||
nationalDebt: 'economic:national-debt:v1',
|
||||
euGasStorage: 'economic:eu-gas-storage:v1',
|
||||
eurostatCountryData: 'economic:eurostat-country-data:v1',
|
||||
eurostatHousePrices: 'economic:eurostat:house-prices:v1',
|
||||
eurostatGovDebtQ: 'economic:eurostat:gov-debt-q:v1',
|
||||
eurostatIndProd: 'economic:eurostat:industrial-production:v1',
|
||||
marketImplications: 'intelligence:market-implications:v1',
|
||||
fearGreedIndex: 'market:fear-greed:v1',
|
||||
crudeInventories: 'economic:crude-inventories:v1',
|
||||
@@ -124,6 +127,9 @@ const SLOW_KEYS = new Set([
|
||||
'nationalDebt',
|
||||
'euGasStorage',
|
||||
'eurostatCountryData',
|
||||
'eurostatHousePrices',
|
||||
'eurostatGovDebtQ',
|
||||
'eurostatIndProd',
|
||||
'marketImplications',
|
||||
'fearGreedIndex',
|
||||
'crudeInventories',
|
||||
|
||||
@@ -77,6 +77,9 @@ const BOOTSTRAP_KEYS = {
|
||||
refineryInputs: 'economic:refinery-inputs:v1',
|
||||
ecbFxRates: 'economic:ecb-fx-rates:v1',
|
||||
eurostatCountryData: 'economic:eurostat-country-data:v1',
|
||||
eurostatHousePrices: 'economic:eurostat:house-prices:v1',
|
||||
eurostatGovDebtQ: 'economic:eurostat:gov-debt-q:v1',
|
||||
eurostatIndProd: 'economic:eurostat:industrial-production:v1',
|
||||
euGasStorage: 'economic:eu-gas-storage:v1',
|
||||
euFsi: 'economic:fsi-eu:v1',
|
||||
shippingStress: 'supply_chain:shipping_stress:v1',
|
||||
@@ -288,6 +291,9 @@ const SEED_META = {
|
||||
refineryInputs: { key: 'seed-meta:economic:refinery-inputs', maxStaleMin: 20160 }, // weekly EIA data; 20160min = 14 days = 2x weekly cadence
|
||||
ecbFxRates: { key: 'seed-meta:economic:ecb-fx-rates', maxStaleMin: 5760 }, // daily seed (weekdays + holidays); 5760min = 96h = covers Wed→Mon Easter gap
|
||||
eurostatCountryData: { key: 'seed-meta:economic:eurostat-country-data', maxStaleMin: 4320 }, // daily seed; 4320min = 3 days = 3x interval
|
||||
eurostatHousePrices: { key: 'seed-meta:economic:eurostat-house-prices', maxStaleMin: 60 * 24 * 50 }, // weekly cron, annual data; 50d threshold = 35d TTL + 15d buffer
|
||||
eurostatGovDebtQ: { key: 'seed-meta:economic:eurostat-gov-debt-q', maxStaleMin: 60 * 24 * 14 }, // 2d cron, quarterly data; 14d threshold matches TTL + quarterly release drift
|
||||
eurostatIndProd: { key: 'seed-meta:economic:eurostat-industrial-production', maxStaleMin: 60 * 24 * 5 }, // daily cron, monthly data; 5d threshold matches TTL
|
||||
euGasStorage: { key: 'seed-meta:economic:eu-gas-storage', maxStaleMin: 2880 }, // daily seed (T+1); 2880min = 48h = 2x interval
|
||||
euYieldCurve: { key: 'seed-meta:economic:yield-curve-eu', maxStaleMin: 4320 }, // daily seed (weekdays only); 4320min = 72h = covers Fri→Mon gap
|
||||
euFsi: { key: 'seed-meta:economic:fsi-eu', maxStaleMin: 20160 }, // weekly seed (Saturday); 20160min = 14d = 2x interval
|
||||
|
||||
24
api/mcp.ts
24
api/mcp.ts
@@ -188,6 +188,30 @@ const TOOL_REGISTRY: ToolDef[] = [
|
||||
{ key: 'seed-meta:economic:imf-external', maxStaleMin: 100800 },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'get_eu_housing_cycle',
|
||||
description: 'Eurostat annual house price index (prc_hpi_a, base 2015=100) for all 27 EU members plus EA20 and EU27_2020 aggregates. Each country entry includes the latest value, prior value, date, unit, and a 10-year sparkline series. Complements BIS WS_SPP with broader EU coverage for the Housing cycle tile.',
|
||||
inputSchema: { type: 'object', properties: {}, required: [] },
|
||||
_cacheKeys: ['economic:eurostat:house-prices:v1'],
|
||||
_seedMetaKey: 'seed-meta:economic:eurostat-house-prices',
|
||||
_maxStaleMin: 60 * 24 * 50, // weekly cron, annual data
|
||||
},
|
||||
{
|
||||
name: 'get_eu_quarterly_gov_debt',
|
||||
description: 'Eurostat quarterly general government gross debt (gov_10q_ggdebt, %GDP) for all 27 EU members plus EA20 and EU27_2020 aggregates. Each country entry includes latest value, prior value, quarter label, and an 8-quarter sparkline series. Provides fresher debt-trajectory signal than annual IMF GGXWDG_NGDP for EU panels.',
|
||||
inputSchema: { type: 'object', properties: {}, required: [] },
|
||||
_cacheKeys: ['economic:eurostat:gov-debt-q:v1'],
|
||||
_seedMetaKey: 'seed-meta:economic:eurostat-gov-debt-q',
|
||||
_maxStaleMin: 60 * 24 * 14, // quarterly data, 2-day cron
|
||||
},
|
||||
{
|
||||
name: 'get_eu_industrial_production',
|
||||
description: 'Eurostat monthly industrial production index (sts_inpr_m, NACE B-D industry excl. construction, SCA, base 2021=100) for all 27 EU members plus EA20 and EU27_2020 aggregates. Each country entry includes latest value, prior value, month label, and a 12-month sparkline series. Leading indicator of real-economy activity used by the "Real economy pulse" sparkline.',
|
||||
inputSchema: { type: 'object', properties: {}, required: [] },
|
||||
_cacheKeys: ['economic:eurostat:industrial-production:v1'],
|
||||
_seedMetaKey: 'seed-meta:economic:eurostat-industrial-production',
|
||||
_maxStaleMin: 60 * 24 * 5, // monthly data, daily cron
|
||||
},
|
||||
{
|
||||
name: 'get_prediction_markets',
|
||||
description: 'Active Polymarket event contracts with current probabilities. Covers geopolitical, economic, and election prediction markets.',
|
||||
|
||||
@@ -87,6 +87,9 @@ const SEED_DOMAINS = {
|
||||
'energy:spr-policies': { key: 'seed-meta:energy:spr-policies', intervalMin: 288000 }, // annual static registry; intervalMin = health.js maxStaleMin / 2 (576000 / 2)
|
||||
'market:aaii-sentiment': { key: 'seed-meta:market:aaii-sentiment', intervalMin: 10080 }, // weekly cron; intervalMin = maxStaleMin / 2 (20160 / 2)
|
||||
'intelligence:regional-briefs': { key: 'seed-meta:intelligence:regional-briefs', intervalMin: 10080 }, // weekly cron; intervalMin = health.js maxStaleMin / 2 (20160 / 2)
|
||||
'economic:eurostat-house-prices': { key: 'seed-meta:economic:eurostat-house-prices', intervalMin: 36000 }, // weekly cron, annual data; intervalMin = health.js maxStaleMin / 2 (72000 / 2)
|
||||
'economic:eurostat-gov-debt-q': { key: 'seed-meta:economic:eurostat-gov-debt-q', intervalMin: 10080 }, // 2d cron, quarterly data; intervalMin = health.js maxStaleMin / 2 (20160 / 2)
|
||||
'economic:eurostat-industrial-production': { key: 'seed-meta:economic:eurostat-industrial-production', intervalMin: 3600 }, // daily cron, monthly data; intervalMin = health.js maxStaleMin / 2 (7200 / 2)
|
||||
};
|
||||
|
||||
async function getMetaBatch(keys) {
|
||||
|
||||
191
scripts/_eurostat-utils.mjs
Normal file
191
scripts/_eurostat-utils.mjs
Normal file
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* Shared Eurostat JSON-stat parser + country list for per-dataset EU overlay seeders.
|
||||
* Used by seed-eurostat-house-prices, seed-eurostat-gov-debt-q, seed-eurostat-industrial-production.
|
||||
*/
|
||||
|
||||
import { CHROME_UA } from './_seed-utils.mjs';
|
||||
|
||||
export const EUROSTAT_BASE =
|
||||
'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data';
|
||||
|
||||
/**
|
||||
* All 27 EU members + EA20 (Euro Area) + EU27_2020 aggregates.
|
||||
* Eurostat geo quirks:
|
||||
* - Greece is 'EL' (not ISO 'GR')
|
||||
* - Euro Area is 'EA20' (post-2023)
|
||||
* - EU aggregate is 'EU27_2020'
|
||||
*/
|
||||
export const EU_GEOS = [
|
||||
'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR',
|
||||
'DE', 'EL', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL',
|
||||
'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE',
|
||||
'EA20', 'EU27_2020',
|
||||
];
|
||||
|
||||
/**
|
||||
* Parse Eurostat JSON-stat response for a specific geo code.
|
||||
* Returns a time series (sorted ascending by period) plus latest/prior.
|
||||
*
|
||||
* @param {object} data — JSON-stat response
|
||||
* @param {string} geoCode — Eurostat geo code (e.g. 'DE', 'EL', 'EA20')
|
||||
* @returns {{ value:number, priorValue:number|null, date:string, series:Array<{date:string,value:number}> }|null}
|
||||
*/
|
||||
export function parseEurostatSeries(data, geoCode) {
|
||||
try {
|
||||
const dims = data?.dimension;
|
||||
const values = data?.value;
|
||||
if (!dims || !values) return null;
|
||||
|
||||
const geoDim = dims.geo;
|
||||
if (!geoDim) return null;
|
||||
|
||||
const geoIndex = geoDim.category?.index;
|
||||
if (!geoIndex || geoIndex[geoCode] === undefined) return null;
|
||||
const geoPos = geoIndex[geoCode];
|
||||
|
||||
const timeIndexObj = dims.time?.category?.index;
|
||||
if (!timeIndexObj) return null;
|
||||
// Map time-position -> time-label (e.g. 0 -> '2020-Q1')
|
||||
const timeLabels = {};
|
||||
for (const [label, pos] of Object.entries(timeIndexObj)) {
|
||||
timeLabels[pos] = label;
|
||||
}
|
||||
|
||||
const dimOrder = data.id || [];
|
||||
const dimSizes = data.size || [];
|
||||
const strides = {};
|
||||
let stride = 1;
|
||||
for (let i = dimOrder.length - 1; i >= 0; i--) {
|
||||
strides[dimOrder[i]] = stride;
|
||||
stride *= dimSizes[i];
|
||||
}
|
||||
|
||||
const series = [];
|
||||
for (const key of Object.keys(values)) {
|
||||
const idx = Number(key);
|
||||
const rawVal = values[key];
|
||||
if (rawVal === null || rawVal === undefined) continue;
|
||||
|
||||
let remaining = idx;
|
||||
const coords = {};
|
||||
for (const dim of dimOrder) {
|
||||
const s = strides[dim];
|
||||
const dimSize = dimSizes[dimOrder.indexOf(dim)];
|
||||
coords[dim] = Math.floor(remaining / s) % dimSize;
|
||||
remaining = remaining % s;
|
||||
}
|
||||
|
||||
if (coords.geo !== geoPos) continue;
|
||||
const label = timeLabels[coords.time];
|
||||
if (!label) continue;
|
||||
series.push({
|
||||
date: label,
|
||||
value: typeof rawVal === 'number' ? Math.round(rawVal * 100) / 100 : null,
|
||||
});
|
||||
}
|
||||
|
||||
if (series.length === 0) return null;
|
||||
|
||||
// Sort ascending by period label. Eurostat labels are lexicographically
|
||||
// orderable for annual (YYYY), quarterly (YYYY-QN), and monthly (YYYY-MM).
|
||||
series.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0));
|
||||
|
||||
const latest = series[series.length - 1];
|
||||
const prior = series.length > 1 ? series[series.length - 2] : null;
|
||||
|
||||
return {
|
||||
value: latest.value,
|
||||
priorValue: prior ? prior.value : null,
|
||||
date: latest.date,
|
||||
series,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single Eurostat dataset for one geo and parse to series.
|
||||
* @param {{id:string, params:object, unit:string, label:string}} ds
|
||||
* @param {string} geoCode
|
||||
* @returns {Promise<{value:number, priorValue:number|null, date:string, series:Array, unit:string}|null>}
|
||||
*/
|
||||
export async function fetchEurostatCountry(ds, geoCode) {
|
||||
const params = new URLSearchParams({
|
||||
format: 'JSON',
|
||||
lang: 'EN',
|
||||
geo: geoCode,
|
||||
...ds.params,
|
||||
});
|
||||
const url = `${EUROSTAT_BASE}/${ds.id}?${params}`;
|
||||
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.warn(` Eurostat ${geoCode}/${ds.id}: HTTP ${resp.status}`);
|
||||
return null;
|
||||
}
|
||||
const data = await resp.json();
|
||||
const parsed = parseEurostatSeries(data, geoCode);
|
||||
if (!parsed || parsed.value === null) return null;
|
||||
return { ...parsed, unit: ds.unit };
|
||||
} catch (err) {
|
||||
console.warn(` Eurostat ${geoCode}/${ds.id}: ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all EU geos for a single dataset in small batches.
|
||||
* @param {{id:string, params:object, unit:string, label:string, sparklineLength?:number}} ds
|
||||
* @returns {Promise<{countries:object, seededAt:number}>}
|
||||
*/
|
||||
export async function fetchEurostatAllGeos(ds) {
|
||||
const BATCH_SIZE = 4;
|
||||
const results = {};
|
||||
let ok = 0;
|
||||
|
||||
for (let i = 0; i < EU_GEOS.length; i += BATCH_SIZE) {
|
||||
const batch = EU_GEOS.slice(i, i + BATCH_SIZE);
|
||||
const batchResults = await Promise.allSettled(
|
||||
batch.map((g) => fetchEurostatCountry(ds, g).then((v) => ({ g, v })))
|
||||
);
|
||||
for (const r of batchResults) {
|
||||
if (r.status !== 'fulfilled' || !r.value.v) continue;
|
||||
const { g, v } = r.value;
|
||||
// Trim series to sparkline length if configured.
|
||||
const sparkLen = ds.sparklineLength || v.series.length;
|
||||
results[g] = {
|
||||
value: v.value,
|
||||
priorValue: v.priorValue,
|
||||
hasPrior: v.priorValue !== null,
|
||||
date: v.date,
|
||||
unit: v.unit,
|
||||
series: v.series.slice(-sparkLen),
|
||||
};
|
||||
ok++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Eurostat ${ds.id}: ${ok}/${EU_GEOS.length} geos with data`);
|
||||
return { countries: results, seededAt: Date.now(), dataset: ds.id, label: ds.label };
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard validator — at least N countries must have data.
|
||||
*/
|
||||
export function makeValidator(minCountries = 10) {
|
||||
return (data) => {
|
||||
const count = Object.keys(data?.countries || {}).length;
|
||||
if (count < minCountries) {
|
||||
console.warn(
|
||||
` Validation failed: only ${count} geos with data (need ≥${minCountries})`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
}
|
||||
@@ -5,6 +5,9 @@ await runBundle('macro', [
|
||||
{ label: 'BIS-Data', script: 'seed-bis-data.mjs', seedMetaKey: 'economic:bis', intervalMs: 12 * HOUR, timeoutMs: 300_000 },
|
||||
{ label: 'BLS-Series', script: 'seed-bls-series.mjs', seedMetaKey: 'economic:bls-series', intervalMs: DAY, timeoutMs: 120_000 },
|
||||
{ label: 'Eurostat', script: 'seed-eurostat-country-data.mjs', seedMetaKey: 'economic:eurostat-country-data', intervalMs: DAY, timeoutMs: 300_000 },
|
||||
{ label: 'Eurostat-HousePrices', script: 'seed-eurostat-house-prices.mjs', seedMetaKey: 'economic:eurostat-house-prices', intervalMs: 7 * DAY, timeoutMs: 300_000 },
|
||||
{ label: 'Eurostat-GovDebtQ', script: 'seed-eurostat-gov-debt-q.mjs', seedMetaKey: 'economic:eurostat-gov-debt-q', intervalMs: 2 * DAY, timeoutMs: 300_000 },
|
||||
{ label: 'Eurostat-IndProd', script: 'seed-eurostat-industrial-production.mjs', seedMetaKey: 'economic:eurostat-industrial-production', intervalMs: DAY, timeoutMs: 300_000 },
|
||||
{ label: 'IMF-Macro', script: 'seed-imf-macro.mjs', seedMetaKey: 'economic:imf-macro', intervalMs: 30 * DAY, timeoutMs: 300_000 },
|
||||
{ label: 'National-Debt', script: 'seed-national-debt.mjs', seedMetaKey: 'economic:national-debt', intervalMs: 30 * DAY, timeoutMs: 300_000 },
|
||||
{ label: 'FAO-FFPI', script: 'seed-fao-food-price-index.mjs', seedMetaKey: 'economic:fao-ffpi', intervalMs: DAY, timeoutMs: 120_000 },
|
||||
|
||||
54
scripts/seed-eurostat-gov-debt-q.mjs
Normal file
54
scripts/seed-eurostat-gov-debt-q.mjs
Normal file
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Seeds Eurostat `gov_10q_ggdebt` (General Government gross debt, quarterly %GDP)
|
||||
* for all 27 EU members + EA20 + EU27_2020 aggregates.
|
||||
*
|
||||
* Upgrades the National Debt card from annual (IMF GGXWDG_NGDP) to quarterly
|
||||
* cadence for EU countries. Rest of world continues to use IMF.
|
||||
*
|
||||
* Cadence: quarterly. TTL: 14 days (= cadence + buffer, matches health threshold).
|
||||
*/
|
||||
|
||||
import { loadEnvFile, runSeed } from './_seed-utils.mjs';
|
||||
import { fetchEurostatAllGeos, makeValidator } from './_eurostat-utils.mjs';
|
||||
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const CANONICAL_KEY = 'economic:eurostat:gov-debt-q:v1';
|
||||
const TTL = 60 * 60 * 24 * 14; // 14 days (quarterly)
|
||||
|
||||
const DATASET = {
|
||||
id: 'gov_10q_ggdebt',
|
||||
// Percentage of GDP, general government (S.13), gross Maastricht debt
|
||||
params: {
|
||||
unit: 'PC_GDP',
|
||||
sector: 'S13',
|
||||
na_item: 'GD',
|
||||
},
|
||||
unit: '% of GDP',
|
||||
label: 'Government gross debt (quarterly, % of GDP)',
|
||||
// Show last 8 quarters (2 years) for sparkline.
|
||||
sparklineLength: 8,
|
||||
};
|
||||
|
||||
async function fetchAll() {
|
||||
return fetchEurostatAllGeos(DATASET);
|
||||
}
|
||||
|
||||
if (process.argv[1]?.endsWith('seed-eurostat-gov-debt-q.mjs')) {
|
||||
runSeed('economic', 'eurostat-gov-debt-q', CANONICAL_KEY, fetchAll, {
|
||||
// Near-complete coverage: quarterly Maastricht gross-debt is reported by
|
||||
// all 27 EU members + EA20/EU27_2020 aggregates; allow up to ~5 of 29 geos
|
||||
// missing (24/29) before refusing to publish.
|
||||
validateFn: makeValidator(24),
|
||||
ttlSeconds: TTL,
|
||||
sourceVersion: 'eurostat-gov-10q-ggdebt-v1',
|
||||
recordCount: (data) => Object.keys(data?.countries || {}).length,
|
||||
}).catch((err) => {
|
||||
const cause = err.cause
|
||||
? ` (cause: ${err.cause.message || err.cause.code || err.cause})`
|
||||
: '';
|
||||
console.error('FATAL:', (err.message || err) + cause);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
50
scripts/seed-eurostat-house-prices.mjs
Normal file
50
scripts/seed-eurostat-house-prices.mjs
Normal file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Seeds Eurostat `prc_hpi_a` (House price index, annual) for all 27 EU members
|
||||
* + EA20 + EU27_2020 aggregates.
|
||||
*
|
||||
* Complements BIS WS_SPP (#3026) — Eurostat provides full EU coverage where
|
||||
* BIS is sparse. Renders in the shared Housing cycle tile.
|
||||
*
|
||||
* Cadence: annual. TTL: 35 days (cadence + 5d buffer).
|
||||
*/
|
||||
|
||||
import { loadEnvFile, runSeed } from './_seed-utils.mjs';
|
||||
import { fetchEurostatAllGeos, makeValidator } from './_eurostat-utils.mjs';
|
||||
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const CANONICAL_KEY = 'economic:eurostat:house-prices:v1';
|
||||
const TTL = 60 * 60 * 24 * 35; // 35 days (annual)
|
||||
|
||||
const DATASET = {
|
||||
id: 'prc_hpi_a',
|
||||
// purchases of newly built + existing dwellings, index base 2015=100
|
||||
params: { unit: 'I15_A_AVG', purchase: 'TOTAL' },
|
||||
unit: 'index (2015=100)',
|
||||
label: 'House price index (annual, 2015=100)',
|
||||
// Show last 10 years for sparkline.
|
||||
sparklineLength: 10,
|
||||
};
|
||||
|
||||
async function fetchAll() {
|
||||
return fetchEurostatAllGeos(DATASET);
|
||||
}
|
||||
|
||||
if (process.argv[1]?.endsWith('seed-eurostat-house-prices.mjs')) {
|
||||
runSeed('economic', 'eurostat-house-prices', CANONICAL_KEY, fetchAll, {
|
||||
// Near-complete coverage: annual house-price index is well-reported across
|
||||
// all 27 EU members; allow up to ~5 of 29 geos missing (24/29) before we
|
||||
// refuse to publish a snapshot that would silently lose most of the EU.
|
||||
validateFn: makeValidator(24),
|
||||
ttlSeconds: TTL,
|
||||
sourceVersion: 'eurostat-prc-hpi-a-v1',
|
||||
recordCount: (data) => Object.keys(data?.countries || {}).length,
|
||||
}).catch((err) => {
|
||||
const cause = err.cause
|
||||
? ` (cause: ${err.cause.message || err.cause.code || err.cause})`
|
||||
: '';
|
||||
console.error('FATAL:', (err.message || err) + cause);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
61
scripts/seed-eurostat-industrial-production.mjs
Normal file
61
scripts/seed-eurostat-industrial-production.mjs
Normal file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Seeds Eurostat `sts_inpr_m` (Industrial production index, monthly) for all
|
||||
* 27 EU members + EA20 + EU27_2020 aggregates.
|
||||
*
|
||||
* Monthly leading indicator of real-economy activity; not currently surfaced
|
||||
* elsewhere. Renders as the "Real economy pulse" sparkline on the Economic
|
||||
* Indicators card (monthly cadence badge).
|
||||
*
|
||||
* Cadence: monthly. TTL: 5 days (covers re-seed + daily retry cadence).
|
||||
*/
|
||||
|
||||
import { loadEnvFile, runSeed } from './_seed-utils.mjs';
|
||||
import { fetchEurostatAllGeos, makeValidator } from './_eurostat-utils.mjs';
|
||||
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const CANONICAL_KEY = 'economic:eurostat:industrial-production:v1';
|
||||
const TTL = 60 * 60 * 24 * 5; // 5 days (monthly)
|
||||
|
||||
const DATASET = {
|
||||
id: 'sts_inpr_m',
|
||||
// Seasonally + calendar adjusted, index (2021=100), NACE B-D (industry excl. construction)
|
||||
params: {
|
||||
unit: 'I21',
|
||||
s_adj: 'SCA',
|
||||
nace_r2: 'B-D',
|
||||
},
|
||||
unit: 'index (2021=100)',
|
||||
label: 'Industrial production index (monthly, SCA, 2021=100)',
|
||||
// Show last 12 months for sparkline.
|
||||
sparklineLength: 12,
|
||||
};
|
||||
|
||||
async function fetchAll() {
|
||||
return fetchEurostatAllGeos(DATASET);
|
||||
}
|
||||
|
||||
if (process.argv[1]?.endsWith('seed-eurostat-industrial-production.mjs')) {
|
||||
runSeed(
|
||||
'economic',
|
||||
'eurostat-industrial-production',
|
||||
CANONICAL_KEY,
|
||||
fetchAll,
|
||||
{
|
||||
// Monthly industrial-production has slightly patchier coverage than the
|
||||
// annual/quarterly datasets (small members lag or skip months); require
|
||||
// at least 22/29 geos so a bad run can't silently drop most of the EU.
|
||||
validateFn: makeValidator(22),
|
||||
ttlSeconds: TTL,
|
||||
sourceVersion: 'eurostat-sts-inpr-m-v1',
|
||||
recordCount: (data) => Object.keys(data?.countries || {}).length,
|
||||
},
|
||||
).catch((err) => {
|
||||
const cause = err.cause
|
||||
? ` (cause: ${err.cause.message || err.cause.code || err.cause})`
|
||||
: '';
|
||||
console.error('FATAL:', (err.message || err) + cause);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -186,6 +186,9 @@ export const BOOTSTRAP_CACHE_KEYS: Record<string, string> = {
|
||||
ecbFxRates: 'economic:ecb-fx-rates:v1',
|
||||
euGasStorage: 'economic:eu-gas-storage:v1',
|
||||
eurostatCountryData: 'economic:eurostat-country-data:v1',
|
||||
eurostatHousePrices: 'economic:eurostat:house-prices:v1',
|
||||
eurostatGovDebtQ: 'economic:eurostat:gov-debt-q:v1',
|
||||
eurostatIndProd: 'economic:eurostat:industrial-production:v1',
|
||||
euFsi: 'economic:fsi-eu:v1',
|
||||
shippingStress: 'supply_chain:shipping_stress:v1',
|
||||
socialVelocity: 'intelligence:social:reddit:v1',
|
||||
@@ -249,6 +252,9 @@ export const BOOTSTRAP_TIERS: Record<string, 'slow' | 'fast'> = {
|
||||
ecbFxRates: 'slow',
|
||||
euGasStorage: 'slow',
|
||||
eurostatCountryData: 'slow',
|
||||
eurostatHousePrices: 'slow',
|
||||
eurostatGovDebtQ: 'slow',
|
||||
eurostatIndProd: 'slow',
|
||||
euFsi: 'slow',
|
||||
shippingStress: 'fast',
|
||||
socialVelocity: 'fast',
|
||||
|
||||
@@ -253,7 +253,7 @@ describe('Bootstrap key hydration coverage', () => {
|
||||
const allSrc = srcFiles.map(f => readFileSync(f, 'utf-8')).join('\n');
|
||||
|
||||
// Keys with planned but not-yet-wired consumers
|
||||
const PENDING_CONSUMERS = new Set(['correlationCards', 'euGasStorage', 'chokepointBaselines', 'imfMacro', 'imfGrowth', 'imfLabor', 'imfExternal', 'portwatchChokepointsRef', 'portwatchPortActivity', 'sprPolicies', 'wsbTickers', 'electricityPrices', 'jodiOil']);
|
||||
const PENDING_CONSUMERS = new Set(['correlationCards', 'euGasStorage', 'chokepointBaselines', 'imfMacro', 'imfGrowth', 'imfLabor', 'imfExternal', 'portwatchChokepointsRef', 'portwatchPortActivity', 'sprPolicies', 'wsbTickers', 'electricityPrices', 'jodiOil', 'eurostatHousePrices', 'eurostatGovDebtQ', 'eurostatIndProd']);
|
||||
for (const key of keys) {
|
||||
if (PENDING_CONSUMERS.has(key)) continue;
|
||||
assert.ok(
|
||||
|
||||
212
tests/eurostat-seeders.test.mjs
Normal file
212
tests/eurostat-seeders.test.mjs
Normal file
@@ -0,0 +1,212 @@
|
||||
/**
|
||||
* Tests for issue #3028 Eurostat overlay seeders:
|
||||
* - prc_hpi_a (house prices, annual)
|
||||
* - gov_10q_ggdebt (gov debt, quarterly)
|
||||
* - sts_inpr_m (industrial production, monthly)
|
||||
*
|
||||
* Covers:
|
||||
* - JSON-stat parser (single-country extraction, series ordering, Greece/EA20 quirks)
|
||||
* - EU-only coverage gating (non-EU geos return null so panels don't render blanks)
|
||||
* - Registry wiring (bootstrap + health + MCP)
|
||||
*/
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
import {
|
||||
parseEurostatSeries,
|
||||
makeValidator,
|
||||
EU_GEOS,
|
||||
} from '../scripts/_eurostat-utils.mjs';
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
/**
|
||||
* Build a minimal JSON-stat v2 response for testing, with 2 geos and 3 time periods.
|
||||
*/
|
||||
function jsonStatFixture({ geos = ['DE', 'FR'], times = ['2022', '2023', '2024'], values }) {
|
||||
const geoIndex = {};
|
||||
geos.forEach((g, i) => { geoIndex[g] = i; });
|
||||
const timeIndex = {};
|
||||
times.forEach((t, i) => { timeIndex[t] = i; });
|
||||
// Build flat value object indexed by (geo_pos * times.length + time_pos).
|
||||
const value = {};
|
||||
geos.forEach((_, gi) => {
|
||||
times.forEach((_, ti) => {
|
||||
const idx = gi * times.length + ti;
|
||||
const v = values[gi]?.[ti];
|
||||
if (v !== undefined && v !== null) value[idx] = v;
|
||||
});
|
||||
});
|
||||
return {
|
||||
id: ['geo', 'time'],
|
||||
size: [geos.length, times.length],
|
||||
dimension: {
|
||||
geo: { category: { index: geoIndex } },
|
||||
time: { category: { index: timeIndex } },
|
||||
},
|
||||
value,
|
||||
};
|
||||
}
|
||||
|
||||
describe('Eurostat JSON-stat parser (issue #3028)', () => {
|
||||
it('extracts a sorted series plus latest/prior for the requested geo', () => {
|
||||
const data = jsonStatFixture({
|
||||
geos: ['DE', 'FR'],
|
||||
times: ['2022', '2023', '2024'],
|
||||
values: [
|
||||
[120.5, 125.8, 128.1], // DE
|
||||
[115.0, 118.3, 121.7], // FR
|
||||
],
|
||||
});
|
||||
|
||||
const de = parseEurostatSeries(data, 'DE');
|
||||
assert.ok(de, 'DE parse should succeed');
|
||||
assert.equal(de.value, 128.1);
|
||||
assert.equal(de.priorValue, 125.8);
|
||||
assert.equal(de.date, '2024');
|
||||
assert.equal(de.series.length, 3);
|
||||
assert.deepEqual(
|
||||
de.series.map((p) => p.date),
|
||||
['2022', '2023', '2024'],
|
||||
'series must be sorted ascending by period',
|
||||
);
|
||||
|
||||
const fr = parseEurostatSeries(data, 'FR');
|
||||
assert.equal(fr.value, 121.7, 'FR must have its own value, not DE mixed in');
|
||||
});
|
||||
|
||||
it('returns null for a geo that is not in the response (non-EU gating)', () => {
|
||||
const data = jsonStatFixture({
|
||||
geos: ['DE', 'FR'],
|
||||
times: ['2024'],
|
||||
values: [[100.0], [101.0]],
|
||||
});
|
||||
// US is not an EU geo — panels must not render blank tiles for it.
|
||||
assert.equal(parseEurostatSeries(data, 'US'), null);
|
||||
assert.equal(parseEurostatSeries(data, 'JP'), null);
|
||||
});
|
||||
|
||||
it('handles the Greece EL quirk (not ISO GR) and EA20 aggregate', () => {
|
||||
const data = jsonStatFixture({
|
||||
geos: ['EL', 'EA20'],
|
||||
times: ['2023-Q3', '2023-Q4', '2024-Q1'],
|
||||
values: [
|
||||
[168.1, 167.5, 166.9],
|
||||
[90.1, 89.8, 89.2],
|
||||
],
|
||||
});
|
||||
const el = parseEurostatSeries(data, 'EL');
|
||||
assert.ok(el, 'Greece must parse under EL, not GR');
|
||||
assert.equal(el.value, 166.9);
|
||||
assert.equal(parseEurostatSeries(data, 'GR'), null, 'ISO GR must not resolve to Greece');
|
||||
|
||||
const ea = parseEurostatSeries(data, 'EA20');
|
||||
assert.ok(ea, 'Euro Area EA20 aggregate must parse');
|
||||
assert.equal(ea.value, 89.2);
|
||||
});
|
||||
|
||||
it('skips null observations and picks the latest non-null value', () => {
|
||||
const data = jsonStatFixture({
|
||||
geos: ['IT'],
|
||||
times: ['2024-01', '2024-02', '2024-03'],
|
||||
values: [[100.0, 101.5, null]],
|
||||
});
|
||||
const it = parseEurostatSeries(data, 'IT');
|
||||
assert.ok(it);
|
||||
assert.equal(it.value, 101.5, 'latest should skip null trailing observation');
|
||||
assert.equal(it.date, '2024-02');
|
||||
assert.equal(it.series.length, 2, 'null observations are dropped from series');
|
||||
});
|
||||
|
||||
it('returns null on malformed / empty responses', () => {
|
||||
assert.equal(parseEurostatSeries(null, 'DE'), null);
|
||||
assert.equal(parseEurostatSeries({}, 'DE'), null);
|
||||
assert.equal(parseEurostatSeries({ value: {} }, 'DE'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('EU coverage list (issue #3028)', () => {
|
||||
it('covers all 27 EU members plus EA20 and EU27_2020 aggregates', () => {
|
||||
assert.equal(EU_GEOS.length, 29);
|
||||
// Spot-check the ones called out explicitly in the issue.
|
||||
for (const g of ['IE', 'PT', 'EL', 'HU', 'RO', 'DK', 'FI', 'BG', 'SK', 'SI', 'LT', 'LV', 'EE', 'LU', 'HR', 'MT', 'CY']) {
|
||||
assert.ok(EU_GEOS.includes(g), `EU_GEOS must include ${g}`);
|
||||
}
|
||||
assert.ok(EU_GEOS.includes('EA20'), 'EU_GEOS must include EA20 (post-2023 Euro Area)');
|
||||
assert.ok(EU_GEOS.includes('EU27_2020'), 'EU_GEOS must include EU27_2020 aggregate');
|
||||
assert.ok(!EU_GEOS.includes('GR'), 'EU_GEOS must use EL not ISO GR for Greece');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Seeder validator (issue #3028)', () => {
|
||||
it('rejects payloads below the minimum country threshold', () => {
|
||||
const validate = makeValidator(10);
|
||||
assert.equal(validate({ countries: {} }), false);
|
||||
assert.equal(validate({ countries: { DE: {}, FR: {}, IT: {} } }), false);
|
||||
const big = Object.fromEntries(EU_GEOS.slice(0, 15).map((g) => [g, {}]));
|
||||
assert.equal(validate({ countries: big }), true);
|
||||
});
|
||||
|
||||
it('each Eurostat overlay seeder enforces near-complete EU coverage', async () => {
|
||||
// Guard against regressions: a bad Eurostat run that returns only a
|
||||
// handful of geos must NOT be accepted as a valid snapshot. Universe is
|
||||
// fixed at 29 (EU27 + EA20 + EU27_2020); require >=22 geos across all
|
||||
// three seeders so no seeder can publish a snapshot missing most of the EU.
|
||||
const files = [
|
||||
'scripts/seed-eurostat-house-prices.mjs',
|
||||
'scripts/seed-eurostat-gov-debt-q.mjs',
|
||||
'scripts/seed-eurostat-industrial-production.mjs',
|
||||
];
|
||||
for (const rel of files) {
|
||||
const src = await readFile(resolve(ROOT, rel), 'utf8');
|
||||
const match = src.match(/makeValidator\((\d+)\)/);
|
||||
assert.ok(match, `${rel} must call makeValidator(N)`);
|
||||
const n = Number(match[1]);
|
||||
assert.ok(
|
||||
n >= 22,
|
||||
`${rel} makeValidator threshold ${n} too low — EU universe is 29, must be >=22 to catch partial-coverage failures`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Registry wiring (issue #3028)', () => {
|
||||
it('bootstrap.js exposes the three new Eurostat overlay keys', async () => {
|
||||
const src = await readFile(resolve(ROOT, 'api/bootstrap.js'), 'utf8');
|
||||
assert.match(src, /eurostatHousePrices:\s*'economic:eurostat:house-prices:v1'/);
|
||||
assert.match(src, /eurostatGovDebtQ:\s*'economic:eurostat:gov-debt-q:v1'/);
|
||||
assert.match(src, /eurostatIndProd:\s*'economic:eurostat:industrial-production:v1'/);
|
||||
// Must also be registered in the SLOW_KEYS tier.
|
||||
assert.match(src, /'eurostatHousePrices'/);
|
||||
assert.match(src, /'eurostatGovDebtQ'/);
|
||||
assert.match(src, /'eurostatIndProd'/);
|
||||
});
|
||||
|
||||
it('health.js maps each new key to a seed-meta freshness check', async () => {
|
||||
const src = await readFile(resolve(ROOT, 'api/health.js'), 'utf8');
|
||||
assert.match(src, /eurostatHousePrices:\s*\{[^}]*seed-meta:economic:eurostat-house-prices/);
|
||||
assert.match(src, /eurostatGovDebtQ:\s*\{[^}]*seed-meta:economic:eurostat-gov-debt-q/);
|
||||
assert.match(src, /eurostatIndProd:\s*\{[^}]*seed-meta:economic:eurostat-industrial-production/);
|
||||
});
|
||||
|
||||
it('MCP tool registry exposes the three new EU overlay tools', async () => {
|
||||
const src = await readFile(resolve(ROOT, 'api/mcp.ts'), 'utf8');
|
||||
assert.match(src, /name: 'get_eu_housing_cycle'/);
|
||||
assert.match(src, /name: 'get_eu_quarterly_gov_debt'/);
|
||||
assert.match(src, /name: 'get_eu_industrial_production'/);
|
||||
assert.match(src, /'economic:eurostat:house-prices:v1'/);
|
||||
assert.match(src, /'economic:eurostat:gov-debt-q:v1'/);
|
||||
assert.match(src, /'economic:eurostat:industrial-production:v1'/);
|
||||
});
|
||||
|
||||
it('macro bundle runner includes the three new scripts with distinct seed-meta keys', async () => {
|
||||
const src = await readFile(resolve(ROOT, 'scripts/seed-bundle-macro.mjs'), 'utf8');
|
||||
assert.match(src, /seed-eurostat-house-prices\.mjs/);
|
||||
assert.match(src, /seed-eurostat-gov-debt-q\.mjs/);
|
||||
assert.match(src, /seed-eurostat-industrial-production\.mjs/);
|
||||
});
|
||||
});
|
||||
@@ -118,12 +118,12 @@ describe('api/mcp.ts — PRO MCP Server', () => {
|
||||
|
||||
// --- tools/list ---
|
||||
|
||||
it('tools/list returns 29 tools with name, description, inputSchema', async () => {
|
||||
it('tools/list returns 32 tools with name, description, inputSchema', async () => {
|
||||
const res = await handler(makeReq('POST', { jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }));
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.ok(Array.isArray(body.result?.tools), 'result.tools must be an array');
|
||||
assert.equal(body.result.tools.length, 29, `Expected 29 tools, got ${body.result.tools.length}`);
|
||||
assert.equal(body.result.tools.length, 32, `Expected 32 tools, got ${body.result.tools.length}`);
|
||||
for (const tool of body.result.tools) {
|
||||
assert.ok(tool.name, 'tool.name must be present');
|
||||
assert.ok(tool.description, 'tool.description must be present');
|
||||
|
||||
Reference in New Issue
Block a user