diff --git a/api/health.js b/api/health.js index 088076d92..ca0401f67 100644 --- a/api/health.js +++ b/api/health.js @@ -129,6 +129,7 @@ const STANDALONE_KEYS = { climateNews: 'climate:news-intelligence:v1', pizzint: 'intelligence:pizzint:seed:v1', resilienceStaticIndex: 'resilience:static:index:v1', + energyExposure: 'energy:exposure:v1:index', }; const SEED_META = { @@ -240,6 +241,7 @@ const SEED_META = { vpdTrackerRealtime: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // daily seed (0 2 * * *); 2880min = 48h = 2x interval vpdTrackerHistorical: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // shares seed-meta key with vpdTrackerRealtime (same run) resilienceStaticIndex: { key: 'seed-meta:resilience:static', maxStaleMin: 576000 }, // annual October snapshot; 400d threshold matches TTL and preserves prior-year data on source outages + energyExposure: { key: 'seed-meta:economic:owid-energy-mix', maxStaleMin: 50400 }, // monthly cron on 1st; 50400min = 35d = TTL matches cron cadence + 5d buffer }; // Standalone keys that are populated on-demand by RPC handlers (not seeds). diff --git a/scripts/seed-owid-energy-mix.mjs b/scripts/seed-owid-energy-mix.mjs new file mode 100644 index 000000000..3332c06b5 --- /dev/null +++ b/scripts/seed-owid-energy-mix.mjs @@ -0,0 +1,339 @@ +#!/usr/bin/env node + +import { + acquireLockSafely, + CHROME_UA, + extendExistingTtl, + getRedisCredentials, + loadEnvFile, + logSeedResult, + releaseLock, + withRetry, +} from './_seed-utils.mjs'; +import { resolveIso2 } from './_country-resolver.mjs'; + +loadEnvFile(import.meta.url); + +export const OWID_ENERGY_MIX_KEY_PREFIX = 'energy:mix:v1:'; +export const OWID_EXPOSURE_INDEX_KEY = 'energy:exposure:v1:index'; +/** Full list of ISO2 codes written in the last successful run — used by the + * failure-preservation path to extend TTL on ALL per-country keys, not just + * those that happen to appear in the top-20 exposure buckets. */ +export const OWID_COUNTRY_LIST_KEY = 'energy:mix:v1:_countries'; +export const OWID_META_KEY = 'seed-meta:economic:owid-energy-mix'; +export const OWID_TTL_SECONDS = 35 * 24 * 3600; +const OWID_CSV_URL = 'https://owid-public.owid.io/data/energy/owid-energy-data.csv'; +const LOCK_DOMAIN = 'economic:owid-energy-mix'; +const LOCK_TTL_MS = 30 * 60 * 1000; +const MIN_COUNTRIES = 150; +const MAX_DROP_PCT = 15; + +const COLS = { + coal: 'coal_share_elec', + gas: 'gas_share_elec', + oil: 'oil_share_elec', + nuclear: 'nuclear_share_elec', + renewables: 'renewables_share_elec', + wind: 'wind_share_elec', + solar: 'solar_share_elec', + hydro: 'hydro_share_elec', + imports: 'net_energy_imports', +}; + +function parseDelimitedRow(line, delimiter) { + const cells = []; + let current = ''; + let inQuotes = false; + + for (let idx = 0; idx < line.length; idx += 1) { + const char = line[idx]; + const next = line[idx + 1]; + + if (char === '"') { + if (inQuotes && next === '"') { + current += '"'; + idx += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + + if (char === delimiter && !inQuotes) { + cells.push(current); + current = ''; + continue; + } + + current += char; + } + + cells.push(current); + return cells.map((cell) => cell.trim()); +} + +function parseDelimitedText(text, delimiter) { + const lines = text + .replace(/^\uFEFF/, '') + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (lines.length < 2) return []; + + const headers = parseDelimitedRow(lines[0], delimiter); + return lines.slice(1).map((line) => { + const values = parseDelimitedRow(line, delimiter); + return Object.fromEntries(headers.map((header, idx) => [header, values[idx] ?? ''])); + }); +} + +function safeFloat(value) { + const n = parseFloat(value); + return Number.isFinite(n) ? n : null; +} + +function hasAnyShareField(row) { + return Object.values(COLS).some((col) => { + const v = parseFloat(row[col]); + return Number.isFinite(v); + }); +} + +export function parseOwidCsv(csvText) { + const rows = parseDelimitedText(csvText, ','); + if (rows.length === 0) throw new Error('OWID CSV: no data rows'); + + const headers = Object.keys(rows[0] || {}); + if (!headers.includes(COLS.coal)) { + throw new Error('OWID column schema changed — update COLS mapping'); + } + + const byCountry = new Map(); + for (const row of rows) { + const iso3 = String(row.iso_code || '').trim(); + if (iso3.startsWith('OWID_')) continue; + if (!iso3) continue; + + const year = parseInt(row.year, 10); + if (!Number.isFinite(year)) continue; + if (!hasAnyShareField(row)) continue; + + const iso2 = resolveIso2({ iso3 }); + if (!iso2) continue; + + const prev = byCountry.get(iso2); + if (prev && prev.year >= year) continue; + + byCountry.set(iso2, { + iso2, + country: row.country || iso2, + year, + coalShare: safeFloat(row[COLS.coal]), + gasShare: safeFloat(row[COLS.gas]), + oilShare: safeFloat(row[COLS.oil]), + nuclearShare: safeFloat(row[COLS.nuclear]), + renewShare: safeFloat(row[COLS.renewables]), + windShare: safeFloat(row[COLS.wind]), + solarShare: safeFloat(row[COLS.solar]), + hydroShare: safeFloat(row[COLS.hydro]), + importShare: safeFloat(row[COLS.imports]), + seededAt: new Date().toISOString(), + }); + } + + return byCountry; +} + +export function buildExposureIndex(countries) { + const all = [...countries.values()]; + + // Each bucket filters only on its own metric so countries with valid + // oil/import/renewables data but no gas/coal value are not excluded. + const top20 = (key) => + all + .filter((c) => c[key] != null) + .sort((a, b) => b[key] - a[key]) + .slice(0, 20) + .map((c) => ({ iso2: c.iso2, name: c.country, share: c[key] })); + + const years = all.map((c) => c.year).filter(Boolean); + + return { + updatedAt: new Date().toISOString(), + year: years.length > 0 ? Math.max(...years) : null, + gas: top20('gasShare'), + coal: top20('coalShare'), + oil: top20('oilShare'), + imported: top20('importShare'), + renewable:top20('renewShare'), + }; +} + +async function redisPipeline(commands) { + const { url, token } = getRedisCredentials(); + const response = await fetch(`${url}/pipeline`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(commands), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`Redis pipeline failed: HTTP ${response.status} — ${text.slice(0, 200)}`); + } + return response.json(); +} + +async function redisGet(key) { + const { url, token } = getRedisCredentials(); + const resp = await fetch(`${url}/get/${encodeURIComponent(key)}`, { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(5_000), + }); + if (!resp.ok) return null; + const data = await resp.json(); + return data.result ? JSON.parse(data.result) : null; +} + +async function preservePreviousSnapshot(errorMsg) { + console.error('[owid-energy-mix] Preserving previous snapshot:', errorMsg); + + // Read the full country list written by the last successful run. + // This covers ALL seeded ISO2 codes, including countries that do not appear + // in any top-20 fuel bucket in the exposure index. + const countryList = await redisGet(OWID_COUNTRY_LIST_KEY).catch(() => null); + const perCountryKeys = Array.isArray(countryList) + ? countryList.map((iso2) => `${OWID_ENERGY_MIX_KEY_PREFIX}${iso2}`) + : []; + + await extendExistingTtl( + [...perCountryKeys, OWID_COUNTRY_LIST_KEY, OWID_EXPOSURE_INDEX_KEY, OWID_META_KEY], + OWID_TTL_SECONDS, + ); + const metaPayload = { + fetchedAt: Date.now(), + recordCount: 0, + sourceVersion: 'owid-energy-mix-v1', + status: 'error', + error: errorMsg, + }; + await redisPipeline([ + ['SET', OWID_META_KEY, JSON.stringify(metaPayload), 'EX', OWID_TTL_SECONDS], + ]); +} + +export async function main() { + const startedAt = Date.now(); + const runId = `owid-energy-mix:${startedAt}`; + const lock = await acquireLockSafely(LOCK_DOMAIN, runId, LOCK_TTL_MS, { label: LOCK_DOMAIN }); + if (lock.skipped) return; + if (!lock.locked) { + console.log('[owid-energy-mix] Lock held, skipping'); + return; + } + + try { + const csvText = await withRetry( + () => + fetch(OWID_CSV_URL, { + headers: { 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(30_000), + }).then((r) => { + if (!r.ok) throw new Error(`OWID HTTP ${r.status}`); + return r.text(); + }), + 2, + 750, + ); + + const countries = parseOwidCsv(csvText); + + if (countries.size < MIN_COUNTRIES) { + throw new Error( + `OWID: only ${countries.size} countries parsed, expected >=${MIN_COUNTRIES}`, + ); + } + + const prevMeta = await redisGet(OWID_META_KEY).catch(() => null); + if (prevMeta && typeof prevMeta === 'object' && prevMeta.recordCount > 0) { + const drop = + ((prevMeta.recordCount - countries.size) / prevMeta.recordCount) * 100; + if (drop > MAX_DROP_PCT) { + throw new Error( + `OWID: country count dropped ${drop.toFixed(1)}% vs previous ${prevMeta.recordCount}`, + ); + } + } + + const exposureIndex = buildExposureIndex(countries); + const metaPayload = { + fetchedAt: Date.now(), + recordCount: countries.size, + sourceVersion: 'owid-energy-mix-v1', + }; + + const commands = []; + for (const [iso2, payload] of countries) { + commands.push([ + 'SET', + `${OWID_ENERGY_MIX_KEY_PREFIX}${iso2}`, + JSON.stringify(payload), + 'EX', + OWID_TTL_SECONDS, + ]); + } + commands.push([ + 'SET', + OWID_EXPOSURE_INDEX_KEY, + JSON.stringify(exposureIndex), + 'EX', + OWID_TTL_SECONDS, + ]); + // Full ISO2 list — used by failure-preservation path to extend TTL on + // ALL per-country keys, including countries outside the top-20 fuel buckets. + commands.push([ + 'SET', + OWID_COUNTRY_LIST_KEY, + JSON.stringify([...countries.keys()]), + 'EX', + OWID_TTL_SECONDS, + ]); + commands.push([ + 'SET', + OWID_META_KEY, + JSON.stringify(metaPayload), + 'EX', + OWID_TTL_SECONDS, // must outlive the monthly cron interval (35 days) + ]); + + const results = await redisPipeline(commands); + const failures = results.filter((r) => r?.error || r?.result === 'ERR'); + if (failures.length > 0) { + throw new Error( + `Redis pipeline: ${failures.length}/${commands.length} commands failed`, + ); + } + + logSeedResult('economic:owid-energy-mix', countries.size, Date.now() - startedAt, { + exposureYear: exposureIndex.year, + }); + console.log(`[owid-energy-mix] Seeded ${countries.size} countries`); + } catch (err) { + await preservePreviousSnapshot(String(err)).catch((e) => + console.error('[owid-energy-mix] Failed to preserve snapshot:', e), + ); + throw err; + } finally { + await releaseLock(LOCK_DOMAIN, runId); + } +} + +if (process.argv[1]?.endsWith('seed-owid-energy-mix.mjs')) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index ca89dd091..dadf19f8c 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -47,6 +47,9 @@ export const CLIMATE_OCEAN_ICE_KEY = 'climate:ocean-ice:v1'; export const CLIMATE_NEWS_KEY = 'climate:news-intelligence:v1'; export const HEALTH_AIR_QUALITY_KEY = 'health:air-quality:v1'; +export const ENERGY_MIX_KEY_PREFIX = 'energy:mix:v1:'; +export const ENERGY_EXPOSURE_INDEX_KEY = 'energy:exposure:v1:index'; + /** * Static cache keys for the bootstrap endpoint. * Only keys with NO request-varying suffixes are included. diff --git a/server/worldmonitor/intelligence/v1/chat-analyst-context.ts b/server/worldmonitor/intelligence/v1/chat-analyst-context.ts index 4ff0f1072..7fc7c6598 100644 --- a/server/worldmonitor/intelligence/v1/chat-analyst-context.ts +++ b/server/worldmonitor/intelligence/v1/chat-analyst-context.ts @@ -27,6 +27,7 @@ export interface AnalystContext { countryBrief: string; liveHeadlines: string; relevantArticles: string; + energyExposure: string; activeSources: string[]; degraded: boolean; } @@ -201,6 +202,34 @@ function buildPredictionMarkets(data: unknown): string { return lines.length ? `Prediction Markets:\n${lines.join('\n')}` : ''; } +function buildEnergyExposure(data: unknown): string { + if (!data || typeof data !== 'object') return ''; + const d = data as Record; + const year = typeof d.year === 'number' ? d.year : ''; + const lines: string[] = [`Energy Generation Mix — ${year || 'recent'} data:`]; + + const fuelLabels: Array<[string, string]> = [ + ['gas', 'Gas-dependent (% electricity from gas)'], + ['coal', 'Coal-dependent'], + ['oil', 'Oil-dependent'], + ['imported', 'Net energy importers (% demand)'], + ['renewable', 'Renewables-insulated'], + ]; + + for (const [fuel, label] of fuelLabels) { + const entries = Array.isArray(d[fuel]) + ? (d[fuel] as Array>).slice(0, 8) + : []; + if (!entries.length) continue; + const formatted = entries + .map((e) => `${safeStr(e.name)} ${typeof e.share === 'number' ? e.share.toFixed(0) : '?'}%`) + .join(', '); + lines.push(`${label}: ${formatted}`); + } + lines.push('(Gas figures are total gas mix; LNG vs. pipeline split not in this dataset.)'); + return lines.join('\n'); +} + function buildCountryBrief(data: unknown): string { if (!data || typeof data !== 'object') return ''; const d = data as Record; @@ -387,6 +416,7 @@ const SOURCE_LABELS: Array<[keyof Omit 0 ? searchDigestByKeywords(keywords) : Promise.resolve(''), @@ -448,9 +486,13 @@ export async function assembleAnalystContext( const getStr = (r: PromiseSettledResult): string => r.status === 'fulfilled' && typeof r.value === 'string' ? r.value : ''; - const failCount = [insightsResult, riskResult, marketImplResult, forecastsResult, - stocksResult, commoditiesResult, macroResult, predResult] - .filter((r) => r.status === 'rejected' || !r.value).length; + // energyExposure only counts toward degraded when it was actually fetched. + const coreResults: PromiseSettledResult[] = [ + insightsResult, riskResult, marketImplResult, forecastsResult, + stocksResult, commoditiesResult, macroResult, predResult, + ]; + if (needsEnergyExposure) coreResults.push(energyExposureResult); + const failCount = coreResults.filter((r) => r.status === 'rejected' || !r.value).length; const ctx: AnalystContext = { timestamp: new Date().toUTCString(), @@ -460,6 +502,7 @@ export async function assembleAnalystContext( forecasts: buildForecasts(get(forecastsResult)), marketData: buildMarketData(get(stocksResult), get(commoditiesResult)), macroSignals: buildMacroSignals(get(macroResult)), + energyExposure: buildEnergyExposure(get(energyExposureResult)), predictionMarkets: buildPredictionMarkets(get(predResult)), countryBrief: buildCountryBrief(get(countryResult)), liveHeadlines: getStr(headlinesResult), diff --git a/server/worldmonitor/intelligence/v1/chat-analyst-prompt.ts b/server/worldmonitor/intelligence/v1/chat-analyst-prompt.ts index 0a43b0723..27d8bea24 100644 --- a/server/worldmonitor/intelligence/v1/chat-analyst-prompt.ts +++ b/server/worldmonitor/intelligence/v1/chat-analyst-prompt.ts @@ -10,9 +10,9 @@ const DOMAIN_EMPHASIS: Record = { /** Context fields included per domain. 'all' includes everything. */ const DOMAIN_SECTIONS: Record> = { market: new Set(['relevantArticles', 'marketData', 'macroSignals', 'marketImplications', 'predictionMarkets', 'forecasts', 'liveHeadlines']), - geo: new Set(['relevantArticles', 'worldBrief', 'riskScores', 'forecasts', 'predictionMarkets', 'countryBrief', 'liveHeadlines']), + geo: new Set(['relevantArticles', 'worldBrief', 'riskScores', 'forecasts', 'predictionMarkets', 'countryBrief', 'energyExposure', 'liveHeadlines']), military: new Set(['relevantArticles', 'worldBrief', 'riskScores', 'forecasts', 'countryBrief', 'liveHeadlines']), - economic: new Set(['relevantArticles', 'marketData', 'macroSignals', 'marketImplications', 'riskScores', 'liveHeadlines']), + economic: new Set(['relevantArticles', 'marketData', 'macroSignals', 'marketImplications', 'riskScores', 'energyExposure', 'liveHeadlines']), }; export function buildAnalystSystemPrompt(ctx: AnalystContext, domainFocus?: string): string { @@ -42,6 +42,8 @@ export function buildAnalystSystemPrompt(ctx: AnalystContext, domainFocus?: stri contextSections.push(`## ${ctx.marketData}`); if (ctx.macroSignals && include('macroSignals')) contextSections.push(`## ${ctx.macroSignals}`); + if (ctx.energyExposure && include('energyExposure')) + contextSections.push(`## Energy Exposure\n${ctx.energyExposure}`); if (ctx.predictionMarkets && include('predictionMarkets')) contextSections.push(`## ${ctx.predictionMarkets}`); if (ctx.countryBrief && include('countryBrief')) diff --git a/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts b/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts index 948fdf6e1..f834024d3 100644 --- a/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts +++ b/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts @@ -4,7 +4,7 @@ import type { GetCountryIntelBriefResponse, } from '../../../../src/generated/server/worldmonitor/intelligence/v1/service_server'; -import { cachedFetchJson } from '../../../_shared/redis'; +import { cachedFetchJson, getCachedJson } from '../../../_shared/redis'; import { UPSTREAM_TIMEOUT_MS, TIER1_COUNTRIES, sha256Hex } from './_shared'; import { callLlm } from '../../../_shared/llm'; import { isCallerPremium } from '../../../_shared/premium-check'; @@ -38,13 +38,25 @@ export async function getCountryIntelBrief( const isPremium = await isCallerPremium(ctx.request); const frameworkRaw = isPremium && typeof req.framework === 'string' ? req.framework.slice(0, 2000) : ''; + + // Fetch energy mix early so its data-year can be included in the cache key. + // This ensures cached briefs are invalidated when OWID publishes updated annual + // data — without it, energy mix changes are silently ignored in cached briefs. + let energyMixData: Record | null = null; + try { + const raw = await getCachedJson(`energy:mix:v1:${req.countryCode.toUpperCase()}`, true); + if (raw && typeof raw === 'object') energyMixData = raw as Record; + } catch { /* graceful omit */ } + const energyYear = typeof energyMixData?.year === 'number' ? String(energyMixData.year) : ''; + const [contextHashFull, frameworkHashFull] = await Promise.all([ contextSnapshot ? sha256Hex(contextSnapshot) : Promise.resolve('base'), frameworkRaw ? sha256Hex(frameworkRaw) : Promise.resolve(''), ]); const contextHash = contextSnapshot ? contextHashFull.slice(0, 16) : 'base'; const frameworkHash = frameworkRaw ? frameworkHashFull.slice(0, 8) : ''; - const cacheKey = `ci-sebuf:v3:${req.countryCode}:${lang}:${contextHash}${frameworkHash ? `:${frameworkHash}` : ''}`; + const energyTag = energyYear ? `:e${energyYear}` : ''; + const cacheKey = `ci-sebuf:v3:${req.countryCode}:${lang}:${contextHash}${frameworkHash ? `:${frameworkHash}` : ''}${energyTag}`; const countryName = TIER1_COUNTRIES[req.countryCode] || req.countryCode; const dateStr = new Date().toISOString().split('T')[0]; @@ -82,6 +94,16 @@ Rules: - No speculation beyond what data supports.${lang === 'fr' ? '\n- IMPORTANT: You MUST respond ENTIRELY in French language.' : ''}`; const userPromptParts = [`Country: ${countryName} (${req.countryCode})`]; + + if (energyMixData) { + const yr = energyYear || ''; + userPromptParts.push( + `Energy generation mix (${yr}): coal ${energyMixData.coalShare ?? '?'}%, ` + + `gas ${energyMixData.gasShare ?? '?'}%, renewables ${energyMixData.renewShare ?? '?'}%, ` + + `nuclear ${energyMixData.nuclearShare ?? '?'}%, net import dependency ${energyMixData.importShare ?? '?'}%.`, + ); + } + if (contextSnapshot) { userPromptParts.push(`Context snapshot:\n${contextSnapshot}`); } diff --git a/server/worldmonitor/resilience/v1/_dimension-scorers.ts b/server/worldmonitor/resilience/v1/_dimension-scorers.ts index 68cd9fcd9..05fbc5f32 100644 --- a/server/worldmonitor/resilience/v1/_dimension-scorers.ts +++ b/server/worldmonitor/resilience/v1/_dimension-scorers.ts @@ -151,6 +151,7 @@ const RESILIENCE_DISPLACEMENT_PREFIX = 'displacement:summary:v1'; const RESILIENCE_SOCIAL_VELOCITY_KEY = 'intelligence:social:reddit:v1'; const RESILIENCE_NEWS_THREAT_SUMMARY_KEY = 'news:threat:summary:v1'; const RESILIENCE_ENERGY_PRICES_KEY = 'economic:energy:v1:all'; +const RESILIENCE_ENERGY_MIX_KEY_PREFIX = 'energy:mix:v1:'; const COUNTRY_NAME_ALIASES = new Map>(); for (const [name, iso2] of Object.entries(countryNames as Record)) { @@ -697,16 +698,28 @@ export async function scoreEnergy( countryCode: string, reader: ResilienceSeedReader = defaultSeedReader, ): Promise { - const [staticRecord, energyPricesRaw] = await Promise.all([ + const [staticRecord, energyPricesRaw, energyMixRaw] = await Promise.all([ readStaticCountry(countryCode, reader), reader(RESILIENCE_ENERGY_PRICES_KEY), + reader(`${RESILIENCE_ENERGY_MIX_KEY_PREFIX}${countryCode}`), ]); - const dependency = safeNum(staticRecord?.iea?.energyImportDependency?.value); + + const mix = energyMixRaw != null && typeof energyMixRaw === 'object' + ? (energyMixRaw as Record) + : null; + + const dependency = safeNum(staticRecord?.iea?.energyImportDependency?.value); + const gasShare = mix && typeof mix.gasShare === 'number' ? mix.gasShare : null; + const coalShare = mix && typeof mix.coalShare === 'number' ? mix.coalShare : null; + const renewShare = mix && typeof mix.renewShare === 'number' ? mix.renewShare : null; const energyStress = getEnergyPriceStress(energyPricesRaw); return weightedBlend([ - { score: dependency == null ? null : normalizeLowerBetter(dependency, 0, 100), weight: 0.7 }, - { score: energyStress == null ? null : normalizeLowerBetter(energyStress, 0, 25), weight: 0.3 }, + { score: dependency == null ? null : normalizeLowerBetter(dependency, 0, 100), weight: 0.35 }, + { score: gasShare == null ? null : normalizeLowerBetter(gasShare, 0, 100), weight: 0.20 }, + { score: coalShare == null ? null : normalizeLowerBetter(coalShare, 0, 100), weight: 0.15 }, + { score: renewShare == null ? null : normalizeHigherBetter(renewShare, 0, 100), weight: 0.20 }, + { score: energyStress == null ? null : normalizeLowerBetter(energyStress, 0, 25), weight: 0.10 }, ]); } diff --git a/tests/chat-analyst.test.mts b/tests/chat-analyst.test.mts index 2b07184f6..7e3cb7754 100644 --- a/tests/chat-analyst.test.mts +++ b/tests/chat-analyst.test.mts @@ -24,6 +24,7 @@ function emptyCtx(): AnalystContext { countryBrief: '', liveHeadlines: '', relevantArticles: '', + energyExposure: '', activeSources: [], degraded: false, }; @@ -42,7 +43,8 @@ function fullCtx(): AnalystContext { countryBrief: 'Country Focus — UA:\nAnalysis of Ukraine situation.', liveHeadlines: 'Latest Headlines:\n- Missile strikes reported', relevantArticles: '', - activeSources: ['Brief', 'Risk', 'Signals', 'Forecasts', 'Markets', 'Macro', 'Prediction', 'Country', 'Live'], + energyExposure: 'Energy Generation Mix — 2023 data:\nGas-dependent (% electricity from gas): Italy 46%, Netherlands 39%\nCoal-dependent: South Africa 88%, Poland 65%\n(Gas figures are total gas mix; LNG vs. pipeline split not in this dataset.)', + activeSources: ['Brief', 'Risk', 'Signals', 'Forecasts', 'Markets', 'EnergyMix', 'Macro', 'Prediction', 'Country', 'Live'], degraded: false, }; } @@ -62,45 +64,50 @@ describe('buildAnalystSystemPrompt — domain filtering', () => { assert.ok(prompt.includes('Prediction Markets'), 'should include predictionMarkets'); assert.ok(prompt.includes('Country Focus'), 'should include countryBrief'); assert.ok(prompt.includes('Latest Headlines'), 'should include liveHeadlines'); + assert.ok(prompt.includes('Energy Exposure'), 'should include energyExposure'); }); - it('"market" domain excludes worldBrief but includes marketData and macroSignals', () => { + it('"market" domain excludes worldBrief and energyExposure but includes marketData and macroSignals', () => { const prompt = buildAnalystSystemPrompt(fullCtx(), 'market'); assert.ok(!prompt.includes('Global tensions elevated'), 'should exclude worldBrief'); assert.ok(!prompt.includes('Country Focus'), 'should exclude countryBrief'); + assert.ok(!prompt.includes('Energy Exposure'), 'should exclude energyExposure'); assert.ok(prompt.includes('Market Data'), 'should include marketData'); assert.ok(prompt.includes('Macro Signals'), 'should include macroSignals'); assert.ok(prompt.includes('AI Market Signals'), 'should include marketImplications'); assert.ok(prompt.includes('Latest Headlines'), 'should include liveHeadlines'); }); - it('"geo" domain excludes marketData and macroSignals but includes worldBrief', () => { + it('"geo" domain excludes marketData and macroSignals but includes worldBrief and energyExposure', () => { const prompt = buildAnalystSystemPrompt(fullCtx(), 'geo'); assert.ok(prompt.includes('Global tensions elevated'), 'should include worldBrief'); assert.ok(prompt.includes('Top Risk Countries'), 'should include riskScores'); assert.ok(prompt.includes('Country Focus'), 'should include countryBrief'); + assert.ok(prompt.includes('Energy Exposure'), 'should include energyExposure'); assert.ok(!prompt.includes('Market Data'), 'should exclude marketData'); assert.ok(!prompt.includes('Macro Signals'), 'should exclude macroSignals'); assert.ok(prompt.includes('Latest Headlines'), 'should include liveHeadlines'); }); - it('"military" domain excludes marketData and marketImplications but includes worldBrief', () => { + it('"military" domain excludes marketData, marketImplications, and energyExposure but includes worldBrief', () => { const prompt = buildAnalystSystemPrompt(fullCtx(), 'military'); assert.ok(prompt.includes('Global tensions elevated'), 'should include worldBrief'); assert.ok(prompt.includes('Top Risk Countries'), 'should include riskScores'); assert.ok(!prompt.includes('Market Data'), 'should exclude marketData'); assert.ok(!prompt.includes('AI Market Signals'), 'should exclude marketImplications'); assert.ok(!prompt.includes('Macro Signals'), 'should exclude macroSignals'); + assert.ok(!prompt.includes('Energy Exposure'), 'should exclude energyExposure'); assert.ok(prompt.includes('Latest Headlines'), 'should include liveHeadlines'); }); - it('"economic" domain excludes worldBrief and predictionMarkets but includes marketData', () => { + it('"economic" domain excludes worldBrief and predictionMarkets but includes marketData and energyExposure', () => { const prompt = buildAnalystSystemPrompt(fullCtx(), 'economic'); assert.ok(!prompt.includes('Global tensions elevated'), 'should exclude worldBrief'); assert.ok(!prompt.includes('Prediction Markets'), 'should exclude predictionMarkets'); assert.ok(prompt.includes('Market Data'), 'should include marketData'); assert.ok(prompt.includes('Macro Signals'), 'should include macroSignals'); assert.ok(prompt.includes('Top Risk Countries'), 'should include riskScores'); + assert.ok(prompt.includes('Energy Exposure'), 'should include energyExposure'); assert.ok(prompt.includes('Latest Headlines'), 'should include liveHeadlines'); }); diff --git a/tests/helpers/resilience-fixtures.mts b/tests/helpers/resilience-fixtures.mts index 884a4a8ab..d3663ab1d 100644 --- a/tests/helpers/resilience-fixtures.mts +++ b/tests/helpers/resilience-fixtures.mts @@ -91,6 +91,51 @@ export const RESILIENCE_FIXTURES: FixtureMap = { aquastat: { indicator: 'Water stress', value: 85, year: 2024 }, iea: { energyImportDependency: { value: 95, year: 2024, source: 'IEA' } }, }, + 'energy:mix:v1:NO': { + iso2: 'NO', + country: 'Norway', + year: 2023, + coalShare: 0, + gasShare: 5, + oilShare: 0, + nuclearShare: 0, + renewShare: 97, + windShare: 12, + solarShare: 1, + hydroShare: 84, + importShare: -500, + seededAt: '2026-04-04T00:00:00.000Z', + }, + 'energy:mix:v1:US': { + iso2: 'US', + country: 'United States', + year: 2023, + coalShare: 16, + gasShare: 42, + oilShare: 1, + nuclearShare: 18, + renewShare: 22, + windShare: 11, + solarShare: 5, + hydroShare: 6, + importShare: 5, + seededAt: '2026-04-04T00:00:00.000Z', + }, + 'energy:mix:v1:YE': { + iso2: 'YE', + country: 'Yemen', + year: 2023, + coalShare: 0, + gasShare: 0, + oilShare: 85, + nuclearShare: 0, + renewShare: 2, + windShare: 0, + solarShare: 2, + hydroShare: 0, + importShare: 95, + seededAt: '2026-04-04T00:00:00.000Z', + }, 'economic:national-debt:v1': { entries: [ { iso3: 'NOR', debtToGdp: 40, annualGrowth: 1 }, diff --git a/tests/owid-energy-mix-seed.test.mjs b/tests/owid-energy-mix-seed.test.mjs new file mode 100644 index 000000000..6e39f7252 --- /dev/null +++ b/tests/owid-energy-mix-seed.test.mjs @@ -0,0 +1,137 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + buildExposureIndex, + OWID_ENERGY_MIX_KEY_PREFIX, + OWID_EXPOSURE_INDEX_KEY, + OWID_COUNTRY_LIST_KEY, + OWID_META_KEY, + OWID_TTL_SECONDS, +} from '../scripts/seed-owid-energy-mix.mjs'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeCountries(overrides = []) { + const base = new Map([ + ['DE', { iso2: 'DE', country: 'Germany', year: 2023, coalShare: 26, gasShare: 15, oilShare: 1, renewShare: 56, importShare: 4, nuclearShare: 2, windShare: 34, solarShare: 12, hydroShare: 3, seededAt: '' }], + ['IT', { iso2: 'IT', country: 'Italy', year: 2023, coalShare: 5, gasShare: 47, oilShare: 2, renewShare: 40, importShare: 15, nuclearShare: 0, windShare: 7, solarShare: 10, hydroShare: 15, seededAt: '' }], + ['ZA', { iso2: 'ZA', country: 'South Africa', year: 2023, coalShare: 88, gasShare: 0, oilShare: 1, renewShare: 8, importShare: 2, nuclearShare: 5, windShare: 3, solarShare: 2, hydroShare: 1, seededAt: '' }], + ['SA', { iso2: 'SA', country: 'Saudi Arabia', year: 2023, coalShare: 0, gasShare: 38, oilShare: 62, renewShare: 0, importShare: 0, nuclearShare: 0, windShare: 0, solarShare: 0, hydroShare: 0, seededAt: '' }], + ['MT', { iso2: 'MT', country: 'Malta', year: 2023, coalShare: null, gasShare: null, oilShare: 3, renewShare: 10, importShare: 93, nuclearShare: null, windShare: 5, solarShare: 5, hydroShare: 0, seededAt: '' }], + ['NO', { iso2: 'NO', country: 'Norway', year: 2023, coalShare: 0, gasShare: 2, oilShare: 0, renewShare: 97, importShare: -2, nuclearShare: 0, windShare: 8, solarShare: 0, hydroShare: 89, seededAt: '' }], + ]); + for (const [iso2, patch] of overrides) base.set(iso2, { ...base.get(iso2), ...patch }); + return base; +} + +// --------------------------------------------------------------------------- +// buildExposureIndex — ranking correctness +// --------------------------------------------------------------------------- + +describe('buildExposureIndex', () => { + it('returns updatedAt, year, and all five fuel buckets', () => { + const idx = buildExposureIndex(makeCountries()); + assert.ok(typeof idx.updatedAt === 'string'); + assert.equal(idx.year, 2023); + assert.ok(Array.isArray(idx.gas)); + assert.ok(Array.isArray(idx.coal)); + assert.ok(Array.isArray(idx.oil)); + assert.ok(Array.isArray(idx.imported)); + assert.ok(Array.isArray(idx.renewable)); + }); + + it('each bucket includes only countries with a non-null value for that metric', () => { + const idx = buildExposureIndex(makeCountries()); + // MT has no gasShare/coalShare but has oilShare and importShare + assert.ok(!idx.gas.some((e) => e.iso2 === 'MT'), 'MT has null gasShare — should not appear in gas bucket'); + assert.ok(!idx.coal.some((e) => e.iso2 === 'MT'), 'MT has null coalShare — should not appear in coal bucket'); + assert.ok(idx.oil.some((e) => e.iso2 === 'MT'), 'MT has oilShare=3 — must appear in oil bucket'); + assert.ok(idx.imported.some((e) => e.iso2 === 'MT'), 'MT has importShare=93 — must appear in imported bucket'); + }); + + it('countries with only oil/import/renewables data are not excluded from those buckets', () => { + // SA has no coalShare=0 (not null), but the key case: a country with gasShare=null, coalShare=null + const countries = makeCountries(); + countries.set('XX', { + iso2: 'XX', country: 'TestOilOnly', year: 2023, + coalShare: null, gasShare: null, oilShare: 80, + renewShare: null, importShare: 50, nuclearShare: null, + windShare: null, solarShare: null, hydroShare: null, seededAt: '', + }); + const idx = buildExposureIndex(countries); + assert.ok(idx.oil.some((e) => e.iso2 === 'XX'), 'oil-only country must appear in oil bucket'); + assert.ok(idx.imported.some((e) => e.iso2 === 'XX'), 'oil-only country must appear in imported bucket'); + assert.ok(!idx.gas.some((e) => e.iso2 === 'XX'), 'oil-only country must not appear in gas bucket'); + assert.ok(!idx.coal.some((e) => e.iso2 === 'XX'), 'oil-only country must not appear in coal bucket'); + }); + + it('each bucket is sorted descending by share', () => { + const idx = buildExposureIndex(makeCountries()); + for (const bucket of [idx.gas, idx.coal, idx.oil, idx.imported, idx.renewable]) { + for (let i = 1; i < bucket.length; i++) { + assert.ok(bucket[i - 1].share >= bucket[i].share, + `bucket not sorted descending at index ${i}: ${bucket[i - 1].share} < ${bucket[i].share}`); + } + } + }); + + it('top of each bucket is the expected country', () => { + const idx = buildExposureIndex(makeCountries()); + assert.equal(idx.coal[0].iso2, 'ZA', 'highest coal share should be ZA (88%)'); + assert.equal(idx.gas[0].iso2, 'IT', 'highest gas share should be IT (47%)'); + assert.equal(idx.oil[0].iso2, 'SA', 'highest oil share should be SA (62%)'); + assert.equal(idx.imported[0].iso2, 'MT', 'highest import share should be MT (93%)'); + assert.equal(idx.renewable[0].iso2, 'NO', 'highest renewable share should be NO (97%)'); + }); + + it('caps each bucket at 20 entries', () => { + // Build 25 countries all with gasShare values, using unique 2-char ISO2 codes + const countries = new Map(); + const letters = 'ABCDEFGHIJKLMNOPQRSTUVWXY'; // 25 unique letters → AA..AY + for (let i = 0; i < 25; i++) { + const iso2 = `A${letters[i]}`; + countries.set(iso2, { iso2, country: `Country${i}`, year: 2023, gasShare: 50 - i, coalShare: null, oilShare: null, renewShare: null, importShare: null, nuclearShare: null, windShare: null, solarShare: null, hydroShare: null, seededAt: '' }); + } + const idx = buildExposureIndex(countries); + assert.equal(idx.gas.length, 20); + }); + + it('handles all-null year values without throwing', () => { + const countries = makeCountries([[ + 'DE', { year: null }], ['IT', { year: null }], ['ZA', { year: null }], + ['SA', { year: null }], ['MT', { year: null }], ['NO', { year: null }], + ]); + const idx = buildExposureIndex(countries); + assert.equal(idx.year, null); + }); +}); + +// --------------------------------------------------------------------------- +// Exported constants — key naming contract +// --------------------------------------------------------------------------- + +describe('exported key constants', () => { + it('OWID_ENERGY_MIX_KEY_PREFIX matches expected pattern', () => { + assert.equal(OWID_ENERGY_MIX_KEY_PREFIX, 'energy:mix:v1:'); + }); + + it('OWID_EXPOSURE_INDEX_KEY matches expected pattern', () => { + assert.equal(OWID_EXPOSURE_INDEX_KEY, 'energy:exposure:v1:index'); + }); + + it('OWID_COUNTRY_LIST_KEY matches expected pattern', () => { + assert.equal(OWID_COUNTRY_LIST_KEY, 'energy:mix:v1:_countries'); + }); + + it('OWID_META_KEY matches expected pattern', () => { + assert.equal(OWID_META_KEY, 'seed-meta:economic:owid-energy-mix'); + }); + + it('OWID_TTL_SECONDS covers the monthly cron cadence (35 days)', () => { + assert.ok(OWID_TTL_SECONDS >= 35 * 24 * 3600, + `TTL ${OWID_TTL_SECONDS}s is less than 35 days — meta would expire before next monthly run`); + }); +}); diff --git a/tests/resilience-dimension-scorers.test.mts b/tests/resilience-dimension-scorers.test.mts index 0efea739c..3face66bd 100644 --- a/tests/resilience-dimension-scorers.test.mts +++ b/tests/resilience-dimension-scorers.test.mts @@ -86,6 +86,42 @@ describe('resilience dimension scorers', () => { } }); + it('scoreEnergy with full OWID data uses 5-metric blend and high coverage', async () => { + const no = await scoreEnergy('NO', fixtureReader); + assert.ok(no.coverage > 0.9, `NO coverage should be >0.9 with full OWID data, got ${no.coverage}`); + assert.ok(no.score > 50, `NO score should be >50 (high renewables, low dependency), got ${no.score}`); + }); + + it('scoreEnergy without OWID data degrades gracefully to 2-metric blend', async () => { + const noOwidReader = async (key: string) => { + if (key.startsWith('energy:mix:v1:')) return null; + return RESILIENCE_FIXTURES[key] ?? null; + }; + const no = await scoreEnergy('NO', noOwidReader); + assert.ok(no.coverage > 0, `Coverage should be >0 even without OWID data, got ${no.coverage}`); + assert.ok(no.coverage < 0.6, `Coverage should be <0.6 without OWID data (only 2 of 5 metrics), got ${no.coverage}`); + assert.ok(no.score > 0, `Score should be non-zero with only iea data, got ${no.score}`); + }); + + it('scoreEnergy: high renewShare country scores better than high coalShare at equal dependency', async () => { + const renewableReader = async (key: string) => { + if (key === 'resilience:static:XX') return { iea: { energyImportDependency: { value: 50 } } }; + if (key === 'energy:mix:v1:XX') return { gasShare: 5, coalShare: 0, renewShare: 90 }; + if (key === 'economic:energy:v1:all') return null; + return null; + }; + const fossilReader = async (key: string) => { + if (key === 'resilience:static:XX') return { iea: { energyImportDependency: { value: 50 } } }; + if (key === 'energy:mix:v1:XX') return { gasShare: 5, coalShare: 80, renewShare: 5 }; + if (key === 'economic:energy:v1:all') return null; + return null; + }; + const renewable = await scoreEnergy('XX', renewableReader); + const fossil = await scoreEnergy('XX', fossilReader); + assert.ok(renewable.score > fossil.score, + `Renewable-heavy (${renewable.score}) should score better than coal-heavy (${fossil.score})`); + }); + it('memoizes repeated seed reads inside scoreAllDimensions', async () => { const hits = new Map(); const countingReader = async (key: string) => { diff --git a/tests/resilience-scorers.test.mts b/tests/resilience-scorers.test.mts index 86c54d568..b40239aac 100644 --- a/tests/resilience-scorers.test.mts +++ b/tests/resilience-scorers.test.mts @@ -69,10 +69,10 @@ describe('resilience scorer contracts', () => { assert.deepEqual(domainAverages, { economic: 70.67, infrastructure: 77, - energy: 74, + energy: 62, 'social-governance': 72.75, 'health-food': 59, }); - assert.equal(overallScore, 70.85); + assert.equal(overallScore, 69.05); }); });