diff --git a/.env.example b/.env.example index 102ef33fd..f8fd02ab3 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,10 @@ WINGBITS_API_KEY= # Register at: https://acleddata.com/ ACLED_ACCESS_TOKEN= +# UCDP (Uppsala Conflict Data Program — access token required since 2025) +# Register at: https://ucdp.uu.se/apidocs/ +UCDP_ACCESS_TOKEN= + # ------ Internet Outages (Vercel) ------ @@ -128,7 +132,6 @@ RELAY_METRICS_WINDOW_SECONDS=60 # ------ Public Data Sources (no keys required) ------ -# UCDP (Uppsala Conflict Data Program) — public API, no auth # UNHCR (UN Refugee Agency) — public API, no auth (CC BY 4.0) # Open-Meteo — public API, no auth (processes Copernicus ERA5) # WorldPop — public API, no auth needed diff --git a/.github/workflows/seed-ucdp-events.yml b/.github/workflows/seed-ucdp-events.yml new file mode 100644 index 000000000..219094ebf --- /dev/null +++ b/.github/workflows/seed-ucdp-events.yml @@ -0,0 +1,18 @@ +name: Seed UCDP Events (Manual) + +on: + workflow_dispatch: {} + +jobs: + seed: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + - run: node scripts/seed-ucdp-events.mjs + env: + UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }} + UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }} + UCDP_ACCESS_TOKEN: ${{ secrets.UCDP_ACCESS_TOKEN }} diff --git a/api/bootstrap.js b/api/bootstrap.js index 712222e63..6965b0d18 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -19,6 +19,7 @@ const BOOTSTRAP_CACHE_KEYS = { giving: 'giving:summary:v1', climateAnomalies: 'climate:anomalies:v1', wildfires: 'wildfire:fires:v1', + ucdpEvents: 'conflict:ucdp-events:v1', }; const NEG_SENTINEL = '__WM_NEG__'; diff --git a/api/cache-purge.js b/api/cache-purge.js index 633f670ad..6f6e8dc33 100644 --- a/api/cache-purge.js +++ b/api/cache-purge.js @@ -8,7 +8,7 @@ const MAX_DELETIONS = 200; const MAX_SCAN_ITERATIONS = 5; const BLOCKLIST_PREFIXES = ['rl:', '__']; -const DURABLE_DATA_PREFIXES = ['military:bases:', 'conflict:iran-events:']; +const DURABLE_DATA_PREFIXES = ['military:bases:', 'conflict:iran-events:', 'conflict:ucdp-events:']; function getKeyPrefix() { const env = process.env.VERCEL_ENV; diff --git a/docs/DESKTOP_CONFIGURATION.md b/docs/DESKTOP_CONFIGURATION.md index 29507ba09..1c79b448a 100644 --- a/docs/DESKTOP_CONFIGURATION.md +++ b/docs/DESKTOP_CONFIGURATION.md @@ -4,7 +4,7 @@ World Monitor desktop now uses a runtime configuration schema with per-feature t ## Secret keys -The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 22 keys: +The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 25 keys: - `GROQ_API_KEY` - `OPENROUTER_API_KEY` @@ -28,8 +28,9 @@ The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 2 - `OLLAMA_MODEL` - `WORLDMONITOR_API_KEY` — gates cloud fallback access (min 16 chars) - `WTO_API_KEY` - -Note: `UC_DP_KEY` exists in the TypeScript `RuntimeSecretKey` union but is not in the desktop Rust keychain or sidecar. +- `AVIATIONSTACK_API` +- `ICAO_API_KEY` +- `UCDP_ACCESS_TOKEN` ## Feature schema diff --git a/scripts/ais-relay.cjs b/scripts/ais-relay.cjs index cc087a07e..1e7c88d04 100644 --- a/scripts/ais-relay.cjs +++ b/scripts/ais-relay.cjs @@ -766,6 +766,124 @@ async function startOrefPollLoop() { console.log(`[Relay] OREF poll loop started (interval ${OREF_POLL_INTERVAL_MS}ms)`); } +// ───────────────────────────────────────────────────────────── +// UCDP GED Events — fetch paginated conflict data, write to Redis +// ───────────────────────────────────────────────────────────── +const UCDP_ACCESS_TOKEN = (process.env.UCDP_ACCESS_TOKEN || process.env.UC_DP_KEY || '').trim(); +const UCDP_REDIS_KEY = 'conflict:ucdp-events:v1'; +const UCDP_PAGE_SIZE = 1000; +const UCDP_MAX_PAGES = 6; +const UCDP_MAX_EVENTS = 2000; // TODO: review cap after observing real map density & panel usage +const UCDP_TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000; +const UCDP_POLL_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours +const UCDP_TTL_SECONDS = 86400; // 24h safety net +const UCDP_VIOLENCE_TYPE_MAP = { 1: 'UCDP_VIOLENCE_TYPE_STATE_BASED', 2: 'UCDP_VIOLENCE_TYPE_NON_STATE', 3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED' }; + +function ucdpFetchPage(version, page) { + return new Promise((resolve, reject) => { + const pageUrl = new URL(`https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`); + const headers = { Accept: 'application/json', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' }; + if (UCDP_ACCESS_TOKEN) headers['x-ucdp-access-token'] = UCDP_ACCESS_TOKEN; + const req = https.request(pageUrl, { method: 'GET', headers, timeout: 30000 }, (resp) => { + if (resp.statusCode < 200 || resp.statusCode >= 300) { + resp.resume(); + return reject(new Error(`UCDP ${version} page ${page}: HTTP ${resp.statusCode}`)); + } + let data = ''; + resp.on('data', (chunk) => { data += chunk; }); + resp.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(e); } }); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('UCDP timeout')); }); + req.end(); + }); +} + +async function ucdpDiscoverVersion() { + const year = new Date().getFullYear() - 2000; + const candidates = [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])]; + const results = await Promise.allSettled( + candidates.map(async (v) => { + const p0 = await ucdpFetchPage(v, 0); + if (!Array.isArray(p0?.Result)) throw new Error('No results'); + return { version: v, page0: p0 }; + }), + ); + for (const r of results) { + if (r.status === 'fulfilled') return r.value; + } + throw new Error('No valid UCDP GED version found'); +} + +async function seedUcdpEvents() { + try { + const { version, page0 } = await ucdpDiscoverVersion(); + const totalPages = Math.max(1, Number(page0?.TotalPages) || 1); + const newestPage = totalPages - 1; + console.log(`[UCDP] Version ${version}, ${totalPages} total pages`); + + const FAILED = Symbol('failed'); + const fetches = []; + for (let offset = 0; offset < UCDP_MAX_PAGES && (newestPage - offset) >= 0; offset++) { + const pg = newestPage - offset; + fetches.push(pg === 0 ? Promise.resolve(page0) : ucdpFetchPage(version, pg).catch(() => FAILED)); + } + const pageResults = await Promise.all(fetches); + + const allEvents = []; + let latestMs = NaN; + let failedPages = 0; + for (const raw of pageResults) { + if (raw === FAILED) { failedPages++; continue; } + const events = Array.isArray(raw?.Result) ? raw.Result : []; + allEvents.push(...events); + for (const e of events) { + const ms = e?.date_start ? Date.parse(String(e.date_start)) : NaN; + if (Number.isFinite(ms) && (!Number.isFinite(latestMs) || ms > latestMs)) latestMs = ms; + } + } + + const filtered = allEvents.filter((e) => { + if (!Number.isFinite(latestMs)) return true; + const ms = e?.date_start ? Date.parse(String(e.date_start)) : NaN; + return Number.isFinite(ms) && ms >= (latestMs - UCDP_TRAILING_WINDOW_MS); + }); + + const mapped = filtered.map((e) => ({ + id: String(e.id || ''), + dateStart: Date.parse(e.date_start) || 0, + dateEnd: Date.parse(e.date_end) || 0, + location: { latitude: Number(e.latitude) || 0, longitude: Number(e.longitude) || 0 }, + country: e.country || '', + sideA: (e.side_a || '').substring(0, 200), + sideB: (e.side_b || '').substring(0, 200), + deathsBest: Number(e.best) || 0, + deathsLow: Number(e.low) || 0, + deathsHigh: Number(e.high) || 0, + violenceType: UCDP_VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED', + sourceOriginal: (e.source_original || '').substring(0, 300), + })).sort((a, b) => b.dateStart - a.dateStart).slice(0, UCDP_MAX_EVENTS); + + const payload = { events: mapped, fetchedAt: Date.now(), version, totalRaw: allEvents.length, filteredCount: mapped.length }; + const ok = await upstashSet(UCDP_REDIS_KEY, payload, UCDP_TTL_SECONDS); + console.log(`[UCDP] Seeded ${mapped.length} events (raw: ${allEvents.length}, failed pages: ${failedPages}, redis: ${ok ? 'OK' : 'FAIL'})`); + } catch (e) { + console.warn('[UCDP] Seed error:', e?.message || e); + } +} + +async function startUcdpSeedLoop() { + if (!UPSTASH_ENABLED) { + console.log('[UCDP] Disabled (no Upstash Redis)'); + return; + } + console.log(`[UCDP] Seed loop starting (interval ${UCDP_POLL_INTERVAL_MS / 1000 / 60}min, token: ${UCDP_ACCESS_TOKEN ? 'yes' : 'no'})`); + seedUcdpEvents().catch(e => console.warn('[UCDP] Initial seed error:', e?.message || e)); + setInterval(() => { + seedUcdpEvents().catch(e => console.warn('[UCDP] Seed error:', e?.message || e)); + }, UCDP_POLL_INTERVAL_MS).unref?.(); +} + function gzipSyncBuffer(body) { try { return zlib.gzipSync(typeof body === 'string' ? Buffer.from(body) : body); @@ -3447,6 +3565,7 @@ server.listen(PORT, () => { console.log(`[Relay] WebSocket relay on port ${PORT}`); startTelegramPollLoop(); startOrefPollLoop(); + startUcdpSeedLoop(); }); wss.on('connection', (ws, req) => { diff --git a/scripts/seed-ucdp-events.mjs b/scripts/seed-ucdp-events.mjs new file mode 100644 index 000000000..0f2a8e33a --- /dev/null +++ b/scripts/seed-ucdp-events.mjs @@ -0,0 +1,234 @@ +#!/usr/bin/env node + +import { readFileSync, existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const REDIS_KEY = 'conflict:ucdp-events:v1'; +const UCDP_PAGE_SIZE = 1000; +const MAX_PAGES = 6; +const MAX_EVENTS = 2000; // TODO: review cap after observing real map density & panel usage +const TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000; + +const VIOLENCE_TYPE_MAP = { + 1: 'UCDP_VIOLENCE_TYPE_STATE_BASED', + 2: 'UCDP_VIOLENCE_TYPE_NON_STATE', + 3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED', +}; + +const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; + +function loadEnvFile() { + let envPath = join(__dirname, '..', '.env.local'); + if (!existsSync(envPath)) { + envPath = join('/Users/eliehabib/Documents/GitHub/worldmonitor', '.env.local'); + } + if (!existsSync(envPath)) return; + const lines = readFileSync(envPath, 'utf8').split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const eqIdx = trimmed.indexOf('='); + if (eqIdx === -1) continue; + const key = trimmed.slice(0, eqIdx).trim(); + let val = trimmed.slice(eqIdx + 1).trim(); + if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) { + val = val.slice(1, -1); + } + if (!process.env[key]) { + process.env[key] = val; + } + } +} + +function maskToken(token) { + if (!token || token.length < 8) return '***'; + return token.slice(0, 4) + '***' + token.slice(-4); +} + +function buildVersionCandidates() { + const year = new Date().getFullYear() - 2000; + return [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])]; +} + +async function fetchGedPage(version, page, token) { + const headers = { Accept: 'application/json', 'User-Agent': CHROME_UA }; + if (token) headers['x-ucdp-access-token'] = token; + const resp = await fetch( + `https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`, + { headers, signal: AbortSignal.timeout(30_000) }, + ); + if (!resp.ok) throw new Error(`UCDP GED API error (${version}, page ${page}): ${resp.status}`); + return resp.json(); +} + +async function discoverVersion(token) { + const candidates = buildVersionCandidates(); + console.log(` Probing versions: ${candidates.join(', ')}`); + const results = await Promise.allSettled( + candidates.map(async (version) => { + const page0 = await fetchGedPage(version, 0, token); + if (!Array.isArray(page0?.Result)) throw new Error('No results'); + return { version, page0 }; + }), + ); + for (const result of results) { + if (result.status === 'fulfilled') return result.value; + } + throw new Error('No valid UCDP GED version found'); +} + +function parseDateMs(value) { + if (!value) return NaN; + return Date.parse(String(value)); +} + +function getMaxDateMs(events) { + let maxMs = NaN; + for (const event of events) { + const ms = parseDateMs(event?.date_start); + if (!Number.isFinite(ms)) continue; + if (!Number.isFinite(maxMs) || ms > maxMs) maxMs = ms; + } + return maxMs; +} + +async function main() { + loadEnvFile(); + + const redisUrl = process.env.UPSTASH_REDIS_REST_URL; + const redisToken = process.env.UPSTASH_REDIS_REST_TOKEN; + const ucdpToken = (process.env.UCDP_ACCESS_TOKEN || process.env.UC_DP_KEY || '').trim(); + + if (!redisUrl || !redisToken) { + console.error('Missing UPSTASH_REDIS_REST_URL or UPSTASH_REDIS_REST_TOKEN'); + process.exit(1); + } + + console.log('=== UCDP Events Seed ==='); + console.log(` Redis: ${redisUrl}`); + console.log(` Redis Token: ${maskToken(redisToken)}`); + console.log(` UCDP Token: ${ucdpToken ? maskToken(ucdpToken) : '(none — unauthenticated)'}`); + console.log(); + + const { version, page0 } = await discoverVersion(ucdpToken); + const totalPages = Math.max(1, Number(page0?.TotalPages) || 1); + const newestPage = totalPages - 1; + console.log(` Version: ${version} | Total pages: ${totalPages}`); + + const FAILED = Symbol('failed'); + const pagesToFetch = []; + for (let offset = 0; offset < MAX_PAGES && (newestPage - offset) >= 0; offset++) { + const page = newestPage - offset; + if (page === 0) { + pagesToFetch.push(Promise.resolve(page0)); + } else { + pagesToFetch.push(fetchGedPage(version, page, ucdpToken).catch(() => FAILED)); + } + } + + const pageResults = await Promise.all(pagesToFetch); + + const allEvents = []; + let latestDatasetMs = NaN; + let failedPages = 0; + + for (const rawData of pageResults) { + if (rawData === FAILED) { failedPages++; continue; } + const events = Array.isArray(rawData?.Result) ? rawData.Result : []; + allEvents.push(...events); + const pageMaxMs = getMaxDateMs(events); + if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) { + latestDatasetMs = pageMaxMs; + } + } + + console.log(` Raw events: ${allEvents.length} | Failed pages: ${failedPages}`); + + const filtered = allEvents.filter((event) => { + if (!Number.isFinite(latestDatasetMs)) return true; + const eventMs = parseDateMs(event?.date_start); + if (!Number.isFinite(eventMs)) return false; + return eventMs >= (latestDatasetMs - TRAILING_WINDOW_MS); + }); + + console.log(` After 1-year trailing window: ${filtered.length}`); + + const mapped = filtered.map((e) => ({ + id: String(e.id || ''), + dateStart: Date.parse(e.date_start) || 0, + dateEnd: Date.parse(e.date_end) || 0, + location: { + latitude: Number(e.latitude) || 0, + longitude: Number(e.longitude) || 0, + }, + country: e.country || '', + sideA: (e.side_a || '').substring(0, 200), + sideB: (e.side_b || '').substring(0, 200), + deathsBest: Number(e.best) || 0, + deathsLow: Number(e.low) || 0, + deathsHigh: Number(e.high) || 0, + violenceType: VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED', + sourceOriginal: (e.source_original || '').substring(0, 300), + })); + + mapped.sort((a, b) => b.dateStart - a.dateStart); + const capped = mapped.slice(0, MAX_EVENTS); + if (mapped.length > MAX_EVENTS) console.log(` Capped: ${mapped.length} → ${MAX_EVENTS}`); + + const payload = { + events: capped, + fetchedAt: Date.now(), + version, + totalRaw: allEvents.length, + filteredCount: mapped.length, + }; + + console.log(` Mapped: ${mapped.length} events`); + if (mapped[0]) { + console.log(` Newest: ${new Date(mapped[0].dateStart).toISOString().slice(0, 10)} — ${mapped[0].country}`); + } + console.log(); + + const body = JSON.stringify(['SET', REDIS_KEY, JSON.stringify(payload), 'EX', 86400]); + const resp = await fetch(redisUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${redisToken}`, + 'Content-Type': 'application/json', + }, + body, + signal: AbortSignal.timeout(15_000), + }); + + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + console.error(`Redis SET failed: HTTP ${resp.status} — ${text.slice(0, 200)}`); + process.exit(1); + } + + const result = await resp.json(); + console.log(' Redis SET result:', result); + + const getResp = await fetch(`${redisUrl}/get/${encodeURIComponent(REDIS_KEY)}`, { + headers: { Authorization: `Bearer ${redisToken}` }, + signal: AbortSignal.timeout(5_000), + }); + if (getResp.ok) { + const getData = await getResp.json(); + if (getData.result) { + const parsed = JSON.parse(getData.result); + console.log(`\n Verified: ${parsed.events?.length} events in Redis`); + console.log(` Version: ${parsed.version} | fetchedAt: ${new Date(parsed.fetchedAt).toISOString()}`); + } + } + + console.log('\n=== Done ==='); +} + +main().catch(err => { + console.error('FATAL:', err.message || err); + process.exit(1); +}); diff --git a/server/worldmonitor/conflict/v1/list-ucdp-events.ts b/server/worldmonitor/conflict/v1/list-ucdp-events.ts index 37f905de2..ee020f019 100644 --- a/server/worldmonitor/conflict/v1/list-ucdp-events.ts +++ b/server/worldmonitor/conflict/v1/list-ucdp-events.ts @@ -1,224 +1,35 @@ -/** - * RPC: listUcdpEvents -- Port from api/ucdp-events.js - * - * Queries the UCDP GED API with automatic version discovery and paginated - * backward fetch over a trailing 1-year window. Supports optional country - * filtering. Returns empty array on upstream failure (graceful degradation). - */ - import type { ServerContext, ListUcdpEventsRequest, ListUcdpEventsResponse, UcdpViolenceEvent, - UcdpViolenceType, } from '../../../../src/generated/server/worldmonitor/conflict/v1/service_server'; -import { cachedFetchJson } from '../../../_shared/redis'; -import { CHROME_UA } from '../../../_shared/constants'; +import { getCachedJson } from '../../../_shared/redis'; -const UCDP_PAGE_SIZE = 1000; -const MAX_PAGES = 12; -const TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000; +const CACHE_KEY = 'conflict:ucdp-events:v1'; +const MAX_AGE_MS = 25 * 60 * 60 * 1000; // 25h — reject if cron hasn't refreshed -const CACHE_KEY = 'ucdp:gedevents:sebuf:v1'; -const CACHE_TTL_FULL = 6 * 60 * 60; // 6 hours for complete results -const CACHE_TTL_PARTIAL = 10 * 60; // 10 minutes for partial results (M-16 port) - -// In-memory fallback cache with per-entry TTL -let fallbackCache: { data: UcdpViolenceEvent[] | null; timestamp: number; ttlMs: number } = { - data: null, - timestamp: 0, - ttlMs: CACHE_TTL_FULL * 1000, -}; - -const VIOLENCE_TYPE_MAP: Record = { - 1: 'UCDP_VIOLENCE_TYPE_STATE_BASED', - 2: 'UCDP_VIOLENCE_TYPE_NON_STATE', - 3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED', -}; - -function parseDateMs(value: unknown): number { - if (!value) return NaN; - return Date.parse(String(value)); -} - -function getMaxDateMs(events: any[]): number { - let maxMs = NaN; - for (const event of events) { - const ms = parseDateMs(event?.date_start); - if (!Number.isFinite(ms)) continue; - if (!Number.isFinite(maxMs) || ms > maxMs) { - maxMs = ms; - } - } - return maxMs; -} - -function buildVersionCandidates(): string[] { - const year = new Date().getFullYear() - 2000; - return Array.from(new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])); -} - -// Negative cache: prevent hammering UCDP when it's down -let lastFailureTimestamp = 0; -const NEGATIVE_CACHE_MS = 60 * 1000; // 60 seconds backoff after failure - -// Discovered version cache: avoid re-probing every request -let discoveredVersion: string | null = null; -let discoveredVersionTimestamp = 0; -const VERSION_CACHE_MS = 60 * 60 * 1000; // 1 hour - -async function fetchGedPage(version: string, page: number): Promise { - const response = await fetch( - `https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`, - { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(15000), - }, - ); - if (!response.ok) { - throw new Error(`UCDP GED API error (${version}, page ${page}): ${response.status}`); - } - return response.json(); -} - -async function discoverGedVersion(): Promise<{ version: string; page0: any }> { - // Use cached version if still valid - if (discoveredVersion && (Date.now() - discoveredVersionTimestamp) < VERSION_CACHE_MS) { - const page0 = await fetchGedPage(discoveredVersion, 0); - if (Array.isArray(page0?.Result)) { - return { version: discoveredVersion, page0 }; - } - discoveredVersion = null; // Cached version no longer works - } - - // Probe all candidates in parallel instead of sequentially - const candidates = buildVersionCandidates(); - const results = await Promise.allSettled( - candidates.map(async (version) => { - const page0 = await fetchGedPage(version, 0); - if (!Array.isArray(page0?.Result)) throw new Error('No results'); - return { version, page0 }; - }), - ); - - for (const result of results) { - if (result.status === 'fulfilled') { - discoveredVersion = result.value.version; - discoveredVersionTimestamp = Date.now(); - return result.value; - } - } - - throw new Error('No valid UCDP GED version found'); -} - -async function fetchUcdpGedEvents(): Promise { - // Negative cache: skip fetch if UCDP failed recently - if (lastFailureTimestamp && (Date.now() - lastFailureTimestamp) < NEGATIVE_CACHE_MS) { - return null; - } - - try { - const { version, page0 } = await discoverGedVersion(); - const totalPages = Math.max(1, Number(page0?.TotalPages) || 1); - const newestPage = totalPages - 1; - - const FAILED = Symbol('failed'); - const pagesToFetch: Promise[] = []; - for (let offset = 0; offset < MAX_PAGES && (newestPage - offset) >= 0; offset++) { - const page = newestPage - offset; - if (page === 0) { - pagesToFetch.push(Promise.resolve(page0)); - } else { - pagesToFetch.push(fetchGedPage(version, page).catch(() => FAILED)); - } - } - - const pageResults = await Promise.all(pagesToFetch); - - const allEvents: any[] = []; - let latestDatasetMs = NaN; - let failedPages = 0; - - for (const rawData of pageResults) { - if (rawData === FAILED) { failedPages++; continue; } - const events: any[] = Array.isArray(rawData?.Result) ? rawData.Result : []; - allEvents.push(...events); - - const pageMaxMs = getMaxDateMs(events); - if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) { - latestDatasetMs = pageMaxMs; - } - } - - const isPartial = failedPages > 0; - - const filtered = allEvents.filter((event) => { - if (!Number.isFinite(latestDatasetMs)) return true; - const eventMs = parseDateMs(event?.date_start); - if (!Number.isFinite(eventMs)) return false; - return eventMs >= (latestDatasetMs - TRAILING_WINDOW_MS); - }); - - const mapped = filtered.map((e: any): UcdpViolenceEvent => ({ - id: String(e.id || ''), - dateStart: Date.parse(e.date_start) || 0, - dateEnd: Date.parse(e.date_end) || 0, - location: { - latitude: Number(e.latitude) || 0, - longitude: Number(e.longitude) || 0, - }, - country: e.country || '', - sideA: (e.side_a || '').substring(0, 200), - sideB: (e.side_b || '').substring(0, 200), - deathsBest: Number(e.best) || 0, - deathsLow: Number(e.low) || 0, - deathsHigh: Number(e.high) || 0, - violenceType: VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED', - sourceOriginal: (e.source_original || '').substring(0, 300), - })); - - mapped.sort((a, b) => b.dateStart - a.dateStart); - lastFailureTimestamp = 0; - - if (mapped.length === 0) return null; - - if (isPartial) { - fallbackCache = { data: mapped, timestamp: Date.now(), ttlMs: CACHE_TTL_PARTIAL * 1000 }; - return null; - } - - return mapped; - } catch { - lastFailureTimestamp = Date.now(); - return null; - } -} +let fallback: { events: UcdpViolenceEvent[]; ts: number } | null = null; export async function listUcdpEvents( _ctx: ServerContext, req: ListUcdpEventsRequest, ): Promise { try { - const cached = await cachedFetchJson(CACHE_KEY, CACHE_TTL_FULL, fetchUcdpGedEvents); - - if (cached && Array.isArray(cached) && cached.length > 0) { - fallbackCache = { data: cached, timestamp: Date.now(), ttlMs: CACHE_TTL_FULL * 1000 }; - let events = cached; + const raw = await getCachedJson(CACHE_KEY, true) as { events?: UcdpViolenceEvent[]; fetchedAt?: number } | null; + if (raw?.events?.length && (!raw.fetchedAt || (Date.now() - raw.fetchedAt) < MAX_AGE_MS)) { + fallback = { events: raw.events, ts: Date.now() }; + let events = raw.events; if (req.country) events = events.filter((e) => e.country === req.country); return { events, pagination: undefined }; } - } catch { - // cachedFetchJson rejected — fall through to fallback - } + } catch { /* fall through */ } - if (fallbackCache.data && (Date.now() - fallbackCache.timestamp) < fallbackCache.ttlMs) { - let events = fallbackCache.data; + if (fallback && (Date.now() - fallback.ts) < 12 * 60 * 60 * 1000) { + let events = fallback.events; if (req.country) events = events.filter((e) => e.country === req.country); return { events, pagination: undefined }; } - fallbackCache = { data: null, timestamp: 0, ttlMs: CACHE_TTL_FULL * 1000 }; return { events: [], pagination: undefined }; } diff --git a/src-tauri/sidecar/local-api-server.mjs b/src-tauri/sidecar/local-api-server.mjs index bde39faa0..baa90ff12 100644 --- a/src-tauri/sidecar/local-api-server.mjs +++ b/src-tauri/sidecar/local-api-server.mjs @@ -105,7 +105,7 @@ const ALLOWED_ENV_KEYS = new Set([ 'VITE_OPENSKY_RELAY_URL', 'OPENSKY_CLIENT_ID', 'OPENSKY_CLIENT_SECRET', 'AISSTREAM_API_KEY', 'VITE_WS_RELAY_URL', 'FINNHUB_API_KEY', 'NASA_FIRMS_API_KEY', 'OLLAMA_API_URL', 'OLLAMA_MODEL', 'WORLDMONITOR_API_KEY', 'WTO_API_KEY', - 'AVIATIONSTACK_API', 'ICAO_API_KEY', + 'AVIATIONSTACK_API', 'ICAO_API_KEY', 'UCDP_ACCESS_TOKEN', ]); const CHROME_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; @@ -821,6 +821,26 @@ async function validateSecretAgainstProvider(key, rawValue, context = {}) { return ok('NASA FIRMS key verified'); } + case 'UCDP_ACCESS_TOKEN': { + const year = new Date().getFullYear() - 2000; + const candidates = [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])]; + for (const version of candidates) { + try { + const response = await fetchWithTimeout( + `https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=1`, + { headers: { Accept: 'application/json', 'x-ucdp-access-token': value, 'User-Agent': CHROME_UA } } + ); + if (isAuthFailure(response.status)) return fail('UCDP rejected this token'); + if (!response.ok) continue; + const text = await response.text(); + let payload = null; + try { payload = JSON.parse(text); } catch { /* ignore */ } + if (Array.isArray(payload?.Result)) return ok(`UCDP token verified (GED v${version})`); + } catch { continue; } + } + return fail('Could not verify UCDP token (all GED versions failed)'); + } + case 'OLLAMA_API_URL': { let probeUrl; try { diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 59ac9fc93..b7dff88c5 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -27,7 +27,7 @@ const MENU_HELP_GITHUB_ID: &str = "help.github"; #[cfg(feature = "devtools")] const MENU_HELP_DEVTOOLS_ID: &str = "help.devtools"; const TRUSTED_WINDOWS: [&str; 3] = ["main", "settings", "live-channels"]; -const SUPPORTED_SECRET_KEYS: [&str; 24] = [ +const SUPPORTED_SECRET_KEYS: [&str; 25] = [ "GROQ_API_KEY", "OPENROUTER_API_KEY", "FRED_API_KEY", @@ -46,6 +46,7 @@ const SUPPORTED_SECRET_KEYS: [&str; 24] = [ "VITE_WS_RELAY_URL", "FINNHUB_API_KEY", "NASA_FIRMS_API_KEY", + "UCDP_ACCESS_TOKEN", "OLLAMA_API_URL", "OLLAMA_MODEL", "WORLDMONITOR_API_KEY", diff --git a/src/locales/en.json b/src/locales/en.json index ce4493b17..b672fbc34 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -235,7 +235,7 @@ "etfFlows": "BTC ETF Tracker", "stablecoins": "Stablecoins", "deduction": "Deduct Situation", - "ucdpEvents": "UCDP Conflict Events", + "ucdpEvents": "Armed Conflict Events", "giving": "Global Giving", "displacement": "UNHCR Displacement", "climate": "Climate Anomalies", @@ -919,7 +919,7 @@ "shipTraffic": "Ship Traffic", "flightDelays": "Flight Delays", "protests": "Protests", - "ucdpEvents": "UCDP Events", + "ucdpEvents": "Armed Conflict Events", "displacementFlows": "Displacement Flows", "climateAnomalies": "Climate Anomalies", "weatherAlerts": "Weather Alerts", @@ -1255,7 +1255,7 @@ "deathsCount": "{{count}} deaths", "moreNotShown": "{{count}} more events not shown", "noEvents": "No events in this category", - "infoTooltip": "UCDP Georeferenced Events Event-level conflict data from Uppsala University.
  • State-Based: Government vs rebel group
  • Non-State: Armed group vs armed group
  • One-Sided: Violence against civilians
Deaths shown as best estimate (low-high range). ACLED duplicates are filtered out automatically." + "infoTooltip": "Armed Conflict Events Event-level conflict data from Uppsala University (UCDP).
  • State-Based: Government vs rebel group
  • Non-State: Armed group vs armed group
  • One-Sided: Violence against civilians
Deaths shown as best estimate (low-high range). ACLED duplicates are filtered out automatically." }, "giving": { "activityIndex": "Activity Index", @@ -2159,7 +2159,7 @@ "failedToLoad": "Failed to load data", "noDataShort": "No data", "upstreamUnavailable": "Upstream API unavailable — will retry automatically", - "loadingUcdpEvents": "Loading UCDP events", + "loadingUcdpEvents": "Loading armed conflict events", "loadingStablecoins": "Loading stablecoins...", "scanningThermalData": "Scanning thermal data", "calculatingExposure": "Calculating exposure", diff --git a/src/services/desktop-readiness.ts b/src/services/desktop-readiness.ts index 7a26a11b7..2791437c4 100644 --- a/src/services/desktop-readiness.ts +++ b/src/services/desktop-readiness.ts @@ -26,6 +26,7 @@ const keyBackedFeatures: RuntimeFeatureId[] = [ 'economicFred', 'internetOutages', 'acledConflicts', + 'ucdpConflicts', 'abuseChThreatIntel', 'alienvaultOtxThreatIntel', 'abuseIpdbThreatIntel', diff --git a/src/services/runtime-config.ts b/src/services/runtime-config.ts index 1408da3b9..0741f4e5e 100644 --- a/src/services/runtime-config.ts +++ b/src/services/runtime-config.ts @@ -19,7 +19,7 @@ export type RuntimeSecretKey = | 'AISSTREAM_API_KEY' | 'FINNHUB_API_KEY' | 'NASA_FIRMS_API_KEY' - | 'UC_DP_KEY' + | 'UCDP_ACCESS_TOKEN' | 'OLLAMA_API_URL' | 'OLLAMA_MODEL' | 'WORLDMONITOR_API_KEY' @@ -47,6 +47,7 @@ export type RuntimeFeatureId = | 'supplyChain' | 'newsPerFeedFallback' | 'aviationStack' + | 'ucdpConflicts' | 'icaoNotams'; export interface RuntimeFeatureDefinition { @@ -83,6 +84,7 @@ const defaultToggles: Record = { energyEia: true, internetOutages: true, acledConflicts: true, + ucdpConflicts: true, abuseChThreatIntel: true, alienvaultOtxThreatIntel: true, abuseIpdbThreatIntel: true, @@ -149,6 +151,13 @@ export const RUNTIME_FEATURES: RuntimeFeatureDefinition[] = [ requiredSecrets: ['ACLED_ACCESS_TOKEN'], fallback: 'Conflict/protest overlays are hidden.', }, + { + id: 'ucdpConflicts', + name: 'UCDP conflict events', + description: 'Armed conflict georeferenced event data from Uppsala Conflict Data Program.', + requiredSecrets: ['UCDP_ACCESS_TOKEN'], + fallback: 'UCDP conflict layer is disabled.', + }, { id: 'abuseChThreatIntel', name: 'abuse.ch cyber IOC feeds', diff --git a/src/services/settings-constants.ts b/src/services/settings-constants.ts index ff931409e..d892d2adf 100644 --- a/src/services/settings-constants.ts +++ b/src/services/settings-constants.ts @@ -16,7 +16,7 @@ export const SIGNUP_URLS: Partial> = { OPENSKY_CLIENT_SECRET: 'https://opensky-network.org/login?view=registration', FINNHUB_API_KEY: 'https://finnhub.io/register', NASA_FIRMS_API_KEY: 'https://firms.modaps.eosdis.nasa.gov/api/area/', - UC_DP_KEY: 'https://ucdp.uu.se/downloads/', + UCDP_ACCESS_TOKEN: 'https://ucdp.uu.se/apidocs/', OLLAMA_API_URL: 'https://ollama.com/download', OLLAMA_MODEL: 'https://ollama.com/library', WTO_API_KEY: 'https://apiportal.wto.org/', @@ -51,7 +51,7 @@ export const HUMAN_LABELS: Record = { AISSTREAM_API_KEY: 'AISStream API Key', FINNHUB_API_KEY: 'Finnhub API Key', NASA_FIRMS_API_KEY: 'NASA FIRMS API Key', - UC_DP_KEY: 'UCDP API Key', + UCDP_ACCESS_TOKEN: 'UCDP Access Token', OLLAMA_API_URL: 'Ollama Server URL', OLLAMA_MODEL: 'Ollama Model', WORLDMONITOR_API_KEY: 'World Monitor License Key', @@ -85,7 +85,7 @@ export const SETTINGS_CATEGORIES: SettingsCategory[] = [ { id: 'security', label: 'Security & Threats', - features: ['internetOutages', 'acledConflicts', 'abuseChThreatIntel', 'alienvaultOtxThreatIntel', 'abuseIpdbThreatIntel'], + features: ['internetOutages', 'acledConflicts', 'ucdpConflicts', 'abuseChThreatIntel', 'alienvaultOtxThreatIntel', 'abuseIpdbThreatIntel'], }, { id: 'tracking',