mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(conflict): wire UCDP (#760)
* feat(conflict): wire UCDP API access token across full stack UCDP API now requires an `x-ucdp-access-token` header. Renames the stub `UC_DP_KEY` to `UCDP_ACCESS_TOKEN` (matching ACLED convention) and wires it through Rust keychain, sidecar allowlist + verification, handler fetch headers, feature toggles, and desktop settings UI. - Rename UC_DP_KEY → UCDP_ACCESS_TOKEN in type system and labels - Add ucdpConflicts feature toggle with required secret - Add UCDP_ACCESS_TOKEN to Rust SUPPORTED_SECRET_KEYS (24→25) - Add sidecar ALLOWED_ENV_KEYS entry + validation with dynamic GED version probing - Handler sends x-ucdp-access-token header when token is present - UC_DP_KEY fallback in handler for one-release migration window - Update .env.example, desktop-readiness, and docs * feat(conflict): pre-fetch UCDP events via Railway cron + Redis cache Replace the 228-line edge handler that fetched UCDP GED API on every request with a thin Redis reader. The heavy fetch logic (version discovery, paginated backward fetch, 1-year trailing window filter) now runs as a setInterval loop in the Railway relay (ais-relay.cjs) every 6 hours, writing to Redis key conflict:ucdp-events:v1. Changes: - Add UCDP seed loop to ais-relay.cjs (6h interval, 6 pages, 2K cap) - Rewrite list-ucdp-events.ts as thin Redis reader (35 lines) - Add conflict:ucdp-events:v1 to bootstrap batch keys - Protect key from cache-purge via durable data prefix - Add manual-only seed-ucdp-events workflow + standalone script - Rename panel "UCDP Events" → "Armed Conflict Events" in locale - Add 24h TTL + 25h staleness check as safety nets
This commit is contained in:
@@ -63,6 +63,10 @@ WINGBITS_API_KEY=
|
||||
# Register at: https://acleddata.com/
|
||||
ACLED_ACCESS_TOKEN=
|
||||
|
||||
# UCDP (Uppsala Conflict Data Program — access token required since 2025)
|
||||
# Register at: https://ucdp.uu.se/apidocs/
|
||||
UCDP_ACCESS_TOKEN=
|
||||
|
||||
|
||||
# ------ Internet Outages (Vercel) ------
|
||||
|
||||
@@ -128,7 +132,6 @@ RELAY_METRICS_WINDOW_SECONDS=60
|
||||
|
||||
# ------ Public Data Sources (no keys required) ------
|
||||
|
||||
# UCDP (Uppsala Conflict Data Program) — public API, no auth
|
||||
# UNHCR (UN Refugee Agency) — public API, no auth (CC BY 4.0)
|
||||
# Open-Meteo — public API, no auth (processes Copernicus ERA5)
|
||||
# WorldPop — public API, no auth needed
|
||||
|
||||
18
.github/workflows/seed-ucdp-events.yml
vendored
Normal file
18
.github/workflows/seed-ucdp-events.yml
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
name: Seed UCDP Events (Manual)
|
||||
|
||||
on:
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
seed:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
- run: node scripts/seed-ucdp-events.mjs
|
||||
env:
|
||||
UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
|
||||
UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}
|
||||
UCDP_ACCESS_TOKEN: ${{ secrets.UCDP_ACCESS_TOKEN }}
|
||||
1
api/bootstrap.js
vendored
1
api/bootstrap.js
vendored
@@ -19,6 +19,7 @@ const BOOTSTRAP_CACHE_KEYS = {
|
||||
giving: 'giving:summary:v1',
|
||||
climateAnomalies: 'climate:anomalies:v1',
|
||||
wildfires: 'wildfire:fires:v1',
|
||||
ucdpEvents: 'conflict:ucdp-events:v1',
|
||||
};
|
||||
|
||||
const NEG_SENTINEL = '__WM_NEG__';
|
||||
|
||||
@@ -8,7 +8,7 @@ const MAX_DELETIONS = 200;
|
||||
const MAX_SCAN_ITERATIONS = 5;
|
||||
|
||||
const BLOCKLIST_PREFIXES = ['rl:', '__'];
|
||||
const DURABLE_DATA_PREFIXES = ['military:bases:', 'conflict:iran-events:'];
|
||||
const DURABLE_DATA_PREFIXES = ['military:bases:', 'conflict:iran-events:', 'conflict:ucdp-events:'];
|
||||
|
||||
function getKeyPrefix() {
|
||||
const env = process.env.VERCEL_ENV;
|
||||
|
||||
@@ -4,7 +4,7 @@ World Monitor desktop now uses a runtime configuration schema with per-feature t
|
||||
|
||||
## Secret keys
|
||||
|
||||
The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 22 keys:
|
||||
The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 25 keys:
|
||||
|
||||
- `GROQ_API_KEY`
|
||||
- `OPENROUTER_API_KEY`
|
||||
@@ -28,8 +28,9 @@ The desktop vault schema (Rust `SUPPORTED_SECRET_KEYS`) supports the following 2
|
||||
- `OLLAMA_MODEL`
|
||||
- `WORLDMONITOR_API_KEY` — gates cloud fallback access (min 16 chars)
|
||||
- `WTO_API_KEY`
|
||||
|
||||
Note: `UC_DP_KEY` exists in the TypeScript `RuntimeSecretKey` union but is not in the desktop Rust keychain or sidecar.
|
||||
- `AVIATIONSTACK_API`
|
||||
- `ICAO_API_KEY`
|
||||
- `UCDP_ACCESS_TOKEN`
|
||||
|
||||
## Feature schema
|
||||
|
||||
|
||||
@@ -766,6 +766,124 @@ async function startOrefPollLoop() {
|
||||
console.log(`[Relay] OREF poll loop started (interval ${OREF_POLL_INTERVAL_MS}ms)`);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// UCDP GED Events — fetch paginated conflict data, write to Redis
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
const UCDP_ACCESS_TOKEN = (process.env.UCDP_ACCESS_TOKEN || process.env.UC_DP_KEY || '').trim();
|
||||
const UCDP_REDIS_KEY = 'conflict:ucdp-events:v1';
|
||||
const UCDP_PAGE_SIZE = 1000;
|
||||
const UCDP_MAX_PAGES = 6;
|
||||
const UCDP_MAX_EVENTS = 2000; // TODO: review cap after observing real map density & panel usage
|
||||
const UCDP_TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000;
|
||||
const UCDP_POLL_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
||||
const UCDP_TTL_SECONDS = 86400; // 24h safety net
|
||||
const UCDP_VIOLENCE_TYPE_MAP = { 1: 'UCDP_VIOLENCE_TYPE_STATE_BASED', 2: 'UCDP_VIOLENCE_TYPE_NON_STATE', 3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED' };
|
||||
|
||||
function ucdpFetchPage(version, page) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const pageUrl = new URL(`https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`);
|
||||
const headers = { Accept: 'application/json', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' };
|
||||
if (UCDP_ACCESS_TOKEN) headers['x-ucdp-access-token'] = UCDP_ACCESS_TOKEN;
|
||||
const req = https.request(pageUrl, { method: 'GET', headers, timeout: 30000 }, (resp) => {
|
||||
if (resp.statusCode < 200 || resp.statusCode >= 300) {
|
||||
resp.resume();
|
||||
return reject(new Error(`UCDP ${version} page ${page}: HTTP ${resp.statusCode}`));
|
||||
}
|
||||
let data = '';
|
||||
resp.on('data', (chunk) => { data += chunk; });
|
||||
resp.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(e); } });
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => { req.destroy(); reject(new Error('UCDP timeout')); });
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function ucdpDiscoverVersion() {
|
||||
const year = new Date().getFullYear() - 2000;
|
||||
const candidates = [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])];
|
||||
const results = await Promise.allSettled(
|
||||
candidates.map(async (v) => {
|
||||
const p0 = await ucdpFetchPage(v, 0);
|
||||
if (!Array.isArray(p0?.Result)) throw new Error('No results');
|
||||
return { version: v, page0: p0 };
|
||||
}),
|
||||
);
|
||||
for (const r of results) {
|
||||
if (r.status === 'fulfilled') return r.value;
|
||||
}
|
||||
throw new Error('No valid UCDP GED version found');
|
||||
}
|
||||
|
||||
async function seedUcdpEvents() {
|
||||
try {
|
||||
const { version, page0 } = await ucdpDiscoverVersion();
|
||||
const totalPages = Math.max(1, Number(page0?.TotalPages) || 1);
|
||||
const newestPage = totalPages - 1;
|
||||
console.log(`[UCDP] Version ${version}, ${totalPages} total pages`);
|
||||
|
||||
const FAILED = Symbol('failed');
|
||||
const fetches = [];
|
||||
for (let offset = 0; offset < UCDP_MAX_PAGES && (newestPage - offset) >= 0; offset++) {
|
||||
const pg = newestPage - offset;
|
||||
fetches.push(pg === 0 ? Promise.resolve(page0) : ucdpFetchPage(version, pg).catch(() => FAILED));
|
||||
}
|
||||
const pageResults = await Promise.all(fetches);
|
||||
|
||||
const allEvents = [];
|
||||
let latestMs = NaN;
|
||||
let failedPages = 0;
|
||||
for (const raw of pageResults) {
|
||||
if (raw === FAILED) { failedPages++; continue; }
|
||||
const events = Array.isArray(raw?.Result) ? raw.Result : [];
|
||||
allEvents.push(...events);
|
||||
for (const e of events) {
|
||||
const ms = e?.date_start ? Date.parse(String(e.date_start)) : NaN;
|
||||
if (Number.isFinite(ms) && (!Number.isFinite(latestMs) || ms > latestMs)) latestMs = ms;
|
||||
}
|
||||
}
|
||||
|
||||
const filtered = allEvents.filter((e) => {
|
||||
if (!Number.isFinite(latestMs)) return true;
|
||||
const ms = e?.date_start ? Date.parse(String(e.date_start)) : NaN;
|
||||
return Number.isFinite(ms) && ms >= (latestMs - UCDP_TRAILING_WINDOW_MS);
|
||||
});
|
||||
|
||||
const mapped = filtered.map((e) => ({
|
||||
id: String(e.id || ''),
|
||||
dateStart: Date.parse(e.date_start) || 0,
|
||||
dateEnd: Date.parse(e.date_end) || 0,
|
||||
location: { latitude: Number(e.latitude) || 0, longitude: Number(e.longitude) || 0 },
|
||||
country: e.country || '',
|
||||
sideA: (e.side_a || '').substring(0, 200),
|
||||
sideB: (e.side_b || '').substring(0, 200),
|
||||
deathsBest: Number(e.best) || 0,
|
||||
deathsLow: Number(e.low) || 0,
|
||||
deathsHigh: Number(e.high) || 0,
|
||||
violenceType: UCDP_VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED',
|
||||
sourceOriginal: (e.source_original || '').substring(0, 300),
|
||||
})).sort((a, b) => b.dateStart - a.dateStart).slice(0, UCDP_MAX_EVENTS);
|
||||
|
||||
const payload = { events: mapped, fetchedAt: Date.now(), version, totalRaw: allEvents.length, filteredCount: mapped.length };
|
||||
const ok = await upstashSet(UCDP_REDIS_KEY, payload, UCDP_TTL_SECONDS);
|
||||
console.log(`[UCDP] Seeded ${mapped.length} events (raw: ${allEvents.length}, failed pages: ${failedPages}, redis: ${ok ? 'OK' : 'FAIL'})`);
|
||||
} catch (e) {
|
||||
console.warn('[UCDP] Seed error:', e?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
async function startUcdpSeedLoop() {
|
||||
if (!UPSTASH_ENABLED) {
|
||||
console.log('[UCDP] Disabled (no Upstash Redis)');
|
||||
return;
|
||||
}
|
||||
console.log(`[UCDP] Seed loop starting (interval ${UCDP_POLL_INTERVAL_MS / 1000 / 60}min, token: ${UCDP_ACCESS_TOKEN ? 'yes' : 'no'})`);
|
||||
seedUcdpEvents().catch(e => console.warn('[UCDP] Initial seed error:', e?.message || e));
|
||||
setInterval(() => {
|
||||
seedUcdpEvents().catch(e => console.warn('[UCDP] Seed error:', e?.message || e));
|
||||
}, UCDP_POLL_INTERVAL_MS).unref?.();
|
||||
}
|
||||
|
||||
function gzipSyncBuffer(body) {
|
||||
try {
|
||||
return zlib.gzipSync(typeof body === 'string' ? Buffer.from(body) : body);
|
||||
@@ -3447,6 +3565,7 @@ server.listen(PORT, () => {
|
||||
console.log(`[Relay] WebSocket relay on port ${PORT}`);
|
||||
startTelegramPollLoop();
|
||||
startOrefPollLoop();
|
||||
startUcdpSeedLoop();
|
||||
});
|
||||
|
||||
wss.on('connection', (ws, req) => {
|
||||
|
||||
234
scripts/seed-ucdp-events.mjs
Normal file
234
scripts/seed-ucdp-events.mjs
Normal file
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const REDIS_KEY = 'conflict:ucdp-events:v1';
|
||||
const UCDP_PAGE_SIZE = 1000;
|
||||
const MAX_PAGES = 6;
|
||||
const MAX_EVENTS = 2000; // TODO: review cap after observing real map density & panel usage
|
||||
const TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const VIOLENCE_TYPE_MAP = {
|
||||
1: 'UCDP_VIOLENCE_TYPE_STATE_BASED',
|
||||
2: 'UCDP_VIOLENCE_TYPE_NON_STATE',
|
||||
3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED',
|
||||
};
|
||||
|
||||
const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
function loadEnvFile() {
|
||||
let envPath = join(__dirname, '..', '.env.local');
|
||||
if (!existsSync(envPath)) {
|
||||
envPath = join('/Users/eliehabib/Documents/GitHub/worldmonitor', '.env.local');
|
||||
}
|
||||
if (!existsSync(envPath)) return;
|
||||
const lines = readFileSync(envPath, 'utf8').split('\n');
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eqIdx = trimmed.indexOf('=');
|
||||
if (eqIdx === -1) continue;
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
let val = trimmed.slice(eqIdx + 1).trim();
|
||||
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
|
||||
val = val.slice(1, -1);
|
||||
}
|
||||
if (!process.env[key]) {
|
||||
process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function maskToken(token) {
|
||||
if (!token || token.length < 8) return '***';
|
||||
return token.slice(0, 4) + '***' + token.slice(-4);
|
||||
}
|
||||
|
||||
function buildVersionCandidates() {
|
||||
const year = new Date().getFullYear() - 2000;
|
||||
return [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])];
|
||||
}
|
||||
|
||||
async function fetchGedPage(version, page, token) {
|
||||
const headers = { Accept: 'application/json', 'User-Agent': CHROME_UA };
|
||||
if (token) headers['x-ucdp-access-token'] = token;
|
||||
const resp = await fetch(
|
||||
`https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`,
|
||||
{ headers, signal: AbortSignal.timeout(30_000) },
|
||||
);
|
||||
if (!resp.ok) throw new Error(`UCDP GED API error (${version}, page ${page}): ${resp.status}`);
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
async function discoverVersion(token) {
|
||||
const candidates = buildVersionCandidates();
|
||||
console.log(` Probing versions: ${candidates.join(', ')}`);
|
||||
const results = await Promise.allSettled(
|
||||
candidates.map(async (version) => {
|
||||
const page0 = await fetchGedPage(version, 0, token);
|
||||
if (!Array.isArray(page0?.Result)) throw new Error('No results');
|
||||
return { version, page0 };
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') return result.value;
|
||||
}
|
||||
throw new Error('No valid UCDP GED version found');
|
||||
}
|
||||
|
||||
function parseDateMs(value) {
|
||||
if (!value) return NaN;
|
||||
return Date.parse(String(value));
|
||||
}
|
||||
|
||||
function getMaxDateMs(events) {
|
||||
let maxMs = NaN;
|
||||
for (const event of events) {
|
||||
const ms = parseDateMs(event?.date_start);
|
||||
if (!Number.isFinite(ms)) continue;
|
||||
if (!Number.isFinite(maxMs) || ms > maxMs) maxMs = ms;
|
||||
}
|
||||
return maxMs;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
loadEnvFile();
|
||||
|
||||
const redisUrl = process.env.UPSTASH_REDIS_REST_URL;
|
||||
const redisToken = process.env.UPSTASH_REDIS_REST_TOKEN;
|
||||
const ucdpToken = (process.env.UCDP_ACCESS_TOKEN || process.env.UC_DP_KEY || '').trim();
|
||||
|
||||
if (!redisUrl || !redisToken) {
|
||||
console.error('Missing UPSTASH_REDIS_REST_URL or UPSTASH_REDIS_REST_TOKEN');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('=== UCDP Events Seed ===');
|
||||
console.log(` Redis: ${redisUrl}`);
|
||||
console.log(` Redis Token: ${maskToken(redisToken)}`);
|
||||
console.log(` UCDP Token: ${ucdpToken ? maskToken(ucdpToken) : '(none — unauthenticated)'}`);
|
||||
console.log();
|
||||
|
||||
const { version, page0 } = await discoverVersion(ucdpToken);
|
||||
const totalPages = Math.max(1, Number(page0?.TotalPages) || 1);
|
||||
const newestPage = totalPages - 1;
|
||||
console.log(` Version: ${version} | Total pages: ${totalPages}`);
|
||||
|
||||
const FAILED = Symbol('failed');
|
||||
const pagesToFetch = [];
|
||||
for (let offset = 0; offset < MAX_PAGES && (newestPage - offset) >= 0; offset++) {
|
||||
const page = newestPage - offset;
|
||||
if (page === 0) {
|
||||
pagesToFetch.push(Promise.resolve(page0));
|
||||
} else {
|
||||
pagesToFetch.push(fetchGedPage(version, page, ucdpToken).catch(() => FAILED));
|
||||
}
|
||||
}
|
||||
|
||||
const pageResults = await Promise.all(pagesToFetch);
|
||||
|
||||
const allEvents = [];
|
||||
let latestDatasetMs = NaN;
|
||||
let failedPages = 0;
|
||||
|
||||
for (const rawData of pageResults) {
|
||||
if (rawData === FAILED) { failedPages++; continue; }
|
||||
const events = Array.isArray(rawData?.Result) ? rawData.Result : [];
|
||||
allEvents.push(...events);
|
||||
const pageMaxMs = getMaxDateMs(events);
|
||||
if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) {
|
||||
latestDatasetMs = pageMaxMs;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Raw events: ${allEvents.length} | Failed pages: ${failedPages}`);
|
||||
|
||||
const filtered = allEvents.filter((event) => {
|
||||
if (!Number.isFinite(latestDatasetMs)) return true;
|
||||
const eventMs = parseDateMs(event?.date_start);
|
||||
if (!Number.isFinite(eventMs)) return false;
|
||||
return eventMs >= (latestDatasetMs - TRAILING_WINDOW_MS);
|
||||
});
|
||||
|
||||
console.log(` After 1-year trailing window: ${filtered.length}`);
|
||||
|
||||
const mapped = filtered.map((e) => ({
|
||||
id: String(e.id || ''),
|
||||
dateStart: Date.parse(e.date_start) || 0,
|
||||
dateEnd: Date.parse(e.date_end) || 0,
|
||||
location: {
|
||||
latitude: Number(e.latitude) || 0,
|
||||
longitude: Number(e.longitude) || 0,
|
||||
},
|
||||
country: e.country || '',
|
||||
sideA: (e.side_a || '').substring(0, 200),
|
||||
sideB: (e.side_b || '').substring(0, 200),
|
||||
deathsBest: Number(e.best) || 0,
|
||||
deathsLow: Number(e.low) || 0,
|
||||
deathsHigh: Number(e.high) || 0,
|
||||
violenceType: VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED',
|
||||
sourceOriginal: (e.source_original || '').substring(0, 300),
|
||||
}));
|
||||
|
||||
mapped.sort((a, b) => b.dateStart - a.dateStart);
|
||||
const capped = mapped.slice(0, MAX_EVENTS);
|
||||
if (mapped.length > MAX_EVENTS) console.log(` Capped: ${mapped.length} → ${MAX_EVENTS}`);
|
||||
|
||||
const payload = {
|
||||
events: capped,
|
||||
fetchedAt: Date.now(),
|
||||
version,
|
||||
totalRaw: allEvents.length,
|
||||
filteredCount: mapped.length,
|
||||
};
|
||||
|
||||
console.log(` Mapped: ${mapped.length} events`);
|
||||
if (mapped[0]) {
|
||||
console.log(` Newest: ${new Date(mapped[0].dateStart).toISOString().slice(0, 10)} — ${mapped[0].country}`);
|
||||
}
|
||||
console.log();
|
||||
|
||||
const body = JSON.stringify(['SET', REDIS_KEY, JSON.stringify(payload), 'EX', 86400]);
|
||||
const resp = await fetch(redisUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${redisToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const text = await resp.text().catch(() => '');
|
||||
console.error(`Redis SET failed: HTTP ${resp.status} — ${text.slice(0, 200)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await resp.json();
|
||||
console.log(' Redis SET result:', result);
|
||||
|
||||
const getResp = await fetch(`${redisUrl}/get/${encodeURIComponent(REDIS_KEY)}`, {
|
||||
headers: { Authorization: `Bearer ${redisToken}` },
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (getResp.ok) {
|
||||
const getData = await getResp.json();
|
||||
if (getData.result) {
|
||||
const parsed = JSON.parse(getData.result);
|
||||
console.log(`\n Verified: ${parsed.events?.length} events in Redis`);
|
||||
console.log(` Version: ${parsed.version} | fetchedAt: ${new Date(parsed.fetchedAt).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n=== Done ===');
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('FATAL:', err.message || err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,224 +1,35 @@
|
||||
/**
|
||||
* RPC: listUcdpEvents -- Port from api/ucdp-events.js
|
||||
*
|
||||
* Queries the UCDP GED API with automatic version discovery and paginated
|
||||
* backward fetch over a trailing 1-year window. Supports optional country
|
||||
* filtering. Returns empty array on upstream failure (graceful degradation).
|
||||
*/
|
||||
|
||||
import type {
|
||||
ServerContext,
|
||||
ListUcdpEventsRequest,
|
||||
ListUcdpEventsResponse,
|
||||
UcdpViolenceEvent,
|
||||
UcdpViolenceType,
|
||||
} from '../../../../src/generated/server/worldmonitor/conflict/v1/service_server';
|
||||
import { cachedFetchJson } from '../../../_shared/redis';
|
||||
import { CHROME_UA } from '../../../_shared/constants';
|
||||
import { getCachedJson } from '../../../_shared/redis';
|
||||
|
||||
const UCDP_PAGE_SIZE = 1000;
|
||||
const MAX_PAGES = 12;
|
||||
const TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000;
|
||||
const CACHE_KEY = 'conflict:ucdp-events:v1';
|
||||
const MAX_AGE_MS = 25 * 60 * 60 * 1000; // 25h — reject if cron hasn't refreshed
|
||||
|
||||
const CACHE_KEY = 'ucdp:gedevents:sebuf:v1';
|
||||
const CACHE_TTL_FULL = 6 * 60 * 60; // 6 hours for complete results
|
||||
const CACHE_TTL_PARTIAL = 10 * 60; // 10 minutes for partial results (M-16 port)
|
||||
|
||||
// In-memory fallback cache with per-entry TTL
|
||||
let fallbackCache: { data: UcdpViolenceEvent[] | null; timestamp: number; ttlMs: number } = {
|
||||
data: null,
|
||||
timestamp: 0,
|
||||
ttlMs: CACHE_TTL_FULL * 1000,
|
||||
};
|
||||
|
||||
const VIOLENCE_TYPE_MAP: Record<number, UcdpViolenceType> = {
|
||||
1: 'UCDP_VIOLENCE_TYPE_STATE_BASED',
|
||||
2: 'UCDP_VIOLENCE_TYPE_NON_STATE',
|
||||
3: 'UCDP_VIOLENCE_TYPE_ONE_SIDED',
|
||||
};
|
||||
|
||||
function parseDateMs(value: unknown): number {
|
||||
if (!value) return NaN;
|
||||
return Date.parse(String(value));
|
||||
}
|
||||
|
||||
function getMaxDateMs(events: any[]): number {
|
||||
let maxMs = NaN;
|
||||
for (const event of events) {
|
||||
const ms = parseDateMs(event?.date_start);
|
||||
if (!Number.isFinite(ms)) continue;
|
||||
if (!Number.isFinite(maxMs) || ms > maxMs) {
|
||||
maxMs = ms;
|
||||
}
|
||||
}
|
||||
return maxMs;
|
||||
}
|
||||
|
||||
function buildVersionCandidates(): string[] {
|
||||
const year = new Date().getFullYear() - 2000;
|
||||
return Array.from(new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1']));
|
||||
}
|
||||
|
||||
// Negative cache: prevent hammering UCDP when it's down
|
||||
let lastFailureTimestamp = 0;
|
||||
const NEGATIVE_CACHE_MS = 60 * 1000; // 60 seconds backoff after failure
|
||||
|
||||
// Discovered version cache: avoid re-probing every request
|
||||
let discoveredVersion: string | null = null;
|
||||
let discoveredVersionTimestamp = 0;
|
||||
const VERSION_CACHE_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
async function fetchGedPage(version: string, page: number): Promise<any> {
|
||||
const response = await fetch(
|
||||
`https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`,
|
||||
{
|
||||
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
|
||||
signal: AbortSignal.timeout(15000),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`UCDP GED API error (${version}, page ${page}): ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function discoverGedVersion(): Promise<{ version: string; page0: any }> {
|
||||
// Use cached version if still valid
|
||||
if (discoveredVersion && (Date.now() - discoveredVersionTimestamp) < VERSION_CACHE_MS) {
|
||||
const page0 = await fetchGedPage(discoveredVersion, 0);
|
||||
if (Array.isArray(page0?.Result)) {
|
||||
return { version: discoveredVersion, page0 };
|
||||
}
|
||||
discoveredVersion = null; // Cached version no longer works
|
||||
}
|
||||
|
||||
// Probe all candidates in parallel instead of sequentially
|
||||
const candidates = buildVersionCandidates();
|
||||
const results = await Promise.allSettled(
|
||||
candidates.map(async (version) => {
|
||||
const page0 = await fetchGedPage(version, 0);
|
||||
if (!Array.isArray(page0?.Result)) throw new Error('No results');
|
||||
return { version, page0 };
|
||||
}),
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') {
|
||||
discoveredVersion = result.value.version;
|
||||
discoveredVersionTimestamp = Date.now();
|
||||
return result.value;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('No valid UCDP GED version found');
|
||||
}
|
||||
|
||||
async function fetchUcdpGedEvents(): Promise<UcdpViolenceEvent[] | null> {
|
||||
// Negative cache: skip fetch if UCDP failed recently
|
||||
if (lastFailureTimestamp && (Date.now() - lastFailureTimestamp) < NEGATIVE_CACHE_MS) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const { version, page0 } = await discoverGedVersion();
|
||||
const totalPages = Math.max(1, Number(page0?.TotalPages) || 1);
|
||||
const newestPage = totalPages - 1;
|
||||
|
||||
const FAILED = Symbol('failed');
|
||||
const pagesToFetch: Promise<any>[] = [];
|
||||
for (let offset = 0; offset < MAX_PAGES && (newestPage - offset) >= 0; offset++) {
|
||||
const page = newestPage - offset;
|
||||
if (page === 0) {
|
||||
pagesToFetch.push(Promise.resolve(page0));
|
||||
} else {
|
||||
pagesToFetch.push(fetchGedPage(version, page).catch(() => FAILED));
|
||||
}
|
||||
}
|
||||
|
||||
const pageResults = await Promise.all(pagesToFetch);
|
||||
|
||||
const allEvents: any[] = [];
|
||||
let latestDatasetMs = NaN;
|
||||
let failedPages = 0;
|
||||
|
||||
for (const rawData of pageResults) {
|
||||
if (rawData === FAILED) { failedPages++; continue; }
|
||||
const events: any[] = Array.isArray(rawData?.Result) ? rawData.Result : [];
|
||||
allEvents.push(...events);
|
||||
|
||||
const pageMaxMs = getMaxDateMs(events);
|
||||
if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) {
|
||||
latestDatasetMs = pageMaxMs;
|
||||
}
|
||||
}
|
||||
|
||||
const isPartial = failedPages > 0;
|
||||
|
||||
const filtered = allEvents.filter((event) => {
|
||||
if (!Number.isFinite(latestDatasetMs)) return true;
|
||||
const eventMs = parseDateMs(event?.date_start);
|
||||
if (!Number.isFinite(eventMs)) return false;
|
||||
return eventMs >= (latestDatasetMs - TRAILING_WINDOW_MS);
|
||||
});
|
||||
|
||||
const mapped = filtered.map((e: any): UcdpViolenceEvent => ({
|
||||
id: String(e.id || ''),
|
||||
dateStart: Date.parse(e.date_start) || 0,
|
||||
dateEnd: Date.parse(e.date_end) || 0,
|
||||
location: {
|
||||
latitude: Number(e.latitude) || 0,
|
||||
longitude: Number(e.longitude) || 0,
|
||||
},
|
||||
country: e.country || '',
|
||||
sideA: (e.side_a || '').substring(0, 200),
|
||||
sideB: (e.side_b || '').substring(0, 200),
|
||||
deathsBest: Number(e.best) || 0,
|
||||
deathsLow: Number(e.low) || 0,
|
||||
deathsHigh: Number(e.high) || 0,
|
||||
violenceType: VIOLENCE_TYPE_MAP[e.type_of_violence] || 'UCDP_VIOLENCE_TYPE_UNSPECIFIED',
|
||||
sourceOriginal: (e.source_original || '').substring(0, 300),
|
||||
}));
|
||||
|
||||
mapped.sort((a, b) => b.dateStart - a.dateStart);
|
||||
lastFailureTimestamp = 0;
|
||||
|
||||
if (mapped.length === 0) return null;
|
||||
|
||||
if (isPartial) {
|
||||
fallbackCache = { data: mapped, timestamp: Date.now(), ttlMs: CACHE_TTL_PARTIAL * 1000 };
|
||||
return null;
|
||||
}
|
||||
|
||||
return mapped;
|
||||
} catch {
|
||||
lastFailureTimestamp = Date.now();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
let fallback: { events: UcdpViolenceEvent[]; ts: number } | null = null;
|
||||
|
||||
export async function listUcdpEvents(
|
||||
_ctx: ServerContext,
|
||||
req: ListUcdpEventsRequest,
|
||||
): Promise<ListUcdpEventsResponse> {
|
||||
try {
|
||||
const cached = await cachedFetchJson<UcdpViolenceEvent[]>(CACHE_KEY, CACHE_TTL_FULL, fetchUcdpGedEvents);
|
||||
|
||||
if (cached && Array.isArray(cached) && cached.length > 0) {
|
||||
fallbackCache = { data: cached, timestamp: Date.now(), ttlMs: CACHE_TTL_FULL * 1000 };
|
||||
let events = cached;
|
||||
const raw = await getCachedJson(CACHE_KEY, true) as { events?: UcdpViolenceEvent[]; fetchedAt?: number } | null;
|
||||
if (raw?.events?.length && (!raw.fetchedAt || (Date.now() - raw.fetchedAt) < MAX_AGE_MS)) {
|
||||
fallback = { events: raw.events, ts: Date.now() };
|
||||
let events = raw.events;
|
||||
if (req.country) events = events.filter((e) => e.country === req.country);
|
||||
return { events, pagination: undefined };
|
||||
}
|
||||
} catch {
|
||||
// cachedFetchJson rejected — fall through to fallback
|
||||
}
|
||||
} catch { /* fall through */ }
|
||||
|
||||
if (fallbackCache.data && (Date.now() - fallbackCache.timestamp) < fallbackCache.ttlMs) {
|
||||
let events = fallbackCache.data;
|
||||
if (fallback && (Date.now() - fallback.ts) < 12 * 60 * 60 * 1000) {
|
||||
let events = fallback.events;
|
||||
if (req.country) events = events.filter((e) => e.country === req.country);
|
||||
return { events, pagination: undefined };
|
||||
}
|
||||
fallbackCache = { data: null, timestamp: 0, ttlMs: CACHE_TTL_FULL * 1000 };
|
||||
|
||||
return { events: [], pagination: undefined };
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ const ALLOWED_ENV_KEYS = new Set([
|
||||
'VITE_OPENSKY_RELAY_URL', 'OPENSKY_CLIENT_ID', 'OPENSKY_CLIENT_SECRET',
|
||||
'AISSTREAM_API_KEY', 'VITE_WS_RELAY_URL', 'FINNHUB_API_KEY', 'NASA_FIRMS_API_KEY',
|
||||
'OLLAMA_API_URL', 'OLLAMA_MODEL', 'WORLDMONITOR_API_KEY', 'WTO_API_KEY',
|
||||
'AVIATIONSTACK_API', 'ICAO_API_KEY',
|
||||
'AVIATIONSTACK_API', 'ICAO_API_KEY', 'UCDP_ACCESS_TOKEN',
|
||||
]);
|
||||
|
||||
const CHROME_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||
@@ -821,6 +821,26 @@ async function validateSecretAgainstProvider(key, rawValue, context = {}) {
|
||||
return ok('NASA FIRMS key verified');
|
||||
}
|
||||
|
||||
case 'UCDP_ACCESS_TOKEN': {
|
||||
const year = new Date().getFullYear() - 2000;
|
||||
const candidates = [...new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])];
|
||||
for (const version of candidates) {
|
||||
try {
|
||||
const response = await fetchWithTimeout(
|
||||
`https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=1`,
|
||||
{ headers: { Accept: 'application/json', 'x-ucdp-access-token': value, 'User-Agent': CHROME_UA } }
|
||||
);
|
||||
if (isAuthFailure(response.status)) return fail('UCDP rejected this token');
|
||||
if (!response.ok) continue;
|
||||
const text = await response.text();
|
||||
let payload = null;
|
||||
try { payload = JSON.parse(text); } catch { /* ignore */ }
|
||||
if (Array.isArray(payload?.Result)) return ok(`UCDP token verified (GED v${version})`);
|
||||
} catch { continue; }
|
||||
}
|
||||
return fail('Could not verify UCDP token (all GED versions failed)');
|
||||
}
|
||||
|
||||
case 'OLLAMA_API_URL': {
|
||||
let probeUrl;
|
||||
try {
|
||||
|
||||
@@ -27,7 +27,7 @@ const MENU_HELP_GITHUB_ID: &str = "help.github";
|
||||
#[cfg(feature = "devtools")]
|
||||
const MENU_HELP_DEVTOOLS_ID: &str = "help.devtools";
|
||||
const TRUSTED_WINDOWS: [&str; 3] = ["main", "settings", "live-channels"];
|
||||
const SUPPORTED_SECRET_KEYS: [&str; 24] = [
|
||||
const SUPPORTED_SECRET_KEYS: [&str; 25] = [
|
||||
"GROQ_API_KEY",
|
||||
"OPENROUTER_API_KEY",
|
||||
"FRED_API_KEY",
|
||||
@@ -46,6 +46,7 @@ const SUPPORTED_SECRET_KEYS: [&str; 24] = [
|
||||
"VITE_WS_RELAY_URL",
|
||||
"FINNHUB_API_KEY",
|
||||
"NASA_FIRMS_API_KEY",
|
||||
"UCDP_ACCESS_TOKEN",
|
||||
"OLLAMA_API_URL",
|
||||
"OLLAMA_MODEL",
|
||||
"WORLDMONITOR_API_KEY",
|
||||
|
||||
@@ -235,7 +235,7 @@
|
||||
"etfFlows": "BTC ETF Tracker",
|
||||
"stablecoins": "Stablecoins",
|
||||
"deduction": "Deduct Situation",
|
||||
"ucdpEvents": "UCDP Conflict Events",
|
||||
"ucdpEvents": "Armed Conflict Events",
|
||||
"giving": "Global Giving",
|
||||
"displacement": "UNHCR Displacement",
|
||||
"climate": "Climate Anomalies",
|
||||
@@ -919,7 +919,7 @@
|
||||
"shipTraffic": "Ship Traffic",
|
||||
"flightDelays": "Flight Delays",
|
||||
"protests": "Protests",
|
||||
"ucdpEvents": "UCDP Events",
|
||||
"ucdpEvents": "Armed Conflict Events",
|
||||
"displacementFlows": "Displacement Flows",
|
||||
"climateAnomalies": "Climate Anomalies",
|
||||
"weatherAlerts": "Weather Alerts",
|
||||
@@ -1255,7 +1255,7 @@
|
||||
"deathsCount": "{{count}} deaths",
|
||||
"moreNotShown": "{{count}} more events not shown",
|
||||
"noEvents": "No events in this category",
|
||||
"infoTooltip": "<strong>UCDP Georeferenced Events</strong> Event-level conflict data from Uppsala University.<ul><li><strong>State-Based</strong>: Government vs rebel group</li><li><strong>Non-State</strong>: Armed group vs armed group</li><li><strong>One-Sided</strong>: Violence against civilians</li></ul>Deaths shown as best estimate (low-high range). ACLED duplicates are filtered out automatically."
|
||||
"infoTooltip": "<strong>Armed Conflict Events</strong> Event-level conflict data from Uppsala University (UCDP).<ul><li><strong>State-Based</strong>: Government vs rebel group</li><li><strong>Non-State</strong>: Armed group vs armed group</li><li><strong>One-Sided</strong>: Violence against civilians</li></ul>Deaths shown as best estimate (low-high range). ACLED duplicates are filtered out automatically."
|
||||
},
|
||||
"giving": {
|
||||
"activityIndex": "Activity Index",
|
||||
@@ -2159,7 +2159,7 @@
|
||||
"failedToLoad": "Failed to load data",
|
||||
"noDataShort": "No data",
|
||||
"upstreamUnavailable": "Upstream API unavailable — will retry automatically",
|
||||
"loadingUcdpEvents": "Loading UCDP events",
|
||||
"loadingUcdpEvents": "Loading armed conflict events",
|
||||
"loadingStablecoins": "Loading stablecoins...",
|
||||
"scanningThermalData": "Scanning thermal data",
|
||||
"calculatingExposure": "Calculating exposure",
|
||||
|
||||
@@ -26,6 +26,7 @@ const keyBackedFeatures: RuntimeFeatureId[] = [
|
||||
'economicFred',
|
||||
'internetOutages',
|
||||
'acledConflicts',
|
||||
'ucdpConflicts',
|
||||
'abuseChThreatIntel',
|
||||
'alienvaultOtxThreatIntel',
|
||||
'abuseIpdbThreatIntel',
|
||||
|
||||
@@ -19,7 +19,7 @@ export type RuntimeSecretKey =
|
||||
| 'AISSTREAM_API_KEY'
|
||||
| 'FINNHUB_API_KEY'
|
||||
| 'NASA_FIRMS_API_KEY'
|
||||
| 'UC_DP_KEY'
|
||||
| 'UCDP_ACCESS_TOKEN'
|
||||
| 'OLLAMA_API_URL'
|
||||
| 'OLLAMA_MODEL'
|
||||
| 'WORLDMONITOR_API_KEY'
|
||||
@@ -47,6 +47,7 @@ export type RuntimeFeatureId =
|
||||
| 'supplyChain'
|
||||
| 'newsPerFeedFallback'
|
||||
| 'aviationStack'
|
||||
| 'ucdpConflicts'
|
||||
| 'icaoNotams';
|
||||
|
||||
export interface RuntimeFeatureDefinition {
|
||||
@@ -83,6 +84,7 @@ const defaultToggles: Record<RuntimeFeatureId, boolean> = {
|
||||
energyEia: true,
|
||||
internetOutages: true,
|
||||
acledConflicts: true,
|
||||
ucdpConflicts: true,
|
||||
abuseChThreatIntel: true,
|
||||
alienvaultOtxThreatIntel: true,
|
||||
abuseIpdbThreatIntel: true,
|
||||
@@ -149,6 +151,13 @@ export const RUNTIME_FEATURES: RuntimeFeatureDefinition[] = [
|
||||
requiredSecrets: ['ACLED_ACCESS_TOKEN'],
|
||||
fallback: 'Conflict/protest overlays are hidden.',
|
||||
},
|
||||
{
|
||||
id: 'ucdpConflicts',
|
||||
name: 'UCDP conflict events',
|
||||
description: 'Armed conflict georeferenced event data from Uppsala Conflict Data Program.',
|
||||
requiredSecrets: ['UCDP_ACCESS_TOKEN'],
|
||||
fallback: 'UCDP conflict layer is disabled.',
|
||||
},
|
||||
{
|
||||
id: 'abuseChThreatIntel',
|
||||
name: 'abuse.ch cyber IOC feeds',
|
||||
|
||||
@@ -16,7 +16,7 @@ export const SIGNUP_URLS: Partial<Record<RuntimeSecretKey, string>> = {
|
||||
OPENSKY_CLIENT_SECRET: 'https://opensky-network.org/login?view=registration',
|
||||
FINNHUB_API_KEY: 'https://finnhub.io/register',
|
||||
NASA_FIRMS_API_KEY: 'https://firms.modaps.eosdis.nasa.gov/api/area/',
|
||||
UC_DP_KEY: 'https://ucdp.uu.se/downloads/',
|
||||
UCDP_ACCESS_TOKEN: 'https://ucdp.uu.se/apidocs/',
|
||||
OLLAMA_API_URL: 'https://ollama.com/download',
|
||||
OLLAMA_MODEL: 'https://ollama.com/library',
|
||||
WTO_API_KEY: 'https://apiportal.wto.org/',
|
||||
@@ -51,7 +51,7 @@ export const HUMAN_LABELS: Record<RuntimeSecretKey, string> = {
|
||||
AISSTREAM_API_KEY: 'AISStream API Key',
|
||||
FINNHUB_API_KEY: 'Finnhub API Key',
|
||||
NASA_FIRMS_API_KEY: 'NASA FIRMS API Key',
|
||||
UC_DP_KEY: 'UCDP API Key',
|
||||
UCDP_ACCESS_TOKEN: 'UCDP Access Token',
|
||||
OLLAMA_API_URL: 'Ollama Server URL',
|
||||
OLLAMA_MODEL: 'Ollama Model',
|
||||
WORLDMONITOR_API_KEY: 'World Monitor License Key',
|
||||
@@ -85,7 +85,7 @@ export const SETTINGS_CATEGORIES: SettingsCategory[] = [
|
||||
{
|
||||
id: 'security',
|
||||
label: 'Security & Threats',
|
||||
features: ['internetOutages', 'acledConflicts', 'abuseChThreatIntel', 'alienvaultOtxThreatIntel', 'abuseIpdbThreatIntel'],
|
||||
features: ['internetOutages', 'acledConflicts', 'ucdpConflicts', 'abuseChThreatIntel', 'alienvaultOtxThreatIntel', 'abuseIpdbThreatIntel'],
|
||||
},
|
||||
{
|
||||
id: 'tracking',
|
||||
|
||||
Reference in New Issue
Block a user