mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-05-13 18:46:21 +02:00
9f378d533b
Replace Access-Control-Allow-Origin: * with an allowlist of our domains (worldmonitor.app, tech.worldmonitor.app, localhost dev ports, Tauri, and *.vercel.app previews). Prevents third parties from using the relay as a free proxy.
1314 lines
45 KiB
JavaScript
1314 lines
45 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* AIS WebSocket Relay Server
|
|
* Proxies aisstream.io data to browsers via WebSocket
|
|
*
|
|
* Deploy on Railway with:
|
|
* AISSTREAM_API_KEY=your_key
|
|
*
|
|
* Local: node scripts/ais-relay.cjs
|
|
*/
|
|
|
|
const http = require('http');
|
|
const zlib = require('zlib');
|
|
const { WebSocketServer, WebSocket } = require('ws');
|
|
|
|
const AISSTREAM_URL = 'wss://stream.aisstream.io/v0/stream';
|
|
const API_KEY = process.env.AISSTREAM_API_KEY || process.env.VITE_AISSTREAM_API_KEY;
|
|
const PORT = process.env.PORT || 3004;
|
|
|
|
if (!API_KEY) {
|
|
console.error('[Relay] Error: AISSTREAM_API_KEY environment variable not set');
|
|
console.error('[Relay] Get a free key at https://aisstream.io');
|
|
process.exit(1);
|
|
}
|
|
|
|
const MAX_WS_CLIENTS = 10; // Cap WS clients — app uses HTTP snapshots, not WS
|
|
|
|
let upstreamSocket = null;
|
|
let clients = new Set();
|
|
let messageCount = 0;
|
|
|
|
// gzip compress & send a response (reduces egress ~80% for JSON)
|
|
function sendCompressed(req, res, statusCode, headers, body) {
|
|
const acceptEncoding = req.headers['accept-encoding'] || '';
|
|
if (acceptEncoding.includes('gzip')) {
|
|
zlib.gzip(typeof body === 'string' ? Buffer.from(body) : body, (err, compressed) => {
|
|
if (err) {
|
|
res.writeHead(statusCode, headers);
|
|
res.end(body);
|
|
return;
|
|
}
|
|
res.writeHead(statusCode, { ...headers, 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding' });
|
|
res.end(compressed);
|
|
});
|
|
} else {
|
|
res.writeHead(statusCode, headers);
|
|
res.end(body);
|
|
}
|
|
}
|
|
|
|
// AIS aggregate state for snapshot API (server-side fanout)
|
|
const GRID_SIZE = 2;
|
|
const DENSITY_WINDOW = 30 * 60 * 1000; // 30 minutes
|
|
const GAP_THRESHOLD = 60 * 60 * 1000; // 1 hour
|
|
const SNAPSHOT_INTERVAL_MS = Math.max(2000, Number(process.env.AIS_SNAPSHOT_INTERVAL_MS || 5000));
|
|
const CANDIDATE_RETENTION_MS = 2 * 60 * 60 * 1000; // 2 hours
|
|
const MAX_DENSITY_ZONES = 200;
|
|
const MAX_CANDIDATE_REPORTS = 1500;
|
|
|
|
const vessels = new Map();
|
|
const vesselHistory = new Map();
|
|
const densityGrid = new Map();
|
|
const candidateReports = new Map();
|
|
|
|
let snapshotSequence = 0;
|
|
let lastSnapshot = null;
|
|
let lastSnapshotAt = 0;
|
|
|
|
const CHOKEPOINTS = [
|
|
{ name: 'Strait of Hormuz', lat: 26.5, lon: 56.5, radius: 2 },
|
|
{ name: 'Suez Canal', lat: 30.0, lon: 32.5, radius: 1 },
|
|
{ name: 'Strait of Malacca', lat: 2.5, lon: 101.5, radius: 2 },
|
|
{ name: 'Bab el-Mandeb', lat: 12.5, lon: 43.5, radius: 1.5 },
|
|
{ name: 'Panama Canal', lat: 9.0, lon: -79.5, radius: 1 },
|
|
{ name: 'Taiwan Strait', lat: 24.5, lon: 119.5, radius: 2 },
|
|
{ name: 'South China Sea', lat: 15.0, lon: 115.0, radius: 5 },
|
|
{ name: 'Black Sea', lat: 43.5, lon: 34.0, radius: 3 },
|
|
];
|
|
|
|
const NAVAL_PREFIX_RE = /^(USS|USNS|HMS|HMAS|HMCS|INS|JS|ROKS|TCG|FS|BNS|RFS|PLAN|PLA|CGC|PNS|KRI|ITS|SNS|MMSI)/i;
|
|
|
|
function getGridKey(lat, lon) {
|
|
const gridLat = Math.floor(lat / GRID_SIZE) * GRID_SIZE;
|
|
const gridLon = Math.floor(lon / GRID_SIZE) * GRID_SIZE;
|
|
return `${gridLat},${gridLon}`;
|
|
}
|
|
|
|
function isLikelyMilitaryCandidate(meta) {
|
|
const mmsi = String(meta?.MMSI || '');
|
|
const shipType = Number(meta?.ShipType);
|
|
const name = (meta?.ShipName || '').trim().toUpperCase();
|
|
|
|
if (Number.isFinite(shipType) && (shipType === 35 || shipType === 55 || (shipType >= 50 && shipType <= 59))) {
|
|
return true;
|
|
}
|
|
|
|
if (name && NAVAL_PREFIX_RE.test(name)) return true;
|
|
|
|
if (mmsi.length >= 9) {
|
|
const suffix = mmsi.substring(3);
|
|
if (suffix.startsWith('00') || suffix.startsWith('99')) return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
function processPositionReportForSnapshot(data) {
|
|
const meta = data?.MetaData;
|
|
const pos = data?.Message?.PositionReport;
|
|
if (!meta || !pos) return;
|
|
|
|
const mmsi = String(meta.MMSI || '');
|
|
if (!mmsi) return;
|
|
|
|
const lat = Number.isFinite(pos.Latitude) ? pos.Latitude : meta.latitude;
|
|
const lon = Number.isFinite(pos.Longitude) ? pos.Longitude : meta.longitude;
|
|
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return;
|
|
|
|
const now = Date.now();
|
|
|
|
vessels.set(mmsi, {
|
|
mmsi,
|
|
name: meta.ShipName || '',
|
|
lat,
|
|
lon,
|
|
timestamp: now,
|
|
shipType: meta.ShipType,
|
|
heading: pos.TrueHeading,
|
|
speed: pos.Sog,
|
|
course: pos.Cog,
|
|
});
|
|
|
|
const history = vesselHistory.get(mmsi) || [];
|
|
history.push(now);
|
|
if (history.length > 10) history.shift();
|
|
vesselHistory.set(mmsi, history);
|
|
|
|
const gridKey = getGridKey(lat, lon);
|
|
let cell = densityGrid.get(gridKey);
|
|
if (!cell) {
|
|
cell = {
|
|
lat: Math.floor(lat / GRID_SIZE) * GRID_SIZE + GRID_SIZE / 2,
|
|
lon: Math.floor(lon / GRID_SIZE) * GRID_SIZE + GRID_SIZE / 2,
|
|
vessels: new Set(),
|
|
lastUpdate: now,
|
|
previousCount: 0,
|
|
};
|
|
densityGrid.set(gridKey, cell);
|
|
}
|
|
cell.vessels.add(mmsi);
|
|
cell.lastUpdate = now;
|
|
|
|
if (isLikelyMilitaryCandidate(meta)) {
|
|
candidateReports.set(mmsi, {
|
|
mmsi,
|
|
name: meta.ShipName || '',
|
|
lat,
|
|
lon,
|
|
shipType: meta.ShipType,
|
|
heading: pos.TrueHeading,
|
|
speed: pos.Sog,
|
|
course: pos.Cog,
|
|
timestamp: now,
|
|
});
|
|
}
|
|
}
|
|
|
|
function cleanupAggregates() {
|
|
const now = Date.now();
|
|
const cutoff = now - DENSITY_WINDOW;
|
|
|
|
for (const [mmsi, vessel] of vessels) {
|
|
if (vessel.timestamp < cutoff) {
|
|
vessels.delete(mmsi);
|
|
}
|
|
}
|
|
|
|
for (const [mmsi, history] of vesselHistory) {
|
|
const filtered = history.filter((ts) => ts >= cutoff);
|
|
if (filtered.length === 0) {
|
|
vesselHistory.delete(mmsi);
|
|
} else {
|
|
vesselHistory.set(mmsi, filtered);
|
|
}
|
|
}
|
|
|
|
for (const [key, cell] of densityGrid) {
|
|
cell.previousCount = cell.vessels.size;
|
|
|
|
for (const mmsi of cell.vessels) {
|
|
const vessel = vessels.get(mmsi);
|
|
if (!vessel || vessel.timestamp < cutoff) {
|
|
cell.vessels.delete(mmsi);
|
|
}
|
|
}
|
|
|
|
if (cell.vessels.size === 0 && now - cell.lastUpdate > DENSITY_WINDOW * 2) {
|
|
densityGrid.delete(key);
|
|
}
|
|
}
|
|
|
|
for (const [mmsi, report] of candidateReports) {
|
|
if (report.timestamp < now - CANDIDATE_RETENTION_MS) {
|
|
candidateReports.delete(mmsi);
|
|
}
|
|
}
|
|
}
|
|
|
|
function detectDisruptions() {
|
|
const disruptions = [];
|
|
const now = Date.now();
|
|
|
|
for (const chokepoint of CHOKEPOINTS) {
|
|
let vesselCount = 0;
|
|
|
|
for (const vessel of vessels.values()) {
|
|
const distance = Math.sqrt(
|
|
Math.pow(vessel.lat - chokepoint.lat, 2) +
|
|
Math.pow(vessel.lon - chokepoint.lon, 2)
|
|
);
|
|
if (distance <= chokepoint.radius) {
|
|
vesselCount++;
|
|
}
|
|
}
|
|
|
|
if (vesselCount >= 5) {
|
|
const normalTraffic = chokepoint.radius * 10;
|
|
const severity = vesselCount > normalTraffic * 1.5
|
|
? 'high'
|
|
: vesselCount > normalTraffic
|
|
? 'elevated'
|
|
: 'low';
|
|
|
|
disruptions.push({
|
|
id: `chokepoint-${chokepoint.name.toLowerCase().replace(/\s+/g, '-')}`,
|
|
name: chokepoint.name,
|
|
type: 'chokepoint_congestion',
|
|
lat: chokepoint.lat,
|
|
lon: chokepoint.lon,
|
|
severity,
|
|
changePct: normalTraffic > 0 ? Math.round((vesselCount / normalTraffic - 1) * 100) : 0,
|
|
windowHours: 1,
|
|
vesselCount,
|
|
region: chokepoint.name,
|
|
description: `${vesselCount} vessels in ${chokepoint.name}`,
|
|
});
|
|
}
|
|
}
|
|
|
|
let darkShipCount = 0;
|
|
for (const history of vesselHistory.values()) {
|
|
if (history.length >= 2) {
|
|
const lastSeen = history[history.length - 1];
|
|
const secondLast = history[history.length - 2];
|
|
if (lastSeen - secondLast > GAP_THRESHOLD && now - lastSeen < 10 * 60 * 1000) {
|
|
darkShipCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (darkShipCount >= 1) {
|
|
disruptions.push({
|
|
id: 'global-gap-spike',
|
|
name: 'AIS Gap Spike Detected',
|
|
type: 'gap_spike',
|
|
lat: 0,
|
|
lon: 0,
|
|
severity: darkShipCount > 20 ? 'high' : darkShipCount > 10 ? 'elevated' : 'low',
|
|
changePct: darkShipCount * 10,
|
|
windowHours: 1,
|
|
darkShips: darkShipCount,
|
|
description: `${darkShipCount} vessels returned after extended AIS silence`,
|
|
});
|
|
}
|
|
|
|
return disruptions;
|
|
}
|
|
|
|
function calculateDensityZones() {
|
|
const zones = [];
|
|
const allCells = Array.from(densityGrid.values()).filter((c) => c.vessels.size >= 2);
|
|
if (allCells.length === 0) return zones;
|
|
|
|
const vesselCounts = allCells.map((c) => c.vessels.size);
|
|
const maxVessels = Math.max(...vesselCounts);
|
|
const minVessels = Math.min(...vesselCounts);
|
|
|
|
for (const [key, cell] of densityGrid) {
|
|
if (cell.vessels.size < 2) continue;
|
|
|
|
const logMax = Math.log(maxVessels + 1);
|
|
const logMin = Math.log(minVessels + 1);
|
|
const logCurrent = Math.log(cell.vessels.size + 1);
|
|
|
|
const intensity = logMax > logMin
|
|
? 0.2 + (0.8 * (logCurrent - logMin) / (logMax - logMin))
|
|
: 0.5;
|
|
|
|
const deltaPct = cell.previousCount > 0
|
|
? Math.round(((cell.vessels.size - cell.previousCount) / cell.previousCount) * 100)
|
|
: 0;
|
|
|
|
zones.push({
|
|
id: `density-${key}`,
|
|
name: `Zone ${key}`,
|
|
lat: cell.lat,
|
|
lon: cell.lon,
|
|
intensity,
|
|
deltaPct,
|
|
shipsPerDay: cell.vessels.size * 48,
|
|
note: cell.vessels.size >= 10 ? 'High traffic area' : undefined,
|
|
});
|
|
}
|
|
|
|
return zones
|
|
.sort((a, b) => b.intensity - a.intensity)
|
|
.slice(0, MAX_DENSITY_ZONES);
|
|
}
|
|
|
|
function getCandidateReportsSnapshot() {
|
|
return Array.from(candidateReports.values())
|
|
.sort((a, b) => b.timestamp - a.timestamp)
|
|
.slice(0, MAX_CANDIDATE_REPORTS);
|
|
}
|
|
|
|
function buildSnapshot() {
|
|
const now = Date.now();
|
|
if (lastSnapshot && now - lastSnapshotAt < Math.floor(SNAPSHOT_INTERVAL_MS / 2)) {
|
|
return lastSnapshot;
|
|
}
|
|
|
|
cleanupAggregates();
|
|
snapshotSequence++;
|
|
|
|
lastSnapshot = {
|
|
sequence: snapshotSequence,
|
|
timestamp: new Date(now).toISOString(),
|
|
status: {
|
|
connected: upstreamSocket?.readyState === WebSocket.OPEN,
|
|
vessels: vessels.size,
|
|
messages: messageCount,
|
|
clients: clients.size,
|
|
},
|
|
disruptions: detectDisruptions(),
|
|
density: calculateDensityZones(),
|
|
};
|
|
lastSnapshotAt = now;
|
|
return lastSnapshot;
|
|
}
|
|
|
|
setInterval(() => {
|
|
if (upstreamSocket?.readyState === WebSocket.OPEN || vessels.size > 0) {
|
|
buildSnapshot();
|
|
}
|
|
}, SNAPSHOT_INTERVAL_MS);
|
|
|
|
// UCDP GED Events cache (persistent in-memory — Railway advantage)
|
|
const UCDP_CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
const UCDP_PAGE_SIZE = 1000;
|
|
const UCDP_MAX_PAGES = 12;
|
|
const UCDP_FETCH_TIMEOUT = 30000; // 30s per page (no Railway limit)
|
|
const UCDP_TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000;
|
|
|
|
let ucdpCache = { data: null, timestamp: 0 };
|
|
let ucdpFetchInProgress = false;
|
|
|
|
const UCDP_VIOLENCE_TYPE_MAP = {
|
|
1: 'state-based',
|
|
2: 'non-state',
|
|
3: 'one-sided',
|
|
};
|
|
|
|
function ucdpParseDateMs(value) {
|
|
if (!value) return NaN;
|
|
return Date.parse(String(value));
|
|
}
|
|
|
|
function ucdpGetMaxDateMs(events) {
|
|
let maxMs = NaN;
|
|
for (const event of events) {
|
|
const ms = ucdpParseDateMs(event?.date_start);
|
|
if (!Number.isFinite(ms)) continue;
|
|
if (!Number.isFinite(maxMs) || ms > maxMs) maxMs = ms;
|
|
}
|
|
return maxMs;
|
|
}
|
|
|
|
function ucdpBuildVersionCandidates() {
|
|
const year = new Date().getFullYear() - 2000;
|
|
return Array.from(new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1']));
|
|
}
|
|
|
|
async function ucdpFetchPage(version, page) {
|
|
const https = require('https');
|
|
const url = `https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const req = https.get(url, { headers: { Accept: 'application/json' }, timeout: UCDP_FETCH_TIMEOUT }, (res) => {
|
|
if (res.statusCode !== 200) {
|
|
res.resume();
|
|
return reject(new Error(`UCDP API ${res.statusCode} (v${version} p${page})`));
|
|
}
|
|
let data = '';
|
|
res.on('data', chunk => data += chunk);
|
|
res.on('end', () => {
|
|
try { resolve(JSON.parse(data)); }
|
|
catch (e) { reject(new Error('UCDP JSON parse error')); }
|
|
});
|
|
});
|
|
req.on('error', reject);
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('UCDP timeout')); });
|
|
});
|
|
}
|
|
|
|
async function ucdpDiscoverVersion() {
|
|
const candidates = ucdpBuildVersionCandidates();
|
|
for (const version of candidates) {
|
|
try {
|
|
const page0 = await ucdpFetchPage(version, 0);
|
|
if (Array.isArray(page0?.Result)) return { version, page0 };
|
|
} catch { /* next candidate */ }
|
|
}
|
|
throw new Error('No valid UCDP GED version found');
|
|
}
|
|
|
|
async function ucdpFetchAllEvents() {
|
|
const { version, page0 } = await ucdpDiscoverVersion();
|
|
const totalPages = Math.max(1, Number(page0?.TotalPages) || 1);
|
|
const newestPage = totalPages - 1;
|
|
|
|
let allEvents = [];
|
|
let latestDatasetMs = NaN;
|
|
|
|
for (let offset = 0; offset < UCDP_MAX_PAGES && (newestPage - offset) >= 0; offset++) {
|
|
const page = newestPage - offset;
|
|
const rawData = page === 0 ? page0 : await ucdpFetchPage(version, page);
|
|
const events = Array.isArray(rawData?.Result) ? rawData.Result : [];
|
|
allEvents = allEvents.concat(events);
|
|
|
|
const pageMaxMs = ucdpGetMaxDateMs(events);
|
|
if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) {
|
|
latestDatasetMs = pageMaxMs;
|
|
}
|
|
if (Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) {
|
|
if (pageMaxMs < latestDatasetMs - UCDP_TRAILING_WINDOW_MS) break;
|
|
}
|
|
console.log(`[UCDP] Fetched v${version} page ${page} (${events.length} events)`);
|
|
}
|
|
|
|
const sanitized = allEvents
|
|
.filter(e => {
|
|
if (!Number.isFinite(latestDatasetMs)) return true;
|
|
const ms = ucdpParseDateMs(e?.date_start);
|
|
return Number.isFinite(ms) && ms >= (latestDatasetMs - UCDP_TRAILING_WINDOW_MS);
|
|
})
|
|
.map(e => ({
|
|
id: String(e.id || ''),
|
|
date_start: e.date_start || '',
|
|
date_end: e.date_end || '',
|
|
latitude: Number(e.latitude) || 0,
|
|
longitude: Number(e.longitude) || 0,
|
|
country: e.country || '',
|
|
side_a: (e.side_a || '').substring(0, 200),
|
|
side_b: (e.side_b || '').substring(0, 200),
|
|
deaths_best: Number(e.best) || 0,
|
|
deaths_low: Number(e.low) || 0,
|
|
deaths_high: Number(e.high) || 0,
|
|
type_of_violence: UCDP_VIOLENCE_TYPE_MAP[e.type_of_violence] || 'state-based',
|
|
source_original: (e.source_original || '').substring(0, 300),
|
|
}))
|
|
.sort((a, b) => {
|
|
const bMs = ucdpParseDateMs(b.date_start);
|
|
const aMs = ucdpParseDateMs(a.date_start);
|
|
return (Number.isFinite(bMs) ? bMs : 0) - (Number.isFinite(aMs) ? aMs : 0);
|
|
});
|
|
|
|
return {
|
|
success: true,
|
|
count: sanitized.length,
|
|
data: sanitized,
|
|
version,
|
|
cached_at: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
async function handleUcdpEventsRequest(req, res) {
|
|
const now = Date.now();
|
|
|
|
if (ucdpCache.data && now - ucdpCache.timestamp < UCDP_CACHE_TTL_MS) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=3600',
|
|
'X-Cache': 'HIT',
|
|
}, JSON.stringify(ucdpCache.data));
|
|
}
|
|
|
|
if (ucdpCache.data && !ucdpFetchInProgress) {
|
|
ucdpFetchInProgress = true;
|
|
ucdpFetchAllEvents()
|
|
.then(result => {
|
|
ucdpCache = { data: result, timestamp: Date.now() };
|
|
console.log(`[UCDP] Background refresh: ${result.count} events (v${result.version})`);
|
|
})
|
|
.catch(err => console.error('[UCDP] Background refresh error:', err.message))
|
|
.finally(() => { ucdpFetchInProgress = false; });
|
|
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=600',
|
|
'X-Cache': 'STALE',
|
|
}, JSON.stringify(ucdpCache.data));
|
|
}
|
|
|
|
if (ucdpFetchInProgress) {
|
|
res.writeHead(202, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify({ success: false, count: 0, data: [], cached_at: '', message: 'Fetch in progress' }));
|
|
}
|
|
|
|
try {
|
|
ucdpFetchInProgress = true;
|
|
console.log('[UCDP] Cold fetch starting...');
|
|
const result = await ucdpFetchAllEvents();
|
|
ucdpCache = { data: result, timestamp: Date.now() };
|
|
ucdpFetchInProgress = false;
|
|
console.log(`[UCDP] Cold fetch complete: ${result.count} events (v${result.version})`);
|
|
|
|
sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=3600',
|
|
'X-Cache': 'MISS',
|
|
}, JSON.stringify(result));
|
|
} catch (err) {
|
|
ucdpFetchInProgress = false;
|
|
console.error('[UCDP] Fetch error:', err.message);
|
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ success: false, error: err.message, count: 0, data: [] }));
|
|
}
|
|
}
|
|
|
|
// ── Response caches (eliminates ~1.2TB/day OpenSky + ~30GB/day RSS egress) ──
|
|
const openskyResponseCache = new Map(); // key: sorted query params → { data, timestamp }
|
|
const OPENSKY_CACHE_TTL_MS = 30 * 1000; // 30s — OpenSky updates every ~10s but 58 clients hammer it
|
|
const rssResponseCache = new Map(); // key: feed URL → { data, contentType, timestamp }
|
|
const RSS_CACHE_TTL_MS = 5 * 60 * 1000; // 5 min — RSS feeds rarely update faster
|
|
|
|
// OpenSky OAuth2 token cache
|
|
let openskyToken = null;
|
|
let openskyTokenExpiry = 0;
|
|
|
|
async function getOpenSkyToken() {
|
|
const clientId = process.env.OPENSKY_CLIENT_ID;
|
|
const clientSecret = process.env.OPENSKY_CLIENT_SECRET;
|
|
|
|
if (!clientId || !clientSecret) {
|
|
return null;
|
|
}
|
|
|
|
// Return cached token if still valid (with 60s buffer)
|
|
if (openskyToken && Date.now() < openskyTokenExpiry - 60000) {
|
|
return openskyToken;
|
|
}
|
|
|
|
try {
|
|
console.log('[Relay] Fetching new OpenSky OAuth2 token...');
|
|
const https = require('https');
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const postData = `grant_type=client_credentials&client_id=${encodeURIComponent(clientId)}&client_secret=${encodeURIComponent(clientSecret)}`;
|
|
|
|
const req = https.request({
|
|
hostname: 'auth.opensky-network.org',
|
|
port: 443,
|
|
path: '/auth/realms/opensky-network/protocol/openid-connect/token',
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/x-www-form-urlencoded',
|
|
'Content-Length': Buffer.byteLength(postData),
|
|
},
|
|
timeout: 10000
|
|
}, (res) => {
|
|
let data = '';
|
|
res.on('data', chunk => data += chunk);
|
|
res.on('end', () => {
|
|
try {
|
|
const json = JSON.parse(data);
|
|
if (json.access_token) {
|
|
openskyToken = json.access_token;
|
|
// Token valid for 30 min, cache with expiry
|
|
openskyTokenExpiry = Date.now() + (json.expires_in || 1800) * 1000;
|
|
console.log('[Relay] OpenSky token acquired, expires in', json.expires_in, 'seconds');
|
|
resolve(openskyToken);
|
|
} else {
|
|
console.error('[Relay] OpenSky token error:', json.error || 'Unknown');
|
|
resolve(null);
|
|
}
|
|
} catch (e) {
|
|
console.error('[Relay] OpenSky token parse error:', e.message);
|
|
resolve(null);
|
|
}
|
|
});
|
|
});
|
|
|
|
req.on('error', (err) => {
|
|
console.error('[Relay] OpenSky token request error:', err.message);
|
|
resolve(null);
|
|
});
|
|
|
|
req.on('timeout', () => {
|
|
req.destroy();
|
|
resolve(null);
|
|
});
|
|
|
|
req.write(postData);
|
|
req.end();
|
|
});
|
|
} catch (err) {
|
|
console.error('[Relay] OpenSky token error:', err.message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function handleOpenSkyRequest(req, res, PORT) {
|
|
try {
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
const params = url.searchParams;
|
|
|
|
// Build cache key from sorted bounding box params
|
|
const cacheKey = ['lamin', 'lomin', 'lamax', 'lomax']
|
|
.map(k => params.get(k) || '')
|
|
.join(',');
|
|
|
|
const cached = openskyResponseCache.get(cacheKey);
|
|
if (cached && Date.now() - cached.timestamp < OPENSKY_CACHE_TTL_MS) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=30',
|
|
'X-Cache': 'HIT',
|
|
}, cached.data);
|
|
}
|
|
|
|
const token = await getOpenSkyToken();
|
|
if (!token) {
|
|
res.writeHead(503, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: 'OpenSky not configured or auth failed', time: Date.now(), states: [] }));
|
|
return;
|
|
}
|
|
|
|
let openskyUrl = 'https://opensky-network.org/api/states/all';
|
|
const queryParams = [];
|
|
for (const key of ['lamin', 'lomin', 'lamax', 'lomax']) {
|
|
if (params.has(key)) queryParams.push(`${key}=${params.get(key)}`);
|
|
}
|
|
if (queryParams.length > 0) {
|
|
openskyUrl += '?' + queryParams.join('&');
|
|
}
|
|
|
|
console.log('[Relay] OpenSky request (MISS):', openskyUrl);
|
|
|
|
const https = require('https');
|
|
const request = https.get(openskyUrl, {
|
|
headers: {
|
|
'Accept': 'application/json',
|
|
'User-Agent': 'WorldMonitor/1.0',
|
|
'Authorization': `Bearer ${token}`,
|
|
},
|
|
timeout: 15000
|
|
}, (response) => {
|
|
let data = '';
|
|
response.on('data', chunk => data += chunk);
|
|
response.on('end', () => {
|
|
if (response.statusCode === 401) {
|
|
openskyToken = null;
|
|
openskyTokenExpiry = 0;
|
|
}
|
|
if (response.statusCode === 200) {
|
|
openskyResponseCache.set(cacheKey, { data, timestamp: Date.now() });
|
|
}
|
|
sendCompressed(req, res, response.statusCode, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=30',
|
|
'X-Cache': 'MISS',
|
|
}, data);
|
|
});
|
|
});
|
|
|
|
request.on('error', (err) => {
|
|
console.error('[Relay] OpenSky error:', err.message);
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE' }, cached.data);
|
|
}
|
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: err.message, time: Date.now(), states: null }));
|
|
});
|
|
|
|
request.on('timeout', () => {
|
|
request.destroy();
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE' }, cached.data);
|
|
}
|
|
res.writeHead(504, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: 'Request timeout', time: Date.now(), states: null }));
|
|
});
|
|
} catch (err) {
|
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: err.message, time: Date.now(), states: null }));
|
|
}
|
|
}
|
|
|
|
// ── World Bank proxy (World Bank blocks Vercel edge IPs with 403) ──
|
|
const worldbankCache = new Map(); // key: query string → { data, timestamp }
|
|
const WORLDBANK_CACHE_TTL_MS = 30 * 60 * 1000; // 30 min — data rarely changes
|
|
|
|
function handleWorldBankRequest(req, res) {
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
const qs = url.search || '';
|
|
const cacheKey = qs;
|
|
|
|
const cached = worldbankCache.get(cacheKey);
|
|
if (cached && Date.now() - cached.timestamp < WORLDBANK_CACHE_TTL_MS) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=1800',
|
|
'X-Cache': 'HIT',
|
|
}, cached.data);
|
|
}
|
|
|
|
const targetUrl = `https://api.worldbank.org/v2${qs.includes('action=indicators') ? '' : '/country'}${url.pathname.replace('/worldbank', '')}${qs}`;
|
|
// Passthrough: forward query params to the Vercel edge handler format
|
|
// The client sends the same params as /api/worldbank, so we re-fetch from upstream
|
|
const wbParams = new URLSearchParams(url.searchParams);
|
|
const action = wbParams.get('action');
|
|
|
|
if (action === 'indicators') {
|
|
// Static response — return indicator list directly (same as api/worldbank.js)
|
|
const indicators = {
|
|
'IT.NET.USER.ZS': 'Internet Users (% of population)',
|
|
'IT.CEL.SETS.P2': 'Mobile Subscriptions (per 100 people)',
|
|
'IT.NET.BBND.P2': 'Fixed Broadband Subscriptions (per 100 people)',
|
|
'IT.NET.SECR.P6': 'Secure Internet Servers (per million people)',
|
|
'GB.XPD.RSDV.GD.ZS': 'R&D Expenditure (% of GDP)',
|
|
'IP.PAT.RESD': 'Patent Applications (residents)',
|
|
'IP.PAT.NRES': 'Patent Applications (non-residents)',
|
|
'IP.TMK.TOTL': 'Trademark Applications',
|
|
'TX.VAL.TECH.MF.ZS': 'High-Tech Exports (% of manufactured exports)',
|
|
'BX.GSR.CCIS.ZS': 'ICT Service Exports (% of service exports)',
|
|
'TM.VAL.ICTG.ZS.UN': 'ICT Goods Imports (% of total goods imports)',
|
|
'SE.TER.ENRR': 'Tertiary Education Enrollment (%)',
|
|
'SE.XPD.TOTL.GD.ZS': 'Education Expenditure (% of GDP)',
|
|
'NY.GDP.MKTP.KD.ZG': 'GDP Growth (annual %)',
|
|
'NY.GDP.PCAP.CD': 'GDP per Capita (current US$)',
|
|
'NE.EXP.GNFS.ZS': 'Exports of Goods & Services (% of GDP)',
|
|
};
|
|
const defaultCountries = [
|
|
'USA','CHN','JPN','DEU','KOR','GBR','IND','ISR','SGP','TWN',
|
|
'FRA','CAN','SWE','NLD','CHE','FIN','IRL','AUS','BRA','IDN',
|
|
'ARE','SAU','QAT','BHR','EGY','TUR','MYS','THA','VNM','PHL',
|
|
'ESP','ITA','POL','CZE','DNK','NOR','AUT','BEL','PRT','EST',
|
|
'MEX','ARG','CHL','COL','ZAF','NGA','KEN',
|
|
];
|
|
const body = JSON.stringify({ indicators, defaultCountries });
|
|
worldbankCache.set(cacheKey, { data: body, timestamp: Date.now() });
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=86400',
|
|
'X-Cache': 'MISS',
|
|
}, body);
|
|
}
|
|
|
|
const indicator = wbParams.get('indicator');
|
|
if (!indicator) {
|
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify({ error: 'Missing indicator parameter' }));
|
|
}
|
|
|
|
const country = wbParams.get('country');
|
|
const countries = wbParams.get('countries');
|
|
const years = parseInt(wbParams.get('years') || '5', 10);
|
|
let countryList = country || (countries ? countries.split(',').join(';') : [
|
|
'USA','CHN','JPN','DEU','KOR','GBR','IND','ISR','SGP','TWN',
|
|
'FRA','CAN','SWE','NLD','CHE','FIN','IRL','AUS','BRA','IDN',
|
|
'ARE','SAU','QAT','BHR','EGY','TUR','MYS','THA','VNM','PHL',
|
|
'ESP','ITA','POL','CZE','DNK','NOR','AUT','BEL','PRT','EST',
|
|
'MEX','ARG','CHL','COL','ZAF','NGA','KEN',
|
|
].join(';'));
|
|
|
|
const currentYear = new Date().getFullYear();
|
|
const startYear = currentYear - years;
|
|
const TECH_INDICATORS = {
|
|
'IT.NET.USER.ZS': 'Internet Users (% of population)',
|
|
'IT.CEL.SETS.P2': 'Mobile Subscriptions (per 100 people)',
|
|
'IT.NET.BBND.P2': 'Fixed Broadband Subscriptions (per 100 people)',
|
|
'IT.NET.SECR.P6': 'Secure Internet Servers (per million people)',
|
|
'GB.XPD.RSDV.GD.ZS': 'R&D Expenditure (% of GDP)',
|
|
'IP.PAT.RESD': 'Patent Applications (residents)',
|
|
'IP.PAT.NRES': 'Patent Applications (non-residents)',
|
|
'IP.TMK.TOTL': 'Trademark Applications',
|
|
'TX.VAL.TECH.MF.ZS': 'High-Tech Exports (% of manufactured exports)',
|
|
'BX.GSR.CCIS.ZS': 'ICT Service Exports (% of service exports)',
|
|
'TM.VAL.ICTG.ZS.UN': 'ICT Goods Imports (% of total goods imports)',
|
|
'SE.TER.ENRR': 'Tertiary Education Enrollment (%)',
|
|
'SE.XPD.TOTL.GD.ZS': 'Education Expenditure (% of GDP)',
|
|
'NY.GDP.MKTP.KD.ZG': 'GDP Growth (annual %)',
|
|
'NY.GDP.PCAP.CD': 'GDP per Capita (current US$)',
|
|
'NE.EXP.GNFS.ZS': 'Exports of Goods & Services (% of GDP)',
|
|
};
|
|
|
|
const wbUrl = `https://api.worldbank.org/v2/country/${countryList}/indicator/${encodeURIComponent(indicator)}?format=json&date=${startYear}:${currentYear}&per_page=1000`;
|
|
|
|
console.log('[Relay] World Bank request (MISS):', indicator);
|
|
|
|
const https = require('https');
|
|
const request = https.get(wbUrl, {
|
|
headers: {
|
|
'Accept': 'application/json',
|
|
'User-Agent': 'Mozilla/5.0 (compatible; WorldMonitor/1.0; +https://worldmonitor.app)',
|
|
},
|
|
timeout: 15000,
|
|
}, (response) => {
|
|
if (response.statusCode !== 200) {
|
|
res.writeHead(response.statusCode, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify({ error: `World Bank API ${response.statusCode}` }));
|
|
}
|
|
let rawData = '';
|
|
response.on('data', chunk => rawData += chunk);
|
|
response.on('end', () => {
|
|
try {
|
|
const parsed = JSON.parse(rawData);
|
|
// Transform raw World Bank response to match client-expected format
|
|
if (!parsed || !Array.isArray(parsed) || parsed.length < 2 || !parsed[1]) {
|
|
const empty = JSON.stringify({
|
|
indicator,
|
|
indicatorName: TECH_INDICATORS[indicator] || indicator,
|
|
metadata: { page: 1, pages: 1, total: 0 },
|
|
byCountry: {}, latestByCountry: {}, timeSeries: [],
|
|
});
|
|
worldbankCache.set(cacheKey, { data: empty, timestamp: Date.now() });
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=1800',
|
|
'X-Cache': 'MISS',
|
|
}, empty);
|
|
}
|
|
|
|
const [metadata, records] = parsed;
|
|
const transformed = {
|
|
indicator,
|
|
indicatorName: TECH_INDICATORS[indicator] || (records[0]?.indicator?.value || indicator),
|
|
metadata: { page: metadata.page, pages: metadata.pages, total: metadata.total },
|
|
byCountry: {}, latestByCountry: {}, timeSeries: [],
|
|
};
|
|
|
|
for (const record of records || []) {
|
|
const cc = record.countryiso3code || record.country?.id;
|
|
const cn = record.country?.value;
|
|
const yr = record.date;
|
|
const val = record.value;
|
|
if (!cc || val === null) continue;
|
|
if (!transformed.byCountry[cc]) transformed.byCountry[cc] = { code: cc, name: cn, values: [] };
|
|
transformed.byCountry[cc].values.push({ year: yr, value: val });
|
|
if (!transformed.latestByCountry[cc] || yr > transformed.latestByCountry[cc].year) {
|
|
transformed.latestByCountry[cc] = { code: cc, name: cn, year: yr, value: val };
|
|
}
|
|
transformed.timeSeries.push({ countryCode: cc, countryName: cn, year: yr, value: val });
|
|
}
|
|
for (const c of Object.values(transformed.byCountry)) c.values.sort((a, b) => a.year - b.year);
|
|
transformed.timeSeries.sort((a, b) => b.year - a.year || a.countryCode.localeCompare(b.countryCode));
|
|
|
|
const body = JSON.stringify(transformed);
|
|
worldbankCache.set(cacheKey, { data: body, timestamp: Date.now() });
|
|
sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=1800',
|
|
'X-Cache': 'MISS',
|
|
}, body);
|
|
} catch (e) {
|
|
console.error('[Relay] World Bank parse error:', e.message);
|
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: 'Parse error' }));
|
|
}
|
|
});
|
|
});
|
|
request.on('error', (err) => {
|
|
console.error('[Relay] World Bank error:', err.message);
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'X-Cache': 'STALE',
|
|
}, cached.data);
|
|
}
|
|
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: err.message }));
|
|
});
|
|
request.on('timeout', () => {
|
|
request.destroy();
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'X-Cache': 'STALE',
|
|
}, cached.data);
|
|
}
|
|
res.writeHead(504, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: 'World Bank request timeout' }));
|
|
});
|
|
}
|
|
|
|
// ── Polymarket proxy (Cloudflare JA3 blocks Vercel edge runtime) ──
|
|
const polymarketCache = new Map(); // key: query string → { data, timestamp }
|
|
const POLYMARKET_CACHE_TTL_MS = 2 * 60 * 1000; // 2 min — market data changes frequently
|
|
|
|
function handlePolymarketRequest(req, res) {
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
const cacheKey = url.search || '';
|
|
|
|
const cached = polymarketCache.get(cacheKey);
|
|
if (cached && Date.now() - cached.timestamp < POLYMARKET_CACHE_TTL_MS) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=120',
|
|
'X-Cache': 'HIT',
|
|
'X-Polymarket-Source': 'railway-cache',
|
|
}, cached.data);
|
|
}
|
|
|
|
const endpoint = url.searchParams.get('endpoint') || 'markets';
|
|
const params = new URLSearchParams();
|
|
params.set('closed', url.searchParams.get('closed') || 'false');
|
|
params.set('order', url.searchParams.get('order') || 'volume');
|
|
params.set('ascending', url.searchParams.get('ascending') || 'false');
|
|
const limit = Math.max(1, Math.min(100, parseInt(url.searchParams.get('limit') || '50', 10) || 50));
|
|
params.set('limit', String(limit));
|
|
const tag = url.searchParams.get('tag') || url.searchParams.get('tag_slug');
|
|
if (tag && endpoint === 'events') params.set('tag_slug', tag.replace(/[^a-z0-9-]/gi, '').slice(0, 100));
|
|
|
|
const gammaUrl = `https://gamma-api.polymarket.com/${endpoint}?${params}`;
|
|
console.log('[Relay] Polymarket request (MISS):', endpoint, tag || '');
|
|
|
|
const https = require('https');
|
|
const request = https.get(gammaUrl, {
|
|
headers: { 'Accept': 'application/json' },
|
|
timeout: 10000,
|
|
}, (response) => {
|
|
if (response.statusCode !== 200) {
|
|
console.error(`[Relay] Polymarket upstream ${response.statusCode}`);
|
|
res.writeHead(response.statusCode, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify([]));
|
|
}
|
|
let data = '';
|
|
response.on('data', chunk => data += chunk);
|
|
response.on('end', () => {
|
|
polymarketCache.set(cacheKey, { data, timestamp: Date.now() });
|
|
sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=120',
|
|
'X-Cache': 'MISS',
|
|
'X-Polymarket-Source': 'railway',
|
|
}, data);
|
|
});
|
|
});
|
|
request.on('error', (err) => {
|
|
console.error('[Relay] Polymarket error:', err.message);
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'X-Cache': 'STALE',
|
|
'X-Polymarket-Source': 'railway-stale',
|
|
}, cached.data);
|
|
}
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify([]));
|
|
});
|
|
request.on('timeout', () => {
|
|
request.destroy();
|
|
if (cached) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'X-Cache': 'STALE',
|
|
'X-Polymarket-Source': 'railway-stale',
|
|
}, cached.data);
|
|
}
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify([]));
|
|
});
|
|
}
|
|
|
|
// Periodic cache cleanup to prevent memory leaks
|
|
setInterval(() => {
|
|
const now = Date.now();
|
|
for (const [key, entry] of openskyResponseCache) {
|
|
if (now - entry.timestamp > OPENSKY_CACHE_TTL_MS * 2) openskyResponseCache.delete(key);
|
|
}
|
|
for (const [key, entry] of rssResponseCache) {
|
|
if (now - entry.timestamp > RSS_CACHE_TTL_MS * 2) rssResponseCache.delete(key);
|
|
}
|
|
for (const [key, entry] of worldbankCache) {
|
|
if (now - entry.timestamp > WORLDBANK_CACHE_TTL_MS * 2) worldbankCache.delete(key);
|
|
}
|
|
for (const [key, entry] of polymarketCache) {
|
|
if (now - entry.timestamp > POLYMARKET_CACHE_TTL_MS * 2) polymarketCache.delete(key);
|
|
}
|
|
}, 60 * 1000);
|
|
|
|
// CORS origin allowlist — only our domains can use this relay
|
|
const ALLOWED_ORIGINS = [
|
|
'https://worldmonitor.app',
|
|
'https://tech.worldmonitor.app',
|
|
'http://localhost:5173', // Vite dev
|
|
'http://localhost:5174', // Vite dev alt port
|
|
'http://localhost:4173', // Vite preview
|
|
'https://localhost', // Tauri desktop
|
|
'tauri://localhost', // Tauri iOS/macOS
|
|
];
|
|
|
|
function getCorsOrigin(req) {
|
|
const origin = req.headers.origin || '';
|
|
if (ALLOWED_ORIGINS.includes(origin)) return origin;
|
|
// Allow Vercel preview deployments
|
|
if (origin.endsWith('.vercel.app')) return origin;
|
|
return '';
|
|
}
|
|
|
|
const server = http.createServer(async (req, res) => {
|
|
const corsOrigin = getCorsOrigin(req);
|
|
if (corsOrigin) {
|
|
res.setHeader('Access-Control-Allow-Origin', corsOrigin);
|
|
res.setHeader('Vary', 'Origin');
|
|
}
|
|
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
|
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
|
|
|
// Handle CORS preflight
|
|
if (req.method === 'OPTIONS') {
|
|
res.writeHead(corsOrigin ? 204 : 403);
|
|
return res.end();
|
|
}
|
|
|
|
if (req.url === '/health' || req.url === '/') {
|
|
sendCompressed(req, res, 200, { 'Content-Type': 'application/json' }, JSON.stringify({
|
|
status: 'ok',
|
|
clients: clients.size,
|
|
messages: messageCount,
|
|
connected: upstreamSocket?.readyState === WebSocket.OPEN,
|
|
vessels: vessels.size,
|
|
densityZones: Array.from(densityGrid.values()).filter(c => c.vessels.size >= 2).length,
|
|
cache: {
|
|
opensky: openskyResponseCache.size,
|
|
rss: rssResponseCache.size,
|
|
ucdp: ucdpCache.data ? 'warm' : 'cold',
|
|
worldbank: worldbankCache.size,
|
|
polymarket: polymarketCache.size,
|
|
},
|
|
}));
|
|
} else if (req.url.startsWith('/ais/snapshot')) {
|
|
// Aggregated AIS snapshot for server-side fanout
|
|
connectUpstream();
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
const includeCandidates = url.searchParams.get('candidates') === 'true';
|
|
const snapshot = buildSnapshot();
|
|
const payload = includeCandidates
|
|
? { ...snapshot, candidateReports: getCandidateReportsSnapshot() }
|
|
: { ...snapshot, candidateReports: [] };
|
|
|
|
sendCompressed(req, res, 200, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'public, max-age=2'
|
|
}, JSON.stringify(payload));
|
|
} else if (req.url.startsWith('/rss')) {
|
|
// Proxy RSS feeds that block Vercel IPs
|
|
try {
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
const feedUrl = url.searchParams.get('url');
|
|
|
|
if (!feedUrl) {
|
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify({ error: 'Missing url parameter' }));
|
|
}
|
|
|
|
// Allow domains that block Vercel IPs (must match feeds.ts railwayRss usage)
|
|
const allowedDomains = [
|
|
// Original
|
|
'rss.cnn.com',
|
|
'www.defensenews.com',
|
|
'layoffs.fyi',
|
|
// International Organizations
|
|
'news.un.org',
|
|
'www.cisa.gov',
|
|
'www.iaea.org',
|
|
'www.who.int',
|
|
'www.crisisgroup.org',
|
|
// Middle East & Regional News
|
|
'english.alarabiya.net',
|
|
'www.arabnews.com',
|
|
'www.timesofisrael.com',
|
|
'www.scmp.com',
|
|
'kyivindependent.com',
|
|
'www.themoscowtimes.com',
|
|
// Africa
|
|
'feeds.24.com',
|
|
'feeds.capi24.com', // News24 redirect destination
|
|
'www.atlanticcouncil.org',
|
|
];
|
|
const parsed = new URL(feedUrl);
|
|
if (!allowedDomains.includes(parsed.hostname)) {
|
|
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
return res.end(JSON.stringify({ error: 'Domain not allowed on Railway proxy' }));
|
|
}
|
|
|
|
// Serve from cache if fresh (5 min TTL)
|
|
const rssCached = rssResponseCache.get(feedUrl);
|
|
if (rssCached && Date.now() - rssCached.timestamp < RSS_CACHE_TTL_MS) {
|
|
return sendCompressed(req, res, 200, {
|
|
'Content-Type': rssCached.contentType || 'application/xml',
|
|
'Cache-Control': 'public, max-age=300',
|
|
'X-Cache': 'HIT',
|
|
}, rssCached.data);
|
|
}
|
|
|
|
console.log('[Relay] RSS request (MISS):', feedUrl);
|
|
|
|
const https = require('https');
|
|
const http = require('http');
|
|
|
|
let responseHandled = false;
|
|
|
|
const sendError = (statusCode, message) => {
|
|
if (responseHandled || res.headersSent) return;
|
|
responseHandled = true;
|
|
res.writeHead(statusCode, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: message }));
|
|
};
|
|
|
|
const fetchWithRedirects = (url, redirectCount = 0) => {
|
|
if (redirectCount > 3) {
|
|
return sendError(502, 'Too many redirects');
|
|
}
|
|
|
|
const protocol = url.startsWith('https') ? https : http;
|
|
const request = protocol.get(url, {
|
|
headers: {
|
|
'Accept': 'application/rss+xml, application/xml, text/xml, */*',
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
},
|
|
timeout: 15000
|
|
}, (response) => {
|
|
if ([301, 302, 303, 307, 308].includes(response.statusCode) && response.headers.location) {
|
|
const redirectUrl = response.headers.location.startsWith('http')
|
|
? response.headers.location
|
|
: new URL(response.headers.location, url).href;
|
|
console.log(`[Relay] Following redirect to: ${redirectUrl}`);
|
|
return fetchWithRedirects(redirectUrl, redirectCount + 1);
|
|
}
|
|
|
|
const encoding = response.headers['content-encoding'];
|
|
let stream = response;
|
|
if (encoding === 'gzip' || encoding === 'deflate') {
|
|
const zlib = require('zlib');
|
|
stream = encoding === 'gzip' ? response.pipe(zlib.createGunzip()) : response.pipe(zlib.createInflate());
|
|
}
|
|
|
|
const chunks = [];
|
|
stream.on('data', chunk => chunks.push(chunk));
|
|
stream.on('end', () => {
|
|
if (responseHandled || res.headersSent) return;
|
|
responseHandled = true;
|
|
const data = Buffer.concat(chunks);
|
|
// Cache successful responses
|
|
if (response.statusCode >= 200 && response.statusCode < 300) {
|
|
rssResponseCache.set(feedUrl, { data, contentType: 'application/xml', timestamp: Date.now() });
|
|
}
|
|
sendCompressed(req, res, response.statusCode, {
|
|
'Content-Type': 'application/xml',
|
|
'Cache-Control': 'public, max-age=300',
|
|
'X-Cache': 'MISS',
|
|
}, data);
|
|
});
|
|
stream.on('error', (err) => {
|
|
console.error('[Relay] Decompression error:', err.message);
|
|
sendError(502, 'Decompression failed: ' + err.message);
|
|
});
|
|
});
|
|
|
|
request.on('error', (err) => {
|
|
console.error('[Relay] RSS error:', err.message);
|
|
// Serve stale on error
|
|
if (rssCached) {
|
|
if (!responseHandled && !res.headersSent) {
|
|
responseHandled = true;
|
|
sendCompressed(req, res, 200, { 'Content-Type': 'application/xml', 'X-Cache': 'STALE' }, rssCached.data);
|
|
}
|
|
return;
|
|
}
|
|
sendError(502, err.message);
|
|
});
|
|
|
|
request.on('timeout', () => {
|
|
request.destroy();
|
|
if (rssCached && !responseHandled && !res.headersSent) {
|
|
responseHandled = true;
|
|
return sendCompressed(req, res, 200, { 'Content-Type': 'application/xml', 'X-Cache': 'STALE' }, rssCached.data);
|
|
}
|
|
sendError(504, 'Request timeout');
|
|
});
|
|
};
|
|
|
|
fetchWithRedirects(feedUrl);
|
|
} catch (err) {
|
|
if (!res.headersSent) {
|
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ error: err.message }));
|
|
}
|
|
}
|
|
} else if (req.url.startsWith('/ucdp-events')) {
|
|
handleUcdpEventsRequest(req, res);
|
|
} else if (req.url.startsWith('/opensky')) {
|
|
handleOpenSkyRequest(req, res, PORT);
|
|
} else if (req.url.startsWith('/worldbank')) {
|
|
handleWorldBankRequest(req, res);
|
|
} else if (req.url.startsWith('/polymarket')) {
|
|
handlePolymarketRequest(req, res);
|
|
} else {
|
|
res.writeHead(404);
|
|
res.end();
|
|
}
|
|
});
|
|
|
|
function connectUpstream() {
|
|
// Skip if already connected or connecting
|
|
if (upstreamSocket?.readyState === WebSocket.OPEN ||
|
|
upstreamSocket?.readyState === WebSocket.CONNECTING) return;
|
|
|
|
console.log('[Relay] Connecting to aisstream.io...');
|
|
const socket = new WebSocket(AISSTREAM_URL);
|
|
upstreamSocket = socket;
|
|
|
|
socket.on('open', () => {
|
|
// Verify this socket is still the current one (race condition guard)
|
|
if (upstreamSocket !== socket) {
|
|
console.log('[Relay] Stale socket open event, closing');
|
|
socket.close();
|
|
return;
|
|
}
|
|
console.log('[Relay] Connected to aisstream.io');
|
|
socket.send(JSON.stringify({
|
|
APIKey: API_KEY,
|
|
BoundingBoxes: [[[-90, -180], [90, 180]]],
|
|
FilterMessageTypes: ['PositionReport'],
|
|
}));
|
|
});
|
|
|
|
socket.on('message', (data) => {
|
|
if (upstreamSocket !== socket) return;
|
|
messageCount++;
|
|
if (messageCount % 1000 === 0) {
|
|
console.log(`[Relay] ${messageCount} messages, ${clients.size} clients, cache: opensky=${openskyResponseCache.size} rss=${rssResponseCache.size}`);
|
|
}
|
|
const message = data.toString();
|
|
try {
|
|
const parsed = JSON.parse(message);
|
|
if (parsed?.MessageType === 'PositionReport') {
|
|
processPositionReportForSnapshot(parsed);
|
|
}
|
|
} catch {
|
|
// Ignore malformed upstream payloads
|
|
}
|
|
|
|
// Throttled fanout: only forward every 10th message to WS clients
|
|
// The app uses HTTP snapshot polling, not WS — this is mostly for external consumers
|
|
if (clients.size > 0 && messageCount % 10 === 0) {
|
|
for (const client of clients) {
|
|
if (client.readyState === WebSocket.OPEN) {
|
|
client.send(message);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
socket.on('close', () => {
|
|
if (upstreamSocket === socket) {
|
|
upstreamSocket = null;
|
|
console.log('[Relay] Disconnected, reconnecting in 5s...');
|
|
setTimeout(connectUpstream, 5000);
|
|
}
|
|
});
|
|
|
|
socket.on('error', (err) => {
|
|
console.error('[Relay] Upstream error:', err.message);
|
|
});
|
|
}
|
|
|
|
const wss = new WebSocketServer({ server });
|
|
|
|
server.listen(PORT, () => {
|
|
console.log(`[Relay] WebSocket relay on port ${PORT}`);
|
|
});
|
|
|
|
wss.on('connection', (ws, req) => {
|
|
if (clients.size >= MAX_WS_CLIENTS) {
|
|
console.log(`[Relay] WS client rejected (max ${MAX_WS_CLIENTS})`);
|
|
ws.close(1013, 'Max clients reached');
|
|
return;
|
|
}
|
|
console.log(`[Relay] Client connected (${clients.size + 1}/${MAX_WS_CLIENTS})`);
|
|
clients.add(ws);
|
|
connectUpstream();
|
|
|
|
ws.on('close', () => {
|
|
clients.delete(ws);
|
|
});
|
|
|
|
ws.on('error', (err) => {
|
|
console.error('[Relay] Client error:', err.message);
|
|
});
|
|
});
|