mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat: add C2IntelFeeds, OTX, and AbuseIPDB as cyber threat sources
Expands from 2 to 5 threat intel sources: - C2IntelFeeds (free, no auth): ~500 active C2 server IPs (CobaltStrike, Metasploit) - AlienVault OTX (optional API key): community-sourced IOCs with tags - AbuseIPDB (optional API key): high-confidence abuse reports with geo - Feodo: now includes recently-offline entries (still threat-relevant) All sources fetched in parallel. Graceful degradation when API keys missing.
This commit is contained in:
@@ -21,6 +21,9 @@ const GEO_CACHE_TTL_MS = GEO_CACHE_TTL_SECONDS * 1000;
|
||||
|
||||
const FEODO_URL = 'https://feodotracker.abuse.ch/downloads/ipblocklist.json';
|
||||
const URLHAUS_RECENT_URL = (limit) => `https://urlhaus-api.abuse.ch/v1/urls/recent/limit/${limit}/`;
|
||||
const C2INTEL_URL = 'https://raw.githubusercontent.com/drb-ra/C2IntelFeeds/master/feeds/IPC2s-30day.csv';
|
||||
const OTX_INDICATORS_URL = 'https://otx.alienvault.com/api/v1/indicators/export?type=IPv4&modified_since=';
|
||||
const ABUSEIPDB_BLACKLIST_URL = 'https://api.abuseipdb.com/api/v2/blacklist';
|
||||
|
||||
const UPSTREAM_TIMEOUT_MS = 8000;
|
||||
const GEO_MAX_UNRESOLVED_PER_RUN = 100;
|
||||
@@ -35,7 +38,7 @@ const rateLimiter = createIpRateLimiter({
|
||||
});
|
||||
|
||||
const ALLOWED_TYPES = new Set(['c2_server', 'malware_host', 'phishing', 'malicious_url']);
|
||||
const ALLOWED_SOURCES = new Set(['feodo', 'urlhaus']);
|
||||
const ALLOWED_SOURCES = new Set(['feodo', 'urlhaus', 'c2intel', 'otx', 'abuseipdb']);
|
||||
const ALLOWED_SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
|
||||
const ALLOWED_INDICATOR_TYPES = new Set(['ip', 'domain', 'url']);
|
||||
|
||||
@@ -232,7 +235,8 @@ function parseFeodoRecord(record, cutoffMs) {
|
||||
if (!isIpAddress(ip)) return null;
|
||||
|
||||
const statusRaw = cleanString(record?.status || record?.c2_status || '', 30).toLowerCase();
|
||||
if (statusRaw !== 'online') return null;
|
||||
// Accept both online and recently-offline (still threat-relevant)
|
||||
if (statusRaw && statusRaw !== 'online' && statusRaw !== 'offline') return null;
|
||||
|
||||
const firstSeen = toIsoDate(record?.first_seen || record?.first_seen_utc || record?.dateadded);
|
||||
const lastSeen = toIsoDate(record?.last_online || record?.last_seen || record?.last_seen_utc || record?.first_seen || record?.first_seen_utc);
|
||||
@@ -255,7 +259,7 @@ function parseFeodoRecord(record, cutoffMs) {
|
||||
lat: toFiniteNumber(record?.latitude ?? record?.lat),
|
||||
lon: toFiniteNumber(record?.longitude ?? record?.lon),
|
||||
country: record?.country || record?.country_code,
|
||||
severity: inferFeodoSeverity(record, malwareFamily),
|
||||
severity: statusRaw === 'online' ? inferFeodoSeverity(record, malwareFamily) : 'medium',
|
||||
malwareFamily,
|
||||
tags: ['botnet', 'c2', ...tags],
|
||||
firstSeen,
|
||||
@@ -321,6 +325,46 @@ function parseUrlhausRecord(record, cutoffMs) {
|
||||
});
|
||||
}
|
||||
|
||||
function parseC2IntelCsvLine(line) {
|
||||
if (!line || line.startsWith('#')) return null;
|
||||
const commaIdx = line.indexOf(',');
|
||||
if (commaIdx < 0) return null;
|
||||
|
||||
const ip = cleanString(line.slice(0, commaIdx), 80).toLowerCase();
|
||||
if (!isIpAddress(ip)) return null;
|
||||
|
||||
const description = cleanString(line.slice(commaIdx + 1), 200);
|
||||
const malwareFamily = description
|
||||
.replace(/^Possible\s+/i, '')
|
||||
.replace(/\s+C2\s+IP$/i, '')
|
||||
.trim() || 'Unknown';
|
||||
|
||||
const tags = ['c2'];
|
||||
const descLower = description.toLowerCase();
|
||||
if (descLower.includes('cobaltstrike') || descLower.includes('cobalt strike')) tags.push('cobaltstrike');
|
||||
if (descLower.includes('metasploit')) tags.push('metasploit');
|
||||
if (descLower.includes('sliver')) tags.push('sliver');
|
||||
if (descLower.includes('brute ratel') || descLower.includes('bruteratel')) tags.push('bruteratel');
|
||||
|
||||
const severity = /cobaltstrike|cobalt.strike|brute.?ratel/i.test(description) ? 'high' : 'medium';
|
||||
|
||||
return sanitizeThreat({
|
||||
id: `c2intel:${ip}`,
|
||||
type: 'c2_server',
|
||||
source: 'c2intel',
|
||||
indicator: ip,
|
||||
indicatorType: 'ip',
|
||||
lat: null,
|
||||
lon: null,
|
||||
country: undefined,
|
||||
severity,
|
||||
malwareFamily,
|
||||
tags: normalizeTags(tags),
|
||||
firstSeen: undefined,
|
||||
lastSeen: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function __testParseFeodoRecords(records, options = {}) {
|
||||
const nowMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now();
|
||||
const days = clampInt(options.days, DEFAULT_DAYS, 1, MAX_DAYS);
|
||||
@@ -640,6 +684,153 @@ async function fetchUrlhausSource(limit, cutoffMs) {
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchOtxSource(limit, days) {
|
||||
const apiKey = cleanString(process.env.OTX_API_KEY || '', 200);
|
||||
if (!apiKey) {
|
||||
return { ok: false, threats: [], reason: 'missing_api_key', enabled: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const since = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
|
||||
const response = await fetchJsonWithTimeout(
|
||||
`${OTX_INDICATORS_URL}${encodeURIComponent(since)}`,
|
||||
{
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'X-OTX-API-KEY': apiKey,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
return { ok: false, threats: [], reason: `otx_http_${response.status}`, enabled: true };
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
const results = Array.isArray(payload?.results) ? payload.results : (Array.isArray(payload) ? payload : []);
|
||||
|
||||
const parsed = [];
|
||||
for (const record of results) {
|
||||
const ip = cleanString(record?.indicator || record?.ip || '', 80).toLowerCase();
|
||||
if (!isIpAddress(ip)) continue;
|
||||
|
||||
const title = cleanString(record?.title || record?.description || '', 200);
|
||||
const tags = normalizeTags(record?.tags || []);
|
||||
|
||||
const severity = tags.some((t) => /ransomware|apt|c2|botnet/.test(t)) ? 'high' : 'medium';
|
||||
|
||||
const sanitized = sanitizeThreat({
|
||||
id: `otx:${ip}`,
|
||||
type: tags.some((t) => /c2|botnet/.test(t)) ? 'c2_server' : 'malware_host',
|
||||
source: 'otx',
|
||||
indicator: ip,
|
||||
indicatorType: 'ip',
|
||||
lat: null,
|
||||
lon: null,
|
||||
country: undefined,
|
||||
severity,
|
||||
malwareFamily: title || undefined,
|
||||
tags,
|
||||
firstSeen: toIsoDate(record?.created),
|
||||
lastSeen: toIsoDate(record?.modified || record?.created),
|
||||
});
|
||||
if (sanitized) parsed.push(sanitized);
|
||||
if (parsed.length >= limit) break;
|
||||
}
|
||||
|
||||
return { ok: true, threats: parsed, enabled: true };
|
||||
} catch (error) {
|
||||
return { ok: false, threats: [], reason: `otx_error:${cleanString(toErrorMessage(error), 120)}`, enabled: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAbuseIpDbSource(limit) {
|
||||
const apiKey = cleanString(process.env.ABUSEIPDB_API_KEY || '', 200);
|
||||
if (!apiKey) {
|
||||
return { ok: false, threats: [], reason: 'missing_api_key', enabled: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const url = `${ABUSEIPDB_BLACKLIST_URL}?confidenceMinimum=90&limit=${Math.min(limit, 500)}`;
|
||||
const response = await fetchJsonWithTimeout(url, {
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Key: apiKey,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return { ok: false, threats: [], reason: `abuseipdb_http_${response.status}`, enabled: true };
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
const records = Array.isArray(payload?.data) ? payload.data : [];
|
||||
|
||||
const parsed = [];
|
||||
for (const record of records) {
|
||||
const ip = cleanString(record?.ipAddress || record?.ip || '', 80).toLowerCase();
|
||||
if (!isIpAddress(ip)) continue;
|
||||
|
||||
const score = toFiniteNumber(record?.abuseConfidenceScore) ?? 0;
|
||||
const severity = score >= 95 ? 'critical' : (score >= 80 ? 'high' : 'medium');
|
||||
|
||||
const sanitized = sanitizeThreat({
|
||||
id: `abuseipdb:${ip}`,
|
||||
type: 'malware_host',
|
||||
source: 'abuseipdb',
|
||||
indicator: ip,
|
||||
indicatorType: 'ip',
|
||||
lat: toFiniteNumber(record?.latitude ?? record?.lat),
|
||||
lon: toFiniteNumber(record?.longitude ?? record?.lon),
|
||||
country: record?.countryCode || record?.country,
|
||||
severity,
|
||||
malwareFamily: undefined,
|
||||
tags: normalizeTags([`score:${score}`]),
|
||||
firstSeen: undefined,
|
||||
lastSeen: toIsoDate(record?.lastReportedAt),
|
||||
});
|
||||
if (sanitized) parsed.push(sanitized);
|
||||
if (parsed.length >= limit) break;
|
||||
}
|
||||
|
||||
return { ok: true, threats: parsed, enabled: true };
|
||||
} catch (error) {
|
||||
return { ok: false, threats: [], reason: `abuseipdb_error:${cleanString(toErrorMessage(error), 120)}`, enabled: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchC2IntelSource(limit) {
|
||||
try {
|
||||
const response = await fetchJsonWithTimeout(C2INTEL_URL, {
|
||||
headers: { Accept: 'text/plain' },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
threats: [],
|
||||
reason: `c2intel_http_${response.status}`,
|
||||
};
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
const lines = text.split('\n');
|
||||
|
||||
const parsed = lines
|
||||
.map((line) => parseC2IntelCsvLine(line))
|
||||
.filter(Boolean)
|
||||
.slice(0, limit);
|
||||
|
||||
return { ok: true, threats: parsed };
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
threats: [],
|
||||
reason: `c2intel_error:${cleanString(toErrorMessage(error), 120)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function __resetCyberThreatsState() {
|
||||
responseMemoryCache.clear();
|
||||
staleFallbackCache.clear();
|
||||
@@ -716,18 +907,25 @@ export default async function handler(req) {
|
||||
}
|
||||
|
||||
try {
|
||||
const [feodo, urlhaus] = await Promise.all([
|
||||
const [feodo, urlhaus, c2intel, otx, abuseipdb] = await Promise.all([
|
||||
fetchFeodoSource(limit, cutoffMs),
|
||||
fetchUrlhausSource(limit, cutoffMs),
|
||||
fetchC2IntelSource(limit),
|
||||
fetchOtxSource(limit, days),
|
||||
fetchAbuseIpDbSource(limit),
|
||||
]);
|
||||
|
||||
if (!feodo.ok && !urlhaus.ok) {
|
||||
const anySuceeded = feodo.ok || urlhaus.ok || c2intel.ok || otx.ok || abuseipdb.ok;
|
||||
if (!anySuceeded) {
|
||||
throw new Error('all_sources_failed');
|
||||
}
|
||||
|
||||
const combined = __testDedupeThreats([
|
||||
...feodo.threats,
|
||||
...urlhaus.threats,
|
||||
...c2intel.threats,
|
||||
...otx.threats,
|
||||
...abuseipdb.threats,
|
||||
]);
|
||||
|
||||
const withGeo = await hydrateThreatCoordinates(combined);
|
||||
@@ -748,23 +946,26 @@ export default async function handler(req) {
|
||||
})
|
||||
.slice(0, limit);
|
||||
|
||||
const partial = !feodo.ok || (urlhaus.enabled === true && !urlhaus.ok);
|
||||
const enabledButFailed = (src) => src.enabled !== false && !src.ok;
|
||||
const partial = !feodo.ok || enabledButFailed(urlhaus) || !c2intel.ok
|
||||
|| enabledButFailed(otx) || enabledButFailed(abuseipdb);
|
||||
|
||||
const sourceStatus = (src) => ({
|
||||
ok: src.ok,
|
||||
count: src.threats.length,
|
||||
...(src.reason ? { reason: src.reason } : {}),
|
||||
});
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
count: mapData.length,
|
||||
partial,
|
||||
sources: {
|
||||
feodo: {
|
||||
ok: feodo.ok,
|
||||
count: feodo.threats.length,
|
||||
...(feodo.reason ? { reason: feodo.reason } : {}),
|
||||
},
|
||||
urlhaus: {
|
||||
ok: urlhaus.ok,
|
||||
count: urlhaus.threats.length,
|
||||
...(urlhaus.reason ? { reason: urlhaus.reason } : {}),
|
||||
},
|
||||
feodo: sourceStatus(feodo),
|
||||
urlhaus: sourceStatus(urlhaus),
|
||||
c2intel: sourceStatus(c2intel),
|
||||
otx: sourceStatus(otx),
|
||||
abuseipdb: sourceStatus(abuseipdb),
|
||||
},
|
||||
data: mapData,
|
||||
cachedAt: new Date().toISOString(),
|
||||
|
||||
@@ -8,6 +8,8 @@ import handler, {
|
||||
|
||||
const ORIGINAL_FETCH = globalThis.fetch;
|
||||
const ORIGINAL_URLHAUS_KEY = process.env.URLHAUS_AUTH_KEY;
|
||||
const ORIGINAL_OTX_KEY = process.env.OTX_API_KEY;
|
||||
const ORIGINAL_ABUSEIPDB_KEY = process.env.ABUSEIPDB_API_KEY;
|
||||
|
||||
function makeRequest(path = '/api/cyber-threats', ip = '198.51.100.10') {
|
||||
const headers = new Headers();
|
||||
@@ -22,13 +24,37 @@ function jsonResponse(body, status = 200) {
|
||||
});
|
||||
}
|
||||
|
||||
function textResponse(body, status = 200) {
|
||||
return new Response(body, {
|
||||
status,
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
// Mock that handles all 5 source URLs + geo enrichment
|
||||
function createMockFetch({ feodo, urlhaus, c2intel, otx, abuseipdb, geo } = {}) {
|
||||
return async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes('feodotracker.abuse.ch') && feodo) return feodo(target);
|
||||
if (target.includes('urlhaus-api.abuse.ch') && urlhaus) return urlhaus(target);
|
||||
if (target.includes('raw.githubusercontent.com') && target.includes('C2IntelFeeds') && c2intel) return c2intel(target);
|
||||
if (target.includes('otx.alienvault.com') && otx) return otx(target);
|
||||
if (target.includes('api.abuseipdb.com') && abuseipdb) return abuseipdb(target);
|
||||
if ((target.includes('ipwho.is') || target.includes('ipapi.co')) && geo) return geo(target);
|
||||
// Default: return 404 for unconfigured sources
|
||||
return new Response('not found', { status: 404 });
|
||||
};
|
||||
}
|
||||
|
||||
test.afterEach(() => {
|
||||
globalThis.fetch = ORIGINAL_FETCH;
|
||||
process.env.URLHAUS_AUTH_KEY = ORIGINAL_URLHAUS_KEY;
|
||||
process.env.OTX_API_KEY = ORIGINAL_OTX_KEY;
|
||||
process.env.ABUSEIPDB_API_KEY = ORIGINAL_ABUSEIPDB_KEY;
|
||||
__resetCyberThreatsState();
|
||||
});
|
||||
|
||||
test('Feodo parser filters offline/stale entries and normalizes dates', () => {
|
||||
test('Feodo parser accepts online and recent offline entries, filters stale', () => {
|
||||
const nowMs = Date.parse('2026-02-15T12:00:00.000Z');
|
||||
const records = [
|
||||
{
|
||||
@@ -43,7 +69,7 @@ test('Feodo parser filters offline/stale entries and normalizes dates', () => {
|
||||
status: 'offline',
|
||||
first_seen: '2026-02-14 10:00:00 UTC',
|
||||
last_online: '2026-02-15 10:00:00 UTC',
|
||||
malware: 'generic',
|
||||
malware: 'Emotet',
|
||||
},
|
||||
{
|
||||
ip_address: '9.9.9.9',
|
||||
@@ -61,9 +87,12 @@ test('Feodo parser filters offline/stale entries and normalizes dates', () => {
|
||||
];
|
||||
|
||||
const parsed = __testParseFeodoRecords(records, { nowMs, days: 14 });
|
||||
assert.equal(parsed.length, 1);
|
||||
// online + offline (recent) + no-status (recent) = 3; stale (9.9.9.9) filtered
|
||||
assert.equal(parsed.length, 3);
|
||||
assert.equal(parsed[0].indicator, '1.2.3.4');
|
||||
assert.equal(parsed[0].severity, 'critical');
|
||||
assert.equal(parsed[1].indicator, '5.6.7.8');
|
||||
assert.equal(parsed[1].severity, 'medium');
|
||||
assert.equal(parsed[0].firstSeen?.endsWith('Z'), true);
|
||||
assert.equal(parsed[0].lastSeen?.endsWith('Z'), true);
|
||||
});
|
||||
@@ -112,27 +141,96 @@ test('dedupes by source + indicatorType + indicator', () => {
|
||||
assert.equal(feodo?.tags.includes('b'), true);
|
||||
});
|
||||
|
||||
test('API returns success without URLhaus key and marks URLhaus as missing_auth_key', async () => {
|
||||
delete process.env.URLHAUS_AUTH_KEY;
|
||||
test('API aggregates from all 5 sources', async () => {
|
||||
process.env.URLHAUS_AUTH_KEY = 'test-key';
|
||||
process.env.OTX_API_KEY = 'test-otx';
|
||||
process.env.ABUSEIPDB_API_KEY = 'test-abuse';
|
||||
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes('feodotracker.abuse.ch')) {
|
||||
return jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]);
|
||||
}
|
||||
throw new Error(`Unexpected fetch target: ${target}`);
|
||||
};
|
||||
globalThis.fetch = createMockFetch({
|
||||
feodo: () => jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]),
|
||||
urlhaus: () => jsonResponse({
|
||||
urls: [{
|
||||
url: 'http://5.5.5.5/malware.exe',
|
||||
host: '5.5.5.5',
|
||||
url_status: 'online',
|
||||
threat: 'malware_download',
|
||||
tags: ['malware'],
|
||||
dateadded: '2026-02-14T08:00:00.000Z',
|
||||
latitude: 48.86,
|
||||
longitude: 2.35,
|
||||
country: 'FR',
|
||||
}],
|
||||
}),
|
||||
c2intel: () => textResponse(
|
||||
'#ip,ioc\n10.10.10.10,Possible Cobaltstrike C2 IP\n10.10.10.11,Possible Metasploit C2 IP',
|
||||
),
|
||||
otx: () => jsonResponse({
|
||||
results: [{
|
||||
indicator: '20.20.20.20',
|
||||
title: 'APT threat',
|
||||
tags: ['apt', 'c2'],
|
||||
created: '2026-02-13T00:00:00.000Z',
|
||||
modified: '2026-02-14T00:00:00.000Z',
|
||||
}],
|
||||
}),
|
||||
abuseipdb: () => jsonResponse({
|
||||
data: [{
|
||||
ipAddress: '30.30.30.30',
|
||||
abuseConfidenceScore: 98,
|
||||
lastReportedAt: '2026-02-15T06:00:00.000Z',
|
||||
countryCode: 'CN',
|
||||
latitude: 39.9,
|
||||
longitude: 116.4,
|
||||
}],
|
||||
}),
|
||||
geo: () => jsonResponse({ success: true, latitude: 40.0, longitude: -74.0, country_code: 'US' }),
|
||||
});
|
||||
|
||||
const response = await handler(makeRequest('/api/cyber-threats?limit=100&days=14', '198.51.100.20'));
|
||||
assert.equal(response.status, 200);
|
||||
|
||||
const body = await response.json();
|
||||
assert.equal(body.success, true);
|
||||
assert.equal(body.sources.feodo.ok, true);
|
||||
assert.equal(body.sources.urlhaus.ok, true);
|
||||
assert.equal(body.sources.c2intel.ok, true);
|
||||
assert.equal(body.sources.otx.ok, true);
|
||||
assert.equal(body.sources.abuseipdb.ok, true);
|
||||
// 5 sources, all with coords (3 native + 3 via geo enrichment mock)
|
||||
assert.equal(body.data.length >= 5, true);
|
||||
});
|
||||
|
||||
test('API works with only free sources when keys missing', async () => {
|
||||
delete process.env.URLHAUS_AUTH_KEY;
|
||||
delete process.env.OTX_API_KEY;
|
||||
delete process.env.ABUSEIPDB_API_KEY;
|
||||
|
||||
globalThis.fetch = createMockFetch({
|
||||
feodo: () => jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]),
|
||||
c2intel: () => textResponse('#ip,ioc\n10.10.10.10,Possible Cobaltstrike C2 IP'),
|
||||
});
|
||||
|
||||
const response = await handler(makeRequest('/api/cyber-threats?limit=100&days=14', '198.51.100.11'));
|
||||
assert.equal(response.status, 200);
|
||||
@@ -142,38 +240,37 @@ test('API returns success without URLhaus key and marks URLhaus as missing_auth_
|
||||
assert.equal(body.success, true);
|
||||
assert.equal(body.partial, false);
|
||||
assert.equal(body.sources.feodo.ok, true);
|
||||
assert.equal(body.sources.c2intel.ok, true);
|
||||
assert.equal(body.sources.urlhaus.ok, false);
|
||||
assert.equal(body.sources.urlhaus.reason, 'missing_auth_key');
|
||||
assert.equal(body.sources.otx.ok, false);
|
||||
assert.equal(body.sources.otx.reason, 'missing_api_key');
|
||||
assert.equal(body.sources.abuseipdb.ok, false);
|
||||
assert.equal(body.sources.abuseipdb.reason, 'missing_api_key');
|
||||
assert.equal(Array.isArray(body.data), true);
|
||||
assert.equal(body.data.length, 1);
|
||||
});
|
||||
|
||||
test('API marks partial=true when URLhaus is enabled but fails', async () => {
|
||||
process.env.URLHAUS_AUTH_KEY = 'test-key';
|
||||
delete process.env.OTX_API_KEY;
|
||||
delete process.env.ABUSEIPDB_API_KEY;
|
||||
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes('feodotracker.abuse.ch')) {
|
||||
return jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (target.includes('urlhaus-api.abuse.ch')) {
|
||||
return new Response('boom', { status: 500 });
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected fetch target: ${target}`);
|
||||
};
|
||||
globalThis.fetch = createMockFetch({
|
||||
feodo: () => jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]),
|
||||
urlhaus: () => new Response('boom', { status: 500 }),
|
||||
c2intel: () => textResponse('#ip,ioc\n10.10.10.10,Possible Cobaltstrike C2 IP'),
|
||||
});
|
||||
|
||||
const response = await handler(makeRequest('/api/cyber-threats?limit=100&days=14', '198.51.100.12'));
|
||||
assert.equal(response.status, 200);
|
||||
@@ -187,11 +284,12 @@ test('API marks partial=true when URLhaus is enabled but fails', async () => {
|
||||
|
||||
test('API returns memory cache hit on repeated request', async () => {
|
||||
delete process.env.URLHAUS_AUTH_KEY;
|
||||
delete process.env.OTX_API_KEY;
|
||||
delete process.env.ABUSEIPDB_API_KEY;
|
||||
|
||||
let feodoCalls = 0;
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes('feodotracker.abuse.ch')) {
|
||||
globalThis.fetch = createMockFetch({
|
||||
feodo: () => {
|
||||
feodoCalls += 1;
|
||||
return jsonResponse([
|
||||
{
|
||||
@@ -205,9 +303,9 @@ test('API returns memory cache hit on repeated request', async () => {
|
||||
lon: -0.12,
|
||||
},
|
||||
]);
|
||||
}
|
||||
throw new Error(`Unexpected fetch target: ${target}`);
|
||||
};
|
||||
},
|
||||
c2intel: () => textResponse('#ip,ioc\n10.10.10.10,Possible Cobaltstrike C2 IP'),
|
||||
});
|
||||
|
||||
const first = await handler(makeRequest('/api/cyber-threats?limit=100&days=14', '198.51.100.13'));
|
||||
assert.equal(first.status, 200);
|
||||
@@ -226,36 +324,35 @@ test('API returns memory cache hit on repeated request', async () => {
|
||||
|
||||
test('API returns stale fallback when upstream fails after fresh cache TTL', async () => {
|
||||
delete process.env.URLHAUS_AUTH_KEY;
|
||||
delete process.env.OTX_API_KEY;
|
||||
delete process.env.ABUSEIPDB_API_KEY;
|
||||
|
||||
const baseNow = Date.parse('2026-02-15T12:00:00.000Z');
|
||||
const originalDateNow = Date.now;
|
||||
Date.now = () => baseNow;
|
||||
|
||||
try {
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes('feodotracker.abuse.ch')) {
|
||||
return jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]);
|
||||
}
|
||||
throw new Error(`Unexpected fetch target: ${target}`);
|
||||
};
|
||||
globalThis.fetch = createMockFetch({
|
||||
feodo: () => jsonResponse([
|
||||
{
|
||||
ip_address: '1.2.3.4',
|
||||
status: 'online',
|
||||
last_online: '2026-02-15T10:00:00.000Z',
|
||||
first_seen: '2026-02-14T10:00:00.000Z',
|
||||
malware: 'QakBot',
|
||||
country: 'GB',
|
||||
lat: 51.5,
|
||||
lon: -0.12,
|
||||
},
|
||||
]),
|
||||
c2intel: () => textResponse('#ip,ioc\n10.10.10.10,Possible Cobaltstrike C2 IP'),
|
||||
});
|
||||
|
||||
const first = await handler(makeRequest('/api/cyber-threats?limit=100&days=14', '198.51.100.14'));
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.headers.get('X-Cache'), 'MISS');
|
||||
|
||||
Date.now = () => baseNow + (11 * 60 * 1000); // exceed 10m fresh TTL, still within stale horizon
|
||||
Date.now = () => baseNow + (11 * 60 * 1000);
|
||||
globalThis.fetch = async () => {
|
||||
throw new Error('forced upstream failure');
|
||||
};
|
||||
@@ -267,7 +364,7 @@ test('API returns stale fallback when upstream fails after fresh cache TTL', asy
|
||||
const body = await stale.json();
|
||||
assert.equal(body.success, true);
|
||||
assert.equal(Array.isArray(body.data), true);
|
||||
assert.equal(body.data.length, 1);
|
||||
assert.equal(body.data.length >= 1, true);
|
||||
} finally {
|
||||
Date.now = originalDateNow;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,9 @@ export interface CyberThreatsMeta {
|
||||
sources: {
|
||||
feodo: CyberThreatSourceStatus;
|
||||
urlhaus: CyberThreatSourceStatus;
|
||||
c2intel: CyberThreatSourceStatus;
|
||||
otx: CyberThreatSourceStatus;
|
||||
abuseipdb: CyberThreatSourceStatus;
|
||||
};
|
||||
cachedAt?: string;
|
||||
}
|
||||
@@ -29,6 +32,9 @@ let lastMeta: CyberThreatsMeta = {
|
||||
sources: {
|
||||
feodo: { ok: false, count: 0 },
|
||||
urlhaus: { ok: false, count: 0 },
|
||||
c2intel: { ok: false, count: 0 },
|
||||
otx: { ok: false, count: 0 },
|
||||
abuseipdb: { ok: false, count: 0 },
|
||||
},
|
||||
};
|
||||
|
||||
@@ -62,7 +68,11 @@ function asIndicatorType(value: unknown): CyberThreatIndicatorType {
|
||||
|
||||
function asSource(value: unknown): CyberThreatSource {
|
||||
const normalized = String(value ?? '').toLowerCase();
|
||||
return normalized === 'urlhaus' ? 'urlhaus' : 'feodo';
|
||||
if (normalized === 'urlhaus') return 'urlhaus';
|
||||
if (normalized === 'c2intel') return 'c2intel';
|
||||
if (normalized === 'otx') return 'otx';
|
||||
if (normalized === 'abuseipdb') return 'abuseipdb';
|
||||
return 'feodo';
|
||||
}
|
||||
|
||||
function hasValidCoordinates(lat: number, lon: number): boolean {
|
||||
@@ -139,7 +149,7 @@ export async function fetchCyberThreats(options: { limit?: number; days?: number
|
||||
success?: boolean;
|
||||
partial?: boolean;
|
||||
data?: unknown[];
|
||||
sources?: { feodo?: unknown; urlhaus?: unknown };
|
||||
sources?: { feodo?: unknown; urlhaus?: unknown; c2intel?: unknown; otx?: unknown; abuseipdb?: unknown };
|
||||
cachedAt?: string;
|
||||
};
|
||||
|
||||
@@ -156,6 +166,9 @@ export async function fetchCyberThreats(options: { limit?: number; days?: number
|
||||
sources: {
|
||||
feodo: sanitizeSourceStatus(payload.sources?.feodo),
|
||||
urlhaus: sanitizeSourceStatus(payload.sources?.urlhaus),
|
||||
c2intel: sanitizeSourceStatus(payload.sources?.c2intel),
|
||||
otx: sanitizeSourceStatus(payload.sources?.otx),
|
||||
abuseipdb: sanitizeSourceStatus(payload.sources?.abuseipdb),
|
||||
},
|
||||
cachedAt: payload.cachedAt,
|
||||
};
|
||||
@@ -170,6 +183,9 @@ export function getCyberThreatsMeta(): CyberThreatsMeta {
|
||||
sources: {
|
||||
feodo: { ...lastMeta.sources.feodo },
|
||||
urlhaus: { ...lastMeta.sources.urlhaus },
|
||||
c2intel: { ...lastMeta.sources.c2intel },
|
||||
otx: { ...lastMeta.sources.otx },
|
||||
abuseipdb: { ...lastMeta.sources.abuseipdb },
|
||||
},
|
||||
cachedAt: lastMeta.cachedAt,
|
||||
};
|
||||
|
||||
@@ -197,7 +197,7 @@ export interface APTGroup {
|
||||
}
|
||||
|
||||
export type CyberThreatType = 'c2_server' | 'malware_host' | 'phishing' | 'malicious_url';
|
||||
export type CyberThreatSource = 'feodo' | 'urlhaus';
|
||||
export type CyberThreatSource = 'feodo' | 'urlhaus' | 'c2intel' | 'otx' | 'abuseipdb';
|
||||
export type CyberThreatSeverity = 'low' | 'medium' | 'high' | 'critical';
|
||||
export type CyberThreatIndicatorType = 'ip' | 'domain' | 'url';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user