mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
1d417af59f
Mexico had only 1 dedicated source (El Universal) despite significant cartel/security events. Add 8 feeds: Mexico News Daily, Animal Político, Proceso, Milenio, Mexico Security (Google News), AP Mexico, InSight Crime (organized crime tracker), and France 24 LatAm.
317 lines
8.3 KiB
JavaScript
317 lines
8.3 KiB
JavaScript
// Non-sebuf: returns XML/HTML, stays as standalone Vercel function
|
|
import { getCorsHeaders, isDisallowedOrigin } from './_cors.js';
|
|
|
|
export const config = { runtime: 'edge' };
|
|
|
|
// Fetch with timeout
|
|
async function fetchWithTimeout(url, options, timeoutMs = 15000) {
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(url, { ...options, signal: controller.signal });
|
|
return response;
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
// Allowed RSS feed domains for security
|
|
const ALLOWED_DOMAINS = [
|
|
'feeds.bbci.co.uk',
|
|
'www.theguardian.com',
|
|
'feeds.npr.org',
|
|
'news.google.com',
|
|
'www.aljazeera.com',
|
|
'rss.cnn.com',
|
|
'hnrss.org',
|
|
'feeds.arstechnica.com',
|
|
'www.theverge.com',
|
|
'www.cnbc.com',
|
|
'feeds.marketwatch.com',
|
|
'www.defenseone.com',
|
|
'breakingdefense.com',
|
|
'www.bellingcat.com',
|
|
'techcrunch.com',
|
|
'huggingface.co',
|
|
'www.technologyreview.com',
|
|
'rss.arxiv.org',
|
|
'export.arxiv.org',
|
|
'www.federalreserve.gov',
|
|
'www.sec.gov',
|
|
'www.whitehouse.gov',
|
|
'www.state.gov',
|
|
'www.defense.gov',
|
|
'home.treasury.gov',
|
|
'www.justice.gov',
|
|
'tools.cdc.gov',
|
|
'www.fema.gov',
|
|
'www.dhs.gov',
|
|
'www.thedrive.com',
|
|
'krebsonsecurity.com',
|
|
'finance.yahoo.com',
|
|
'thediplomat.com',
|
|
'venturebeat.com',
|
|
'foreignpolicy.com',
|
|
'www.ft.com',
|
|
'openai.com',
|
|
'www.reutersagency.com',
|
|
'feeds.reuters.com',
|
|
'rsshub.app',
|
|
'asia.nikkei.com',
|
|
'www.cfr.org',
|
|
'www.csis.org',
|
|
'www.politico.com',
|
|
'www.brookings.edu',
|
|
'layoffs.fyi',
|
|
'www.defensenews.com',
|
|
'www.militarytimes.com',
|
|
'taskandpurpose.com',
|
|
'news.usni.org',
|
|
'www.oryxspioenkop.com',
|
|
'www.gov.uk',
|
|
'www.foreignaffairs.com',
|
|
'www.atlanticcouncil.org',
|
|
// Tech variant domains
|
|
'www.zdnet.com',
|
|
'www.techmeme.com',
|
|
'www.darkreading.com',
|
|
'www.schneier.com',
|
|
'rss.politico.com',
|
|
'www.anandtech.com',
|
|
'www.tomshardware.com',
|
|
'www.semianalysis.com',
|
|
'feed.infoq.com',
|
|
'thenewstack.io',
|
|
'devops.com',
|
|
'dev.to',
|
|
'lobste.rs',
|
|
'changelog.com',
|
|
'seekingalpha.com',
|
|
'news.crunchbase.com',
|
|
'www.saastr.com',
|
|
'feeds.feedburner.com',
|
|
// Additional tech variant domains
|
|
'www.producthunt.com',
|
|
'www.axios.com',
|
|
'github.blog',
|
|
'githubnext.com',
|
|
'mshibanami.github.io',
|
|
'www.engadget.com',
|
|
'news.mit.edu',
|
|
'dev.events',
|
|
// VC blogs
|
|
'www.ycombinator.com',
|
|
'a16z.com',
|
|
'review.firstround.com',
|
|
'www.sequoiacap.com',
|
|
'www.nfx.com',
|
|
'www.aaronsw.com',
|
|
'bothsidesofthetable.com',
|
|
'www.lennysnewsletter.com',
|
|
'stratechery.com',
|
|
// Regional startup news
|
|
'www.eu-startups.com',
|
|
'tech.eu',
|
|
'sifted.eu',
|
|
'www.techinasia.com',
|
|
'kr-asia.com',
|
|
'techcabal.com',
|
|
'disrupt-africa.com',
|
|
'lavca.org',
|
|
'contxto.com',
|
|
'inc42.com',
|
|
'yourstory.com',
|
|
// Funding & VC
|
|
'pitchbook.com',
|
|
'www.cbinsights.com',
|
|
// Accelerators
|
|
'www.techstars.com',
|
|
// Middle East & Regional News
|
|
'english.alarabiya.net',
|
|
'www.arabnews.com',
|
|
'www.timesofisrael.com',
|
|
'www.haaretz.com',
|
|
'www.scmp.com',
|
|
'kyivindependent.com',
|
|
'www.themoscowtimes.com',
|
|
'feeds.24.com',
|
|
'feeds.capi24.com', // News24 redirect destination
|
|
// International News Sources
|
|
'www.france24.com',
|
|
'www.euronews.com',
|
|
'www.lemonde.fr',
|
|
'rss.dw.com',
|
|
'www.bild.de',
|
|
'www.africanews.com',
|
|
// Nigeria
|
|
'www.premiumtimesng.com',
|
|
'www.vanguardngr.com',
|
|
'www.channelstv.com',
|
|
'dailytrust.com',
|
|
'www.thisdaylive.com',
|
|
// Greek
|
|
'www.naftemporiki.gr',
|
|
'www.in.gr',
|
|
'www.iefimerida.gr',
|
|
'www.lasillavacia.com',
|
|
'www.channelnewsasia.com',
|
|
'www.thehindu.com',
|
|
// International Organizations
|
|
'news.un.org',
|
|
'www.iaea.org',
|
|
'www.who.int',
|
|
'www.cisa.gov',
|
|
'www.crisisgroup.org',
|
|
// Think Tanks & Research (Added 2026-01-29)
|
|
'rusi.org',
|
|
'warontherocks.com',
|
|
'www.aei.org',
|
|
'responsiblestatecraft.org',
|
|
'www.fpri.org',
|
|
'jamestown.org',
|
|
'www.chathamhouse.org',
|
|
'ecfr.eu',
|
|
'www.gmfus.org',
|
|
'www.wilsoncenter.org',
|
|
'www.lowyinstitute.org',
|
|
'www.mei.edu',
|
|
'www.stimson.org',
|
|
'www.cnas.org',
|
|
'carnegieendowment.org',
|
|
'www.rand.org',
|
|
'fas.org',
|
|
'www.armscontrol.org',
|
|
'www.nti.org',
|
|
'thebulletin.org',
|
|
'www.iss.europa.eu',
|
|
// Economic & Food Security
|
|
'www.fao.org',
|
|
'worldbank.org',
|
|
'www.imf.org',
|
|
// Regional locale feeds (tr, pl, ru, th, vi, pt)
|
|
'www.hurriyet.com.tr',
|
|
'tvn24.pl',
|
|
'www.polsatnews.pl',
|
|
'www.rp.pl',
|
|
'meduza.io',
|
|
'novayagazeta.eu',
|
|
'www.bangkokpost.com',
|
|
'vnexpress.net',
|
|
'www.abc.net.au',
|
|
'islandtimes.org',
|
|
'www.brasilparalelo.com.br',
|
|
// Mexico & LatAm Security
|
|
'mexiconewsdaily.com',
|
|
'animalpolitico.com',
|
|
'www.proceso.com.mx',
|
|
'www.milenio.com',
|
|
'insightcrime.org',
|
|
// Additional
|
|
'news.ycombinator.com',
|
|
// Finance variant
|
|
'seekingalpha.com',
|
|
'www.coindesk.com',
|
|
'cointelegraph.com',
|
|
];
|
|
|
|
export default async function handler(req) {
|
|
const corsHeaders = getCorsHeaders(req, 'GET, OPTIONS');
|
|
|
|
// Handle CORS preflight
|
|
if (req.method === 'OPTIONS') {
|
|
return new Response(null, { status: 204, headers: corsHeaders });
|
|
}
|
|
|
|
const requestUrl = new URL(req.url);
|
|
const feedUrl = requestUrl.searchParams.get('url');
|
|
|
|
if (!feedUrl) {
|
|
return new Response(JSON.stringify({ error: 'Missing url parameter' }), {
|
|
status: 400,
|
|
headers: { 'Content-Type': 'application/json', ...corsHeaders },
|
|
});
|
|
}
|
|
|
|
try {
|
|
const parsedUrl = new URL(feedUrl);
|
|
|
|
// Security: Check if domain is allowed
|
|
if (!ALLOWED_DOMAINS.includes(parsedUrl.hostname)) {
|
|
return new Response(JSON.stringify({ error: 'Domain not allowed' }), {
|
|
status: 403,
|
|
headers: { 'Content-Type': 'application/json', ...corsHeaders },
|
|
});
|
|
}
|
|
|
|
// Google News is slow - use longer timeout
|
|
const isGoogleNews = feedUrl.includes('news.google.com');
|
|
const timeout = isGoogleNews ? 20000 : 12000;
|
|
|
|
const response = await fetchWithTimeout(feedUrl, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'application/rss+xml, application/xml, text/xml, */*',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
},
|
|
redirect: 'manual',
|
|
}, timeout);
|
|
|
|
if (response.status >= 300 && response.status < 400) {
|
|
const location = response.headers.get('location');
|
|
if (location) {
|
|
try {
|
|
const redirectUrl = new URL(location, feedUrl);
|
|
if (!ALLOWED_DOMAINS.includes(redirectUrl.hostname)) {
|
|
return new Response(JSON.stringify({ error: 'Redirect to disallowed domain' }), {
|
|
status: 403,
|
|
headers: { 'Content-Type': 'application/json', ...corsHeaders },
|
|
});
|
|
}
|
|
const redirectResponse = await fetchWithTimeout(redirectUrl.href, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'application/rss+xml, application/xml, text/xml, */*',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
},
|
|
}, timeout);
|
|
const data = await redirectResponse.text();
|
|
return new Response(data, {
|
|
status: redirectResponse.status,
|
|
headers: {
|
|
'Content-Type': 'application/xml',
|
|
'Cache-Control': 'public, max-age=300, s-maxage=600, stale-while-revalidate=300',
|
|
...corsHeaders,
|
|
},
|
|
});
|
|
} catch {
|
|
return new Response(JSON.stringify({ error: 'Invalid redirect' }), {
|
|
status: 502,
|
|
headers: { 'Content-Type': 'application/json', ...corsHeaders },
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const data = await response.text();
|
|
return new Response(data, {
|
|
status: response.status,
|
|
headers: {
|
|
'Content-Type': 'application/xml',
|
|
'Cache-Control': 'public, max-age=600, s-maxage=600, stale-while-revalidate=300',
|
|
...corsHeaders,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
const isTimeout = error.name === 'AbortError';
|
|
console.error('RSS proxy error:', feedUrl, error.message);
|
|
return new Response(JSON.stringify({
|
|
error: isTimeout ? 'Feed timeout' : 'Failed to fetch feed',
|
|
details: error.message,
|
|
url: feedUrl
|
|
}), {
|
|
status: isTimeout ? 504 : 502,
|
|
headers: { 'Content-Type': 'application/json', ...corsHeaders },
|
|
});
|
|
}
|
|
}
|