diff --git a/.env.example b/.env.example index 21dec4fbf..102ef33fd 100644 --- a/.env.example +++ b/.env.example @@ -147,10 +147,6 @@ VITE_WS_API_URL= # Client-side Sentry DSN (optional). Leave empty to disable error reporting. VITE_SENTRY_DSN= -# PostHog product analytics (optional). Leave empty to disable analytics. -VITE_POSTHOG_KEY= -VITE_POSTHOG_HOST= - # Map interaction mode: # - "flat" keeps pitch/rotation disabled (2D interaction) # - "3d" enables pitch/rotation interactions (default) diff --git a/.gitignore b/.gitignore index 16ae40166..964f2f9a0 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ scripts/data/osm-military-processed.json scripts/data/military-bases-final.json scripts/data/dedup-dropped-pairs.json scripts/data/gpsjam-latest.json +scripts/data/mirta-raw.geojson +scripts/data/osm-military-raw.json diff --git a/README.md b/README.md index d1827a7c0..09a3c1955 100644 --- a/README.md +++ b/README.md @@ -850,28 +850,6 @@ With Ollama or LM Studio configured, AI summarization runs entirely on local har The desktop readiness framework (`desktop-readiness.ts`) catalogs each feature's locality class — `fully-local` (no API required), `api-key` (degrades gracefully without keys), or `cloud-fallback` (proxy available) — enabling clear communication about what works offline. -### Product Analytics - -World Monitor includes privacy-first product analytics via PostHog to understand usage patterns and improve the dashboard. The implementation enforces strict data safety at multiple levels: - -**Typed event allowlists** — every analytics event has a schema defining exactly which properties are permitted. Unlisted properties are silently dropped before transmission. This prevents accidental inclusion of sensitive data in analytics payloads, even if a developer passes extra fields. - -**API key stripping** — a `sanitize_properties` callback runs on every outgoing event. Any string value matching common API key prefixes (`sk-`, `gsk_`, `or-`, `Bearer `) is replaced with `[REDACTED]` before it leaves the browser. This is defense-in-depth: even if a key somehow ends up in an event payload, it never reaches the analytics backend. - -**No session recordings, no autocapture** — PostHog's session replay and automatic DOM event capture are explicitly disabled. Only explicitly instrumented events are tracked. - -**Pseudonymous identity** — each installation generates a random UUID stored in localStorage. There is no user login, no email collection, and no cross-device tracking. The UUID is purely pseudonymous — it enables session attribution without identifying individuals. - -**Ad-blocker bypass** — on the web, PostHog traffic is routed through a reverse proxy on the app's own domain (`/ingest`) rather than directly to PostHog's servers. This prevents ad blockers from silently dropping analytics requests, ensuring usage data is representative. Desktop builds use PostHog's direct endpoint since ad blockers aren't a factor in native apps. - -**Offline event queue** — the desktop app may launch without network connectivity. Events captured while offline are queued in localStorage (capped at 200 entries) and flushed to PostHog when connectivity is restored. A `window.online` listener triggers automatic flush on reconnection. - -**Super properties** — every event automatically carries platform context: variant (world/tech/finance), app version, platform (web/desktop), screen dimensions, viewport size, device pixel ratio, browser language, and desktop OS/arch. This enables segmentation without per-event instrumentation. - -30+ typed events cover core user interactions: app load timing, panel views, LLM summary generation (provider, model, cache status), API key configuration snapshots, map layer toggles, variant switches, country brief opens, theme changes, language changes, search usage, panel resizing, webcam selections, and auto-update interactions. - -Analytics is entirely opt-out by omitting the `VITE_POSTHOG_KEY` environment variable. When the key is absent, all analytics functions are no-ops with zero runtime overhead. - ### Responsive Layout System The dashboard adapts to four screen categories without JavaScript layout computation — all breakpoints are CSS-only: diff --git a/api/[domain]/v1/[rpc].ts b/api/[domain]/v1/[rpc].ts index e40f42d80..610086377 100644 --- a/api/[domain]/v1/[rpc].ts +++ b/api/[domain]/v1/[rpc].ts @@ -132,7 +132,7 @@ const RPC_CACHE_TIER: Record = { '/api/economic/v1/get-macro-signals': 'medium', '/api/prediction/v1/list-prediction-markets': 'medium', '/api/supply-chain/v1/get-chokepoint-status': 'medium', - '/api/news/v1/list-feed-digest': 'medium', + '/api/news/v1/list-feed-digest': 'slow', }; const serverOptions: ServerOptions = { onError: mapErrorToResponse }; diff --git a/api/_rate-limit.js b/api/_rate-limit.js new file mode 100644 index 000000000..bf2b4be15 --- /dev/null +++ b/api/_rate-limit.js @@ -0,0 +1,58 @@ +import { Ratelimit } from '@upstash/ratelimit'; +import { Redis } from '@upstash/redis'; + +let ratelimit = null; + +function getRatelimit() { + if (ratelimit) return ratelimit; + + const url = process.env.UPSTASH_REDIS_REST_URL; + const token = process.env.UPSTASH_REDIS_REST_TOKEN; + if (!url || !token) return null; + + ratelimit = new Ratelimit({ + redis: new Redis({ url, token }), + limiter: Ratelimit.slidingWindow(300, '60 s'), + prefix: 'rl', + analytics: false, + }); + + return ratelimit; +} + +function getClientIp(request) { + return ( + request.headers.get('x-real-ip') || + request.headers.get('cf-connecting-ip') || + request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() || + '0.0.0.0' + ); +} + +export async function checkRateLimit(request, corsHeaders) { + const rl = getRatelimit(); + if (!rl) return null; + + const ip = getClientIp(request); + try { + const { success, limit, reset } = await rl.limit(ip); + + if (!success) { + return new Response(JSON.stringify({ error: 'Too many requests' }), { + status: 429, + headers: { + 'Content-Type': 'application/json', + 'X-RateLimit-Limit': String(limit), + 'X-RateLimit-Remaining': '0', + 'X-RateLimit-Reset': String(reset), + 'Retry-After': String(Math.ceil((reset - Date.now()) / 1000)), + ...corsHeaders, + }, + }); + } + + return null; + } catch { + return null; + } +} diff --git a/api/ais-snapshot.js b/api/ais-snapshot.js index cf035be66..2b4e3019a 100644 --- a/api/ais-snapshot.js +++ b/api/ais-snapshot.js @@ -1,4 +1,6 @@ import { getCorsHeaders, isDisallowedOrigin } from './_cors.js'; +import { validateApiKey } from './_api-key.js'; +import { checkRateLimit } from './_rate-limit.js'; export const config = { runtime: 'edge' }; @@ -49,6 +51,17 @@ export default async function handler(req) { }); } + const keyCheck = validateApiKey(req); + if (keyCheck.required && !keyCheck.valid) { + return new Response(JSON.stringify({ error: keyCheck.error }), { + status: 401, + headers: { 'Content-Type': 'application/json', ...corsHeaders }, + }); + } + + const rateLimitResponse = await checkRateLimit(req, corsHeaders); + if (rateLimitResponse) return rateLimitResponse; + const relayBaseUrl = getRelayBaseUrl(); if (!relayBaseUrl) { return new Response(JSON.stringify({ error: 'WS_RELAY_URL is not configured' }), { @@ -65,9 +78,13 @@ export default async function handler(req) { }, 12000); const body = await response.text(); + const isSuccess = response.status >= 200 && response.status < 300; const headers = { 'Content-Type': response.headers.get('content-type') || 'application/json', - 'Cache-Control': response.headers.get('cache-control') || 'no-cache', + 'Cache-Control': isSuccess + ? 'public, max-age=60, s-maxage=180, stale-while-revalidate=300, stale-if-error=900' + : 'public, max-age=10, s-maxage=30, stale-while-revalidate=120', + ...(isSuccess && { 'CDN-Cache-Control': 'public, s-maxage=180, stale-while-revalidate=300, stale-if-error=900' }), ...corsHeaders, }; diff --git a/api/polymarket.js b/api/polymarket.js index 0d7e0967a..ab9ba12f1 100644 --- a/api/polymarket.js +++ b/api/polymarket.js @@ -1,4 +1,6 @@ import { getCorsHeaders, isDisallowedOrigin } from './_cors.js'; +import { validateApiKey } from './_api-key.js'; +import { checkRateLimit } from './_rate-limit.js'; export const config = { runtime: 'edge' }; @@ -49,6 +51,17 @@ export default async function handler(req) { }); } + const keyCheck = validateApiKey(req); + if (keyCheck.required && !keyCheck.valid) { + return new Response(JSON.stringify({ error: keyCheck.error }), { + status: 401, + headers: { 'Content-Type': 'application/json', ...corsHeaders }, + }); + } + + const rateLimitResponse = await checkRateLimit(req, corsHeaders); + if (rateLimitResponse) return rateLimitResponse; + const relayBaseUrl = getRelayBaseUrl(); if (!relayBaseUrl) { return new Response(JSON.stringify({ error: 'WS_RELAY_URL is not configured' }), { @@ -65,9 +78,13 @@ export default async function handler(req) { }, 15000); const body = await response.text(); + const isSuccess = response.status >= 200 && response.status < 300; const headers = { 'Content-Type': response.headers.get('content-type') || 'application/json', - 'Cache-Control': 'public, s-maxage=120, stale-while-revalidate=60', + 'Cache-Control': isSuccess + ? 'public, max-age=120, s-maxage=300, stale-while-revalidate=900, stale-if-error=1800' + : 'public, max-age=10, s-maxage=30, stale-while-revalidate=120', + ...(isSuccess && { 'CDN-Cache-Control': 'public, s-maxage=300, stale-while-revalidate=900, stale-if-error=1800' }), ...corsHeaders, }; diff --git a/api/rss-proxy.js b/api/rss-proxy.js index 4c47d96d9..a4b40837a 100644 --- a/api/rss-proxy.js +++ b/api/rss-proxy.js @@ -1,5 +1,7 @@ // Non-sebuf: returns XML/HTML, stays as standalone Vercel function import { getCorsHeaders, isDisallowedOrigin } from './_cors.js'; +import { validateApiKey } from './_api-key.js'; +import { checkRateLimit } from './_rate-limit.js'; export const config = { runtime: 'edge' }; @@ -328,10 +330,34 @@ const ALLOWED_DOMAINS = [ export default async function handler(req) { const corsHeaders = getCorsHeaders(req, 'GET, OPTIONS'); + if (isDisallowedOrigin(req)) { + return new Response(JSON.stringify({ error: 'Origin not allowed' }), { + status: 403, + headers: { 'Content-Type': 'application/json', ...corsHeaders }, + }); + } + // Handle CORS preflight if (req.method === 'OPTIONS') { return new Response(null, { status: 204, headers: corsHeaders }); } + if (req.method !== 'GET') { + return new Response(JSON.stringify({ error: 'Method not allowed' }), { + status: 405, + headers: { 'Content-Type': 'application/json', ...corsHeaders }, + }); + } + + const keyCheck = validateApiKey(req); + if (keyCheck.required && !keyCheck.valid) { + return new Response(JSON.stringify({ error: keyCheck.error }), { + status: 401, + headers: { 'Content-Type': 'application/json', ...corsHeaders }, + }); + } + + const rateLimitResponse = await checkRateLimit(req, corsHeaders); + if (rateLimitResponse) return rateLimitResponse; const requestUrl = new URL(req.url); const feedUrl = requestUrl.searchParams.get('url'); @@ -415,9 +441,9 @@ export default async function handler(req) { headers: { 'Content-Type': response.headers.get('content-type') || 'application/xml', 'Cache-Control': isSuccess - ? 'public, max-age=120, s-maxage=300, stale-while-revalidate=600' - : 'public, max-age=10, s-maxage=30, stale-while-revalidate=60', - ...(isSuccess && { 'CDN-Cache-Control': 'public, s-maxage=300, stale-while-revalidate=600' }), + ? 'public, max-age=180, s-maxage=900, stale-while-revalidate=1800, stale-if-error=3600' + : 'public, max-age=15, s-maxage=60, stale-while-revalidate=120', + ...(isSuccess && { 'CDN-Cache-Control': 'public, s-maxage=900, stale-while-revalidate=1800, stale-if-error=3600' }), ...corsHeaders, }, }); diff --git a/index.html b/index.html index 29f6413d0..acb3eb823 100644 --- a/index.html +++ b/index.html @@ -3,7 +3,7 @@ - + diff --git a/intelhq b/intelhq new file mode 160000 index 000000000..4034b55d6 --- /dev/null +++ b/intelhq @@ -0,0 +1 @@ +Subproject commit 4034b55d688cd2e662693770d92e96affd98c009 diff --git a/middleware.ts b/middleware.ts index 3ff933643..5707dda7c 100644 --- a/middleware.ts +++ b/middleware.ts @@ -22,10 +22,6 @@ const SOCIAL_IMAGE_UA = export default function middleware(request: Request) { const url = new URL(request.url); - if (url.hostname === 'api.worldmonitor.app') { - return; - } - const ua = request.headers.get('user-agent') ?? ''; const path = url.pathname; diff --git a/package-lock.json b/package-lock.json index 962b4f0d2..e095c9e42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,6 @@ "maplibre-gl": "^5.16.0", "onnxruntime-web": "^1.23.2", "papaparse": "^5.5.3", - "posthog-js": "^1.356.1", "telegram": "^2.26.22", "topojson-client": "^3.1.0", "ws": "^8.19.0", @@ -3379,252 +3378,6 @@ "license": "MIT", "peer": true }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/exporter-logs-otlp-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.208.0.tgz", - "integrity": "sha512-jOv40Bs9jy9bZVLo/i8FwUiuCvbjWDI+ZW13wimJm4LjnlwJxGgB+N/VWOZUTpM+ah/awXeQqKdNlpLf2EjvYg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "@opentelemetry/core": "2.2.0", - "@opentelemetry/otlp-exporter-base": "0.208.0", - "@opentelemetry/otlp-transformer": "0.208.0", - "@opentelemetry/sdk-logs": "0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.208.0.tgz", - "integrity": "sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/otlp-transformer": "0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/otlp-transformer": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.208.0.tgz", - "integrity": "sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0", - "@opentelemetry/sdk-logs": "0.208.0", - "@opentelemetry/sdk-metrics": "2.2.0", - "@opentelemetry/sdk-trace-base": "2.2.0", - "protobufjs": "^7.3.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.1.tgz", - "integrity": "sha512-BViBCdE/GuXRlp9k7nS1w6wJvY5fnFX5XvuEtWsTAOQFIO89Eru7lGW3WbfbxtCuZ/GbrJfAziXG0w0dpxL7eQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.5.1", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", - "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.208.0.tgz", - "integrity": "sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-metrics": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.2.0.tgz", - "integrity": "sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz", - "integrity": "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.39.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.39.0.tgz", - "integrity": "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, "node_modules/@playwright/test": { "version": "1.58.2", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", @@ -3651,21 +3404,6 @@ "@webcomponents/shadycss": "^1.9.1" } }, - "node_modules/@posthog/core": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.23.1.tgz", - "integrity": "sha512-GViD5mOv/mcbZcyzz3z9CS0R79JzxVaqEz4sP5Dsea178M/j3ZWe6gaHDZB9yuyGfcmIMQ/8K14yv+7QrK4sQQ==", - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.6" - } - }, - "node_modules/@posthog/types": { - "version": "1.356.1", - "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.356.1.tgz", - "integrity": "sha512-miIUjs4LiBDMOxKkC87HEJLIih0pNGMAjxx+mW4X7jLpN41n0PLMW7swRE6uuxcMV0z3H6MllRSCYmsokkyfuQ==", - "license": "MIT" - }, "node_modules/@probe.gl/env": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/@probe.gl/env/-/env-4.1.0.tgz", @@ -6619,17 +6357,6 @@ "node": ">=0.10.0" } }, - "node_modules/core-js": { - "version": "3.48.0", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.48.0.tgz", - "integrity": "sha512-zpEHTy1fjTMZCKLHUZoVeylt9XrzaIN2rbPXEt0k+q7JE5CkCZdo6bNq55bn24a69CH7ErAVLKijxJja4fw+UQ==", - "hasInstallScript": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/core-js" - } - }, "node_modules/core-js-compat": { "version": "3.48.0", "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.48.0.tgz", @@ -6672,6 +6399,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -7592,15 +7320,6 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, - "node_modules/dompurify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.1.tgz", - "integrity": "sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==", - "license": "(MPL-2.0 OR Apache-2.0)", - "optionalDependencies": { - "@types/trusted-types": "^2.0.7" - } - }, "node_modules/domutils": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", @@ -9346,6 +9065,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, "license": "ISC" }, "node_modules/jackspeak": { @@ -10886,6 +10606,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -11119,33 +10840,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/posthog-js": { - "version": "1.356.1", - "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.356.1.tgz", - "integrity": "sha512-4EQliSyTp3j/xOaWpZmu7fk1b4S+J3qy4JOu5Xy3/MYFxv1SlAylgifRdCbXZxCQWb6PViaNvwRf4EmburgfWA==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/api-logs": "^0.208.0", - "@opentelemetry/exporter-logs-otlp-http": "^0.208.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-logs": "^0.208.0", - "@posthog/core": "1.23.1", - "@posthog/types": "1.356.1", - "core-js": "^3.38.1", - "dompurify": "^3.3.1", - "fflate": "^0.4.8", - "preact": "^10.28.2", - "query-selector-shadow-dom": "^1.0.1", - "web-vitals": "^5.1.0" - } - }, - "node_modules/posthog-js/node_modules/fflate": { - "version": "0.4.8", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", - "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==", - "license": "MIT" - }, "node_modules/potpack": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.1.0.tgz", @@ -11342,12 +11036,6 @@ "node": ">=18" } }, - "node_modules/query-selector-shadow-dom": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/query-selector-shadow-dom/-/query-selector-shadow-dom-1.0.1.tgz", - "integrity": "sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==", - "license": "MIT" - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -11935,6 +11623,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -11947,6 +11636,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -13701,12 +13391,6 @@ "@esbuild/win32-x64": "0.25.12" } }, - "node_modules/web-vitals": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-5.1.0.tgz", - "integrity": "sha512-ArI3kx5jI0atlTtmV0fWU3fjpLmq/nD3Zr1iFFlJLaqa5wLBkUSzINwBPySCX/8jRyjlmy1Volw1kz1g9XE4Jg==", - "license": "Apache-2.0" - }, "node_modules/webidl-conversions": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", @@ -13768,6 +13452,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index 6671c4f50..832040d7b 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,6 @@ "maplibre-gl": "^5.16.0", "onnxruntime-web": "^1.23.2", "papaparse": "^5.5.3", - "posthog-js": "^1.356.1", "telegram": "^2.26.22", "topojson-client": "^3.1.0", "ws": "^8.19.0", diff --git a/playground-settings-A-sidebar.html b/playground-settings-A-sidebar.html new file mode 100644 index 000000000..6818e2b3e --- /dev/null +++ b/playground-settings-A-sidebar.html @@ -0,0 +1,457 @@ + + + + + +Settings Option A — Sidebar + Content Panel + + + + +
Option A — Sidebar
+ +
+
+
+ + World Monitor Settings +
+ v2.5.19 +
+ +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/playground-settings-B-accordion.html b/playground-settings-B-accordion.html new file mode 100644 index 000000000..491a1d151 --- /dev/null +++ b/playground-settings-B-accordion.html @@ -0,0 +1,330 @@ + + + + + +Settings Option B — Accordion Sections + + + + +
Option B — Accordion
+ +
+
+
+ + Settings +
+ +
+
+ 12/17 +
+
+ +
+ +
+
+ +Cancel + + + + + \ No newline at end of file diff --git a/playground-settings-C-cards.html b/playground-settings-C-cards.html new file mode 100644 index 000000000..c648a925f --- /dev/null +++ b/playground-settings-C-cards.html @@ -0,0 +1,321 @@ + + + + + +Settings Option C — Dashboard Card Grid + + + + +
Option C — Card Grid
+ +
+
+
+ + World Monitor +
+
+
+ 12/17 + features configured +
+
+
+ + +
+ +
+ + +
+ + + + \ No newline at end of file diff --git a/scripts/fetch-mirta-bases.mjs b/scripts/fetch-mirta-bases.mjs new file mode 100644 index 000000000..859416bef --- /dev/null +++ b/scripts/fetch-mirta-bases.mjs @@ -0,0 +1,304 @@ +#!/usr/bin/env node +/** + * Fetch MIRTA (Military Installations, Ranges and Training Areas) dataset + * from the US Army Corps of Engineers ArcGIS FeatureServer. + * + * Source: https://geospatial-usace.opendata.arcgis.com/maps/fc0f38c5a19a46dbacd92f2fb823ef8c + * API: https://services7.arcgis.com/n1YM8pTrFmm7L4hs/arcgis/rest/services/mirta/FeatureServer + * + * Layers: + * 0 = DoD Sites - Point (737 features) + * 1 = DoD Sites - Boundary (825 features, polygons) + */ + +import { writeFileSync, mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = resolve(__dirname, 'data'); +mkdirSync(DATA_DIR, { recursive: true }); + +const BASE = 'https://services7.arcgis.com/n1YM8pTrFmm7L4hs/arcgis/rest/services/mirta/FeatureServer'; +const PAGE_SIZE = 1000; // server maxRecordCount + +// --------------------------------------------------------------------------- +// Branch / component mapping +// --------------------------------------------------------------------------- +const BRANCH_MAP = { + usa: 'Army', + usar: 'Army Reserve', + armynationalguard: 'Army National Guard', + usaf: 'Air Force', + afr: 'Air Force Reserve', + airnationalguard: 'Air National Guard', + usmc: 'Marine Corps', + usmcr: 'Marine Corps Reserve', + usn: 'Navy', + usnr: 'Navy Reserve', + whs: 'Washington Headquarters Services', + other: 'Other', +}; + +const COMPONENT_MAP = { + usa: 'Active', + usar: 'Reserve', + armynationalguard: 'National Guard', + usaf: 'Active', + afr: 'Reserve', + airnationalguard: 'National Guard', + usmc: 'Active', + usmcr: 'Reserve', + usn: 'Active', + usnr: 'Reserve', + whs: 'Active', + other: 'Unknown', +}; + +const STATUS_MAP = { + act: 'Active', + clsd: 'Closed', + semi: 'Semi-Active', + care: 'Caretaker', + excs: 'Excess', +}; + +const STATE_MAP = { + al: 'Alabama', ak: 'Alaska', az: 'Arizona', ar: 'Arkansas', ca: 'California', + co: 'Colorado', ct: 'Connecticut', de: 'Delaware', fl: 'Florida', ga: 'Georgia', + hi: 'Hawaii', id: 'Idaho', il: 'Illinois', in: 'Indiana', ia: 'Iowa', + ks: 'Kansas', ky: 'Kentucky', la: 'Louisiana', me: 'Maine', md: 'Maryland', + ma: 'Massachusetts', mi: 'Michigan', mn: 'Minnesota', ms: 'Mississippi', + mo: 'Missouri', mt: 'Montana', ne: 'Nebraska', nv: 'Nevada', nh: 'New Hampshire', + nj: 'New Jersey', nm: 'New Mexico', ny: 'New York', nc: 'North Carolina', + nd: 'North Dakota', oh: 'Ohio', ok: 'Oklahoma', or: 'Oregon', pa: 'Pennsylvania', + ri: 'Rhode Island', sc: 'South Carolina', sd: 'South Dakota', tn: 'Tennessee', + tx: 'Texas', ut: 'Utah', vt: 'Vermont', va: 'Virginia', wa: 'Washington', + wv: 'West Virginia', wi: 'Wisconsin', wy: 'Wyoming', dc: 'District of Columbia', + pr: 'Puerto Rico', gu: 'Guam', vi: 'Virgin Islands', as: 'American Samoa', + mp: 'Northern Mariana Islands', +}; + +// --------------------------------------------------------------------------- +// Paginated ArcGIS fetch +// --------------------------------------------------------------------------- +async function fetchAllFeatures(layerIndex) { + let offset = 0; + let page = 0; + const allFeatures = []; + let exceeded = true; + + while (exceeded) { + page++; + const params = new URLSearchParams({ + where: '1=1', + outFields: '*', + f: 'geojson', + resultRecordCount: String(PAGE_SIZE), + resultOffset: String(offset), + }); + const url = `${BASE}/${layerIndex}/query?${params}`; + console.log(` Page ${page}: offset=${offset} ...`); + + const resp = await fetch(url); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + const json = await resp.json(); + + const features = json.features || []; + allFeatures.push(...features); + console.log(` Page ${page}: got ${features.length} features (total so far: ${allFeatures.length})`); + + exceeded = json.properties?.exceededTransferLimit === true; + offset += PAGE_SIZE; + } + + return { + type: 'FeatureCollection', + features: allFeatures, + }; +} + +// --------------------------------------------------------------------------- +// Centroid of a polygon (simple average of all coordinates) +// --------------------------------------------------------------------------- +function centroid(geometry) { + if (!geometry) return { lat: null, lon: null }; + + if (geometry.type === 'Point') { + return { lon: geometry.coordinates[0], lat: geometry.coordinates[1] }; + } + + let rings; + if (geometry.type === 'Polygon') { + rings = geometry.coordinates; + } else if (geometry.type === 'MultiPolygon') { + rings = geometry.coordinates.flat(); + } else { + return { lat: null, lon: null }; + } + + let sumLon = 0, sumLat = 0, count = 0; + for (const ring of rings) { + for (const [lon, lat] of ring) { + sumLon += lon; + sumLat += lat; + count++; + } + } + return count > 0 + ? { lon: +(sumLon / count).toFixed(6), lat: +(sumLat / count).toFixed(6) } + : { lat: null, lon: null }; +} + +// --------------------------------------------------------------------------- +// Process features into clean records +// --------------------------------------------------------------------------- +function processFeature(feature) { + const p = feature.properties || {}; + const comp = (p.SITEREPORTINGCOMPONENT || '').toLowerCase().trim(); + const statusRaw = (p.SITEOPERATIONALSTATUS || '').toLowerCase().trim(); + const stateRaw = (p.STATENAMECODE || '').toLowerCase().trim(); + const { lat, lon } = centroid(feature.geometry); + + return { + name: (p.SITENAME || p.FEATURENAME || '').trim(), + branch: BRANCH_MAP[comp] || comp || 'Unknown', + status: STATUS_MAP[statusRaw] || statusRaw || 'Unknown', + state: STATE_MAP[stateRaw] || stateRaw.toUpperCase() || 'Unknown', + lat, + lon, + kind: (p.FEATUREDESCRIPTION && p.FEATUREDESCRIPTION !== 'na') + ? p.FEATUREDESCRIPTION + : 'Installation', + component: COMPONENT_MAP[comp] || 'Unknown', + jointBase: p.ISJOINTBASE === 'yes', + }; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + console.log('=== MIRTA Dataset Fetcher ===\n'); + + // ---------- Points layer (layer 0) ---------- + console.log('[1/4] Fetching Points layer (layer 0)...'); + const pointsGeoJson = await fetchAllFeatures(0); + console.log(` Total point features: ${pointsGeoJson.features.length}\n`); + + // ---------- Boundary layer (layer 1) ---------- + console.log('[2/4] Fetching Boundary layer (layer 1)...'); + const boundaryGeoJson = await fetchAllFeatures(1); + console.log(` Total boundary features: ${boundaryGeoJson.features.length}\n`); + + // ---------- Save raw GeoJSON ---------- + console.log('[3/4] Saving raw GeoJSON...'); + + const combinedRaw = { + type: 'FeatureCollection', + metadata: { + source: 'MIRTA - Military Installations, Ranges and Training Areas', + url: 'https://geospatial-usace.opendata.arcgis.com/maps/fc0f38c5a19a46dbacd92f2fb823ef8c', + fetchedAt: new Date().toISOString(), + pointFeatures: pointsGeoJson.features.length, + boundaryFeatures: boundaryGeoJson.features.length, + }, + features: [ + ...pointsGeoJson.features, + ...boundaryGeoJson.features, + ], + }; + + const rawPath = resolve(DATA_DIR, 'mirta-raw.geojson'); + writeFileSync(rawPath, JSON.stringify(combinedRaw, null, 2)); + const rawSizeMB = (Buffer.byteLength(JSON.stringify(combinedRaw)) / 1024 / 1024).toFixed(2); + console.log(` Saved ${rawPath} (${rawSizeMB} MB)\n`); + + // ---------- Process into clean records ---------- + console.log('[4/4] Processing into clean records...'); + + // Use points layer as primary (has exact coordinates). + // Supplement with boundary-only entries (those not in points). + const pointNames = new Set( + pointsGeoJson.features.map(f => (f.properties?.SITENAME || '').toLowerCase().trim()) + ); + + const processed = []; + + for (const f of pointsGeoJson.features) { + processed.push(processFeature(f)); + } + + let boundaryOnly = 0; + for (const f of boundaryGeoJson.features) { + const name = (f.properties?.SITENAME || '').toLowerCase().trim(); + if (!pointNames.has(name)) { + processed.push(processFeature(f)); + boundaryOnly++; + } + } + + // Sort by name + processed.sort((a, b) => a.name.localeCompare(b.name)); + + const output = { + metadata: { + source: 'MIRTA - Military Installations, Ranges and Training Areas', + url: 'https://geospatial-usace.opendata.arcgis.com/maps/fc0f38c5a19a46dbacd92f2fb823ef8c', + fetchedAt: new Date().toISOString(), + totalInstallations: processed.length, + fromPoints: pointsGeoJson.features.length, + fromBoundariesOnly: boundaryOnly, + }, + installations: processed, + }; + + const processedPath = resolve(DATA_DIR, 'mirta-processed.json'); + writeFileSync(processedPath, JSON.stringify(output, null, 2)); + const procSizeMB = (Buffer.byteLength(JSON.stringify(output)) / 1024 / 1024).toFixed(2); + console.log(` Saved ${processedPath} (${procSizeMB} MB)\n`); + + // ---------- Summary ---------- + console.log('=== Summary ==='); + console.log(`Total installations: ${processed.length}`); + console.log(` From points layer: ${pointsGeoJson.features.length}`); + console.log(` From boundaries only: ${boundaryOnly}`); + + // Branch breakdown + const branchCounts = {}; + const statusCounts = {}; + const componentCounts = {}; + for (const inst of processed) { + branchCounts[inst.branch] = (branchCounts[inst.branch] || 0) + 1; + statusCounts[inst.status] = (statusCounts[inst.status] || 0) + 1; + componentCounts[inst.component] = (componentCounts[inst.component] || 0) + 1; + } + + console.log('\nBy branch:'); + for (const [k, v] of Object.entries(branchCounts).sort((a, b) => b[1] - a[1])) { + console.log(` ${k}: ${v}`); + } + + console.log('\nBy status:'); + for (const [k, v] of Object.entries(statusCounts).sort((a, b) => b[1] - a[1])) { + console.log(` ${k}: ${v}`); + } + + console.log('\nBy component:'); + for (const [k, v] of Object.entries(componentCounts).sort((a, b) => b[1] - a[1])) { + console.log(` ${k}: ${v}`); + } + + console.log('\nSample entries:'); + const samples = [processed[0], processed[Math.floor(processed.length / 3)], processed[Math.floor(processed.length * 2 / 3)], processed[processed.length - 1]]; + for (const s of samples) { + console.log(` ${s.name} | ${s.branch} | ${s.status} | ${s.state} | (${s.lat}, ${s.lon})`); + } + + console.log('\nDone.'); +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +}); diff --git a/scripts/fetch-osm-bases.mjs b/scripts/fetch-osm-bases.mjs new file mode 100644 index 000000000..4eeeedbeb --- /dev/null +++ b/scripts/fetch-osm-bases.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node + +import { writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = join(__dirname, 'data'); +const RAW_PATH = join(DATA_DIR, 'osm-military-raw.json'); +const PROCESSED_PATH = join(DATA_DIR, 'osm-military-processed.json'); + +const OVERPASS_URL = 'https://overpass-api.de/api/interpreter'; +const OVERPASS_QUERY = ` +[out:json][timeout:300]; +( + node["military"]["name"]; + way["military"]["name"]; + relation["military"]["name"]; +); +out center tags; +`.trim(); + +const TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +function ensureDataDir() { + if (!existsSync(DATA_DIR)) { + mkdirSync(DATA_DIR, { recursive: true }); + console.log(`Created directory: ${DATA_DIR}`); + } +} + +async function fetchOverpassData() { + console.log('Querying Overpass API for military features with names...'); + console.log(`Query:\n${OVERPASS_QUERY}\n`); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS); + + try { + const res = await fetch(OVERPASS_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: `data=${encodeURIComponent(OVERPASS_QUERY)}`, + signal: controller.signal, + }); + + clearTimeout(timeout); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Overpass API returned ${res.status}: ${text.slice(0, 500)}`); + } + + console.log('Response received, reading body...'); + const json = await res.json(); + return json; + } catch (err) { + clearTimeout(timeout); + if (err.name === 'AbortError') { + throw new Error('Overpass API request timed out after 5 minutes'); + } + throw err; + } +} + +function processFeatures(raw) { + const elements = raw.elements || []; + console.log(`Raw elements count: ${elements.length}`); + + const processed = elements.map((el) => { + const tags = el.tags || {}; + + // Coordinates: nodes have lat/lon directly; ways/relations use center + const lat = el.lat ?? el.center?.lat ?? null; + const lon = el.lon ?? el.center?.lon ?? null; + + const typePrefix = el.type; // node, way, relation + const osmId = `${typePrefix}/${el.id}`; + + const name = tags['name:en'] || tags.name || ''; + const country = tags['addr:country'] || ''; + const kind = tags.military || ''; + const operator = tags.operator || ''; + const description = tags.description || ''; + const militaryBranch = tags.military_branch || ''; + + return { + osm_id: osmId, + name, + country, + kind, + lat, + lon, + operator, + description, + military_branch: militaryBranch, + }; + }); + + // Filter out entries without coordinates + const withCoords = processed.filter((f) => f.lat != null && f.lon != null); + const skipped = processed.length - withCoords.length; + if (skipped > 0) { + console.log(`Skipped ${skipped} features without coordinates`); + } + + return withCoords; +} + +function printSummary(features) { + console.log(`\n--- Summary ---`); + console.log(`Total processed features: ${features.length}`); + + // Count by kind + const kindCounts = {}; + for (const f of features) { + kindCounts[f.kind] = (kindCounts[f.kind] || 0) + 1; + } + console.log('\nBy military tag value:'); + const sorted = Object.entries(kindCounts).sort((a, b) => b[1] - a[1]); + for (const [kind, count] of sorted) { + console.log(` ${kind}: ${count}`); + } + + // Count with country + const withCountry = features.filter((f) => f.country).length; + console.log(`\nFeatures with country tag: ${withCountry}`); + + // Sample entries + console.log('\nSample entries (first 5):'); + for (const f of features.slice(0, 5)) { + console.log(` ${f.osm_id} | ${f.name} | ${f.kind} | ${f.lat?.toFixed(4)},${f.lon?.toFixed(4)} | ${f.country || '(no country)'}`); + } +} + +async function main() { + const start = Date.now(); + ensureDataDir(); + + const raw = await fetchOverpassData(); + + // Save raw + console.log(`Saving raw response to ${RAW_PATH}...`); + writeFileSync(RAW_PATH, JSON.stringify(raw, null, 2)); + console.log('Raw data saved.'); + + // Process + const features = processFeatures(raw); + + // Save processed + console.log(`Saving processed data to ${PROCESSED_PATH}...`); + writeFileSync(PROCESSED_PATH, JSON.stringify(features, null, 2)); + console.log('Processed data saved.'); + + printSummary(features); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\nDone in ${elapsed}s`); +} + +main().catch((err) => { + console.error('Fatal error:', err.message); + process.exit(1); +}); diff --git a/scripts/need-work.csv b/scripts/need-work.csv new file mode 100644 index 000000000..6a405702e --- /dev/null +++ b/scripts/need-work.csv @@ -0,0 +1,46 @@ +#,Variant,Category,Feed Name,Status,Newest Date,Error,URL,Replacement,Notes +32,full,europe,Corriere della Sera,DEAD,,HTTP 404,https://xml2.corriereobjects.it/rss/incipit.xml,https://www.corriere.it/rss/homepage.xml,Direct RSS 69 items verified +36,full,europe,De Telegraaf,DEAD,,HTTP 403,https://www.telegraaf.nl/rss,"https://news.google.com/rss/search?q=site:telegraaf.nl+when:1d&hl=nl&gl=NL&ceid=NL:nl",Direct feed blocks crawlers; Google News 100 items +38,full,europe,Dagens Nyheter,DEAD,,HTTP 404,https://www.dn.se/rss/senaste-nytt/,https://www.dn.se/rss/,Drop /senaste-nytt/ suffix; 116 items verified +65,full,middleeast,L'Orient-Le Jour,DEAD,,HTTP 403,https://www.lorientlejour.com/rss,"https://news.google.com/rss/search?q=site:lorientlejour.com+when:1d&hl=fr&gl=LB&ceid=LB:fr",Direct 403; Google News 74 items French +132,full,latam,O Globo,DEAD,,HTTP 404,https://oglobo.globo.com/rss/top_noticias/,"https://news.google.com/rss/search?q=site:oglobo.globo.com+when:1d&hl=pt-BR&gl=BR&ceid=BR:pt-419",Direct RSS empty shell; Google News 100 items +133,full,latam,Folha de S.Paulo,DEAD,,fetch failed,https://feeds.folha.uol.com.br/emcimadahora/rss091.xml,KEEP,Transient failure; 100 items on retest +136,full,latam,El Universal,DEAD,,HTTP 404,https://www.eluniversal.com.mx/rss.xml,"https://news.google.com/rss/search?q=site:eluniversal.com.mx+when:1d&hl=es-419&gl=MX&ceid=MX:es-419",All direct paths 404; Google News 100 items +139,full,latam,Animal Político,DEAD,,HTTP 404,https://animalpolitico.com/feed/,"https://news.google.com/rss/search?q=site:animalpolitico.com+when:1d&hl=es-419&gl=MX&ceid=MX:es-419",Direct 404; Google News 98 items +140,full,latam,Proceso,DEAD,,HTTP 404,https://www.proceso.com.mx/feed/,"https://news.google.com/rss/search?q=site:proceso.com.mx+when:1d&hl=es-419&gl=MX&ceid=MX:es-419",Direct 404; Google News 100 items +141,full,latam,Milenio,DEAD,,HTTP 404,https://www.milenio.com/rss,"https://news.google.com/rss/search?q=site:milenio.com+when:1d&hl=es-419&gl=MX&ceid=MX:es-419",All direct paths 404; Google News 100 items +161,full,asia,Bangkok Post,DEAD,,HTTP 451,https://www.bangkokpost.com/rss,"https://news.google.com/rss/search?q=site:bangkokpost.com+when:1d&hl=en-US&gl=US&ceid=US:en",Geo-blocked 451; Google News 42 items +377,happy,science,ScienceDaily,DEAD,,Timeout,https://www.sciencedaily.com/rss/top.xml,https://www.sciencedaily.com/rss/all.xml,top.xml empty; all.xml has 40 items verified +388,intel,inspiring,Breaking Defense,DEAD,,HTTP 403,https://breakingdefense.com/feed/,KEEP,Works with proper User-Agent; 15 items verified +402,intel,inspiring,RAND,DEAD,,HTTP 404,https://www.rand.org/rss/all.xml,"https://news.google.com/rss/search?q=site:rand.org+when:7d&hl=en-US&gl=US&ceid=US:en",Direct 403; Google News 50 items +406,intel,inspiring,NTI,DEAD,,HTTP 403,https://www.nti.org/rss/,"https://news.google.com/rss/search?q=site:nti.org+when:30d&hl=en-US&gl=US&ceid=US:en",Direct feed empty; Google News 30d window 27 items +415,intel,inspiring,Bellingcat,DEAD,,fetch failed,https://www.bellingcat.com/feed/,"https://news.google.com/rss/search?q=site:bellingcat.com+when:30d&hl=en-US&gl=US&ceid=US:en",SSL handshake fails; Google News 30d 19 items (low pub freq) +23,full,europe,DW News [es],EMPTY,,No dates found,https://rss.dw.com/xml/rss-es-all,"https://news.google.com/rss/search?q=site:dw.com/es&hl=es-419&gl=MX&ceid=MX:es-419",DW deprecated es RSS endpoint; Google News 100 items +28,full,europe,Bild,EMPTY,,No dates found,https://www.bild.de/feed/alles.xml,KEEP (parser fix),Feed works; dates use CET/CEST timezone abbreviation not RFC 2822 +110,full,crisis,CrisisWatch,EMPTY,,No dates found,https://www.crisisgroup.org/rss,KEEP (parser fix),"Feed works; Drupal date format: ""Wednesday, February 25, 2026 - 21:07""" +111,full,crisis,IAEA,EMPTY,,No dates found,https://www.iaea.org/feeds/topnews,KEEP (parser fix),"Feed works; 2-digit year: ""Thu, 26 Feb 26"" needs expansion to 2026" +116,full,africa,News24,EMPTY,,No dates found,https://feeds.capi24.com/v1/Search/articles/news24/Africa/rss,https://feeds.news24.com/articles/news24/TopStories/rss,Old CAPI feed empty; new URL 20 items verified +157,full,asia,India News Network,EMPTY,,No dates found,https://www.indianewsnetwork.com/rss.en.diplomacy.xml,"https://news.google.com/rss/search?q=India+diplomacy+foreign+policy+news&hl=en&gl=US&ceid=US:en",Original feed has zero date fields in any item +162,full,asia,Thai PBS,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:thaipbsworld.com+when:2d&hl=th&gl=TH&ceid=TH:th","https://news.google.com/rss/search?q=Thai+PBS+World+news&hl=en&gl=US&ceid=US:en",Site moved to world.thaipbs.or.th no RSS; sparse results consider REMOVE +163,full,asia,VnExpress,EMPTY,,No dates found,https://vnexpress.net/rss,https://vnexpress.net/rss/tin-moi-nhat.rss,Bare /rss is HTML index; correct endpoint is /rss/tin-moi-nhat.rss 55 items +164,full,asia,Tuoi Tre News,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:tuoitrenews.vn+when:2d&hl=vi&gl=VN&ceid=VN:vi",https://tuoitrenews.vn/rss,Direct RSS works 50 items; Google News was stale +231,tech,regionalStartups,Disrupt Africa,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:disrupt-africa.com+when:7d&hl=en-US&gl=US&ceid=US:en",REMOVE,Last post Jan 2024; site inactive; no Google News results +237,tech,github,GitHub Trending,EMPTY,,No dates found,https://mshibanami.github.io/GitHubTrendingRSS/daily/all.xml,KEEP (parser fix),Feed works with current items; parser may not handle its date format +268,tech,thinktanks,MIT Tech Policy,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:techpolicypress.org+when:14d&hl=en-US&gl=US&ceid=US:en","https://news.google.com/rss/search?q=%22Tech+Policy+Press%22&hl=en&gl=US&ceid=US:en",Domain DNS fails; search by name returns 100 items +270,tech,thinktanks,AI Now Institute,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:ainowinstitute.org+when:14d&hl=en-US&gl=US&ceid=US:en","https://news.google.com/rss/search?q=%22AI+Now+Institute%22&hl=en&gl=US&ceid=US:en",SSL issue on direct; Google News 59 items (infrequent publisher) +279,tech,thinktanks,DigiChina,EMPTY,,No dates found,"https://news.google.com/rss/search?q=site:digichina.stanford.edu+when:14d&hl=en-US&gl=US&ceid=US:en","https://news.google.com/rss/search?q=DigiChina+Stanford+China+technology&hl=en&gl=US&ceid=US:en",WordPress RSS empty; Google News 20 items to Jul 2025 +306,tech,podcasts,20VC Episodes,EMPTY,,No dates found,"https://news.google.com/rss/search?q=""20+Minute+VC""+Harry+Stebbings+when:14d&hl=en-US&gl=US&ceid=US:en",https://rss.libsyn.com/shows/61840/destinations/240976.xml,Official podcast RSS via Apple; 1423 episodes current +310,tech,podcasts,Pivot Podcast,EMPTY,,No dates found,"https://news.google.com/rss/search?q=""Pivot+podcast""+(Kara+Swisher+OR+Scott+Galloway)+when:14d&hl=en-US&gl=US&ceid=US:en",https://feeds.megaphone.fm/pivot,Megaphone RSS; 750 episodes current +315,tech,podcasts,Startup Podcasts,EMPTY,,No dates found,"https://news.google.com/rss/search?q=(""Masters+of+Scale""+OR+""The+Pitch+podcast""+OR+""startup+podcast"")+episode+when:14d&hl=en-US&gl=US&ceid=US:en",https://rss.art19.com/masters-of-scale,"Masters of Scale RSS 670 eps; ""The Pitch"" feeds 404 — drop it" +379,happy,science,Live Science,EMPTY,,No dates found,https://www.livescience.com/feeds/all,https://www.livescience.com/feeds.xml,/feeds/all redirects to /feeds.xml; 20+ items current +383,happy,science,Greater Good (Berkeley),EMPTY,,No dates found,https://greatergood.berkeley.edu/rss,https://greatergood.berkeley.edu/site/rss/articles,/rss is 404; correct path /site/rss/articles 50 items; uses dc:date +398,intel,inspiring,CSIS,EMPTY,,No dates found,https://www.csis.org/analysis?type=analysis,"https://news.google.com/rss/search?q=site:csis.org&hl=en&gl=US&ceid=US:en",Not an RSS URL (HTML); all RSS paths 403; Google News 100 items +403,intel,inspiring,Brookings,EMPTY,,No dates found,https://www.brookings.edu/feed/,"https://news.google.com/rss/search?q=site:brookings.edu&hl=en&gl=US&ceid=US:en",WordPress feed bot-blocked; Google News 100 items +404,intel,inspiring,Carnegie,EMPTY,,No dates found,https://carnegieendowment.org/rss/,"https://news.google.com/rss/search?q=site:carnegieendowment.org&hl=en&gl=US&ceid=US:en",Next.js site returns HTML for RSS paths; Google News 100 items +5,full,politics,CNN World,STALE,18/09/2023,Stale,http://rss.cnn.com/rss/cnn_world.rss,"https://news.google.com/rss/search?q=site:cnn.com+world+news+when:1d&hl=en-US&gl=US&ceid=US:en",rss.cnn.com SSL failures; use Google News proxy for CNN world +43,full,europe,TVN24,STALE,01/04/2025,Stale,https://tvn24.pl/najwazniejsze.xml,https://tvn24.pl/swiat.xml,najwazniejsze.xml stale; swiat.xml (world) 30 items current +73,full,ai,VentureBeat AI,STALE,22/01/2026,Stale,https://venturebeat.com/category/ai/feed/,KEEP,Borderline stale; 308 redirect issue; sparse 7-item feed by design +84,full,gov,Pentagon,STALE,23/01/2026,Stale,"https://news.google.com/rss/search?q=site:defense.gov+OR+Pentagon&hl=en-US&gl=US&ceid=US:en",KEEP,Borderline; defense.gov low recent output; Google News proxy working +94,full,layoffs,Layoffs.fyi,STALE,29/12/2020,Stale,https://layoffs.fyi/feed/,"https://news.google.com/rss/search?q=tech+company+layoffs+announced&hl=en&gl=US&ceid=US:en",Feed abandoned Dec 2020; Google News 100 items +396,intel,inspiring,Oryx OSINT,STALE,07/12/2024,Stale,https://www.oryxspioenkop.com/feeds/posts/default?alt=rss,KEEP,Publishes infrequently by design (detailed equipment loss lists) +405,intel,inspiring,FAS,STALE,14/02/2023,Stale,https://fas.org/feed/,"https://news.google.com/rss/search?q=site:fas.org+nuclear+weapons+security&hl=en&gl=US&ceid=US:en",RSS broken (1 item from 2023); Google News proxy available diff --git a/scripts/rss-feeds-report.csv b/scripts/rss-feeds-report.csv new file mode 100644 index 000000000..196405d8d --- /dev/null +++ b/scripts/rss-feeds-report.csv @@ -0,0 +1,421 @@ +#,Variant,Category,Feed Name,Status,Newest Date,Error,URL +1,full,politics,"BBC World",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/news/world/rss.xml" +2,full,politics,"Guardian World",OK,2026-02-26,"OK","https://www.theguardian.com/world/rss" +3,full,politics,"AP News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:apnews.com&hl=en-US&gl=US&ceid=US:en" +4,full,politics,"Reuters World",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+world&hl=en-US&gl=US&ceid=US:en" +5,full,politics,"CNN World",STALE,2023-09-18,"Stale","http://rss.cnn.com/rss/cnn_world.rss" +6,full,us,"NPR News",OK,2026-02-26,"OK","https://feeds.npr.org/1001/rss.xml" +7,full,us,"Politico",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:politico.com+when:1d&hl=en-US&gl=US&ceid=US:en" +8,full,europe,"France 24 [en]",OK,2026-02-26,"OK","https://www.france24.com/en/rss" +9,full,europe,"France 24 [fr]",OK,2026-02-26,"OK","https://www.france24.com/fr/rss" +10,full,europe,"France 24 [es]",OK,2026-02-26,"OK","https://www.france24.com/es/rss" +11,full,europe,"France 24 [ar]",OK,2026-02-26,"OK","https://www.france24.com/ar/rss" +12,full,europe,"EuroNews [en]",OK,2026-02-26,"OK","https://www.euronews.com/rss?format=xml" +13,full,europe,"EuroNews [fr]",OK,2026-02-26,"OK","https://fr.euronews.com/rss?format=xml" +14,full,europe,"EuroNews [de]",OK,2026-02-26,"OK","https://de.euronews.com/rss?format=xml" +15,full,europe,"EuroNews [it]",OK,2026-02-26,"OK","https://it.euronews.com/rss?format=xml" +16,full,europe,"EuroNews [es]",OK,2026-02-26,"OK","https://es.euronews.com/rss?format=xml" +17,full,europe,"EuroNews [pt]",OK,2026-02-26,"OK","https://pt.euronews.com/rss?format=xml" +18,full,europe,"EuroNews [ru]",OK,2026-02-26,"OK","https://ru.euronews.com/rss?format=xml" +19,full,europe,"Le Monde [en]",OK,2026-02-26,"OK","https://www.lemonde.fr/en/rss/une.xml" +20,full,europe,"Le Monde [fr]",OK,2026-02-26,"OK","https://www.lemonde.fr/rss/une.xml" +21,full,europe,"DW News [en]",OK,2026-02-26,"OK","https://rss.dw.com/xml/rss-en-all" +22,full,europe,"DW News [de]",OK,2026-02-26,"OK","https://rss.dw.com/xml/rss-de-all" +23,full,europe,"DW News [es]",EMPTY,,"No dates found","https://rss.dw.com/xml/rss-es-all" +24,full,europe,"El País",OK,2026-02-26,"OK","https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/portada" +25,full,europe,"El Mundo",OK,2026-02-26,"OK","https://e00-elmundo.uecdn.es/elmundo/rss/portada.xml" +26,full,europe,"BBC Mundo",OK,2026-02-26,"OK","https://www.bbc.com/mundo/index.xml" +27,full,europe,"Tagesschau",OK,2026-02-26,"OK","https://www.tagesschau.de/xml/rss2/" +28,full,europe,"Bild",EMPTY,,"No dates found","https://www.bild.de/feed/alles.xml" +29,full,europe,"Der Spiegel",OK,2026-02-26,"OK","https://www.spiegel.de/schlagzeilen/tops/index.rss" +30,full,europe,"Die Zeit",OK,2026-02-26,"OK","https://newsfeed.zeit.de/index" +31,full,europe,"ANSA",OK,2026-02-26,"OK","https://www.ansa.it/sito/notizie/topnews/topnews_rss.xml" +32,full,europe,"Corriere della Sera",DEAD,,"HTTP 404","https://xml2.corriereobjects.it/rss/incipit.xml" +33,full,europe,"Repubblica",OK,2026-02-26,"OK","https://www.repubblica.it/rss/homepage/rss2.0.xml" +34,full,europe,"NOS Nieuws",OK,2026-02-26,"OK","https://feeds.nos.nl/nosnieuwsalgemeen" +35,full,europe,"NRC",OK,2026-02-26,"OK","https://www.nrc.nl/rss/" +36,full,europe,"De Telegraaf",DEAD,,"HTTP 403","https://www.telegraaf.nl/rss" +37,full,europe,"SVT Nyheter",OK,2026-02-26,"OK","https://www.svt.se/nyheter/rss.xml" +38,full,europe,"Dagens Nyheter",DEAD,,"HTTP 404","https://www.dn.se/rss/senaste-nytt/" +39,full,europe,"Svenska Dagbladet",OK,2026-02-26,"OK","https://www.svd.se/feed/articles.rss" +40,full,europe,"BBC Turkce",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/turkce/rss.xml" +41,full,europe,"DW Turkish",OK,2026-02-26,"OK","https://rss.dw.com/xml/rss-tur-all" +42,full,europe,"Hurriyet",OK,2026-02-26,"OK","https://www.hurriyet.com.tr/rss/anasayfa" +43,full,europe,"TVN24",STALE,2025-04-01,"Stale","https://tvn24.pl/najwazniejsze.xml" +44,full,europe,"Polsat News",OK,2026-02-26,"OK","https://www.polsatnews.pl/rss/wszystkie.xml" +45,full,europe,"Rzeczpospolita",OK,2026-02-26,"OK","https://www.rp.pl/rss_main" +46,full,europe,"Kathimerini",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:kathimerini.gr+when:2d&hl=el&gl=GR&ceid=GR:el" +47,full,europe,"Naftemporiki",OK,2026-02-26,"OK","https://www.naftemporiki.gr/feed/" +48,full,europe,"in.gr",OK,2026-02-26,"OK","https://www.in.gr/feed/" +49,full,europe,"iefimerida",OK,2026-02-26,"OK","https://www.iefimerida.gr/rss.xml" +50,full,europe,"Proto Thema",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:protothema.gr+when:2d&hl=el&gl=GR&ceid=GR:el" +51,full,europe,"BBC Russian",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/russian/rss.xml" +52,full,europe,"Meduza",OK,2026-02-26,"OK","https://meduza.io/rss/all" +53,full,europe,"Novaya Gazeta Europe",OK,2026-02-26,"OK","https://novayagazeta.eu/feed/rss" +54,full,europe,"TASS",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:tass.com+OR+TASS+Russia+when:1d&hl=en-US&gl=US&ceid=US:en" +55,full,europe,"Kyiv Independent",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:kyivindependent.com+when:3d&hl=en-US&gl=US&ceid=US:en" +56,full,europe,"Moscow Times",OK,2026-02-26,"OK","https://www.themoscowtimes.com/rss/news" +57,full,middleeast,"BBC Middle East",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/news/world/middle_east/rss.xml" +58,full,middleeast,"Al Jazeera [en]",OK,2026-02-26,"OK","https://www.aljazeera.com/xml/rss/all.xml" +59,full,middleeast,"Al Jazeera [ar]",OK,2026-02-26,"OK","https://www.aljazeera.net/aljazeerarss/a7c186be-1adb-4b11-a982-4783e765316e/4e17ecdc-8fb9-40de-a5d6-d00f72384a51" +60,full,middleeast,"Al Arabiya",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:english.alarabiya.net+when:2d&hl=en-US&gl=US&ceid=US:en" +61,full,middleeast,"Guardian ME",OK,2026-02-26,"OK","https://www.theguardian.com/world/middleeast/rss" +62,full,middleeast,"BBC Persian",OK,2026-02-26,"OK","http://feeds.bbci.co.uk/persian/tv-and-radio-37434376/rss.xml" +63,full,middleeast,"Iran International",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:iranintl.com+when:2d&hl=en-US&gl=US&ceid=US:en" +64,full,middleeast,"Fars News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:farsnews.ir+when:2d&hl=en-US&gl=US&ceid=US:en" +65,full,middleeast,"L\",DEAD,,"HTTP 403","https://www.lorientlejour.com/rss" +66,full,middleeast,"Haaretz",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:haaretz.com+when:7d&hl=en-US&gl=US&ceid=US:en" +67,full,middleeast,"Arab News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:arabnews.com+when:7d&hl=en-US&gl=US&ceid=US:en" +68,full,tech,"Hacker News",OK,2026-02-26,"OK","https://hnrss.org/frontpage" +69,full,tech,"Ars Technica",OK,2026-02-26,"OK","https://feeds.arstechnica.com/arstechnica/technology-lab" +70,full,tech,"The Verge",OK,2026-02-26,"OK","https://www.theverge.com/rss/index.xml" +71,full,tech,"MIT Tech Review",OK,2026-02-26,"OK","https://www.technologyreview.com/feed/" +72,full,ai,"AI News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(OpenAI+OR+Anthropic+OR+Google+AI+OR+""large+language+model""+OR+ChatGPT)+when:2d&hl=en-US&gl=US&ceid=US:en" +73,full,ai,"VentureBeat AI",STALE,2026-01-22,"Stale","https://venturebeat.com/category/ai/feed/" +74,full,ai,"The Verge AI",OK,2026-02-26,"OK","https://www.theverge.com/rss/ai-artificial-intelligence/index.xml" +75,full,ai,"MIT Tech Review",OK,2026-02-26,"OK","https://www.technologyreview.com/topic/artificial-intelligence/feed" +76,full,ai,"ArXiv AI",OK,2026-02-26,"OK","https://export.arxiv.org/rss/cs.AI" +77,full,finance,"CNBC",OK,2026-02-26,"OK","https://www.cnbc.com/id/100003114/device/rss/rss.html" +78,full,finance,"MarketWatch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:marketwatch.com+markets+when:1d&hl=en-US&gl=US&ceid=US:en" +79,full,finance,"Yahoo Finance",OK,2026-02-25,"OK","https://finance.yahoo.com/news/rssindex" +80,full,finance,"Financial Times",OK,2026-02-26,"OK","https://www.ft.com/rss/home" +81,full,finance,"Reuters Business",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+business+markets&hl=en-US&gl=US&ceid=US:en" +82,full,gov,"White House",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:whitehouse.gov&hl=en-US&gl=US&ceid=US:en" +83,full,gov,"State Dept",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:state.gov+OR+""State+Department""&hl=en-US&gl=US&ceid=US:en" +84,full,gov,"Pentagon",STALE,2026-01-23,"Stale","https://news.google.com/rss/search?q=site:defense.gov+OR+Pentagon&hl=en-US&gl=US&ceid=US:en" +85,full,gov,"Treasury",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:treasury.gov+OR+""Treasury+Department""&hl=en-US&gl=US&ceid=US:en" +86,full,gov,"DOJ",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:justice.gov+OR+""Justice+Department""+DOJ&hl=en-US&gl=US&ceid=US:en" +87,full,gov,"Federal Reserve",OK,2026-02-24,"OK","https://www.federalreserve.gov/feeds/press_all.xml" +88,full,gov,"SEC",OK,2026-02-26,"OK","https://www.sec.gov/news/pressreleases.rss" +89,full,gov,"CDC",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:cdc.gov+OR+CDC+health&hl=en-US&gl=US&ceid=US:en" +90,full,gov,"FEMA",OK,2026-02-21,"OK","https://news.google.com/rss/search?q=site:fema.gov+OR+FEMA+emergency&hl=en-US&gl=US&ceid=US:en" +91,full,gov,"DHS",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:dhs.gov+OR+""Homeland+Security""&hl=en-US&gl=US&ceid=US:en" +92,full,gov,"UN News",OK,2026-02-26,"OK","https://news.un.org/feed/subscribe/en/news/all/rss.xml" +93,full,gov,"CISA",OK,2026-02-26,"OK","https://www.cisa.gov/cybersecurity-advisories/all.xml" +94,full,layoffs,"Layoffs.fyi",STALE,2020-12-29,"Stale","https://layoffs.fyi/feed/" +95,full,layoffs,"TechCrunch Layoffs",OK,2026-02-26,"OK","https://techcrunch.com/tag/layoffs/feed/" +96,full,layoffs,"Layoffs News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(layoffs+OR+""job+cuts""+OR+""workforce+reduction"")+when:3d&hl=en-US&gl=US&ceid=US:en" +97,full,thinktanks,"Foreign Policy",OK,2026-02-26,"OK","https://foreignpolicy.com/feed/" +98,full,thinktanks,"Atlantic Council",OK,2026-02-26,"OK","https://www.atlanticcouncil.org/feed/" +99,full,thinktanks,"Foreign Affairs",OK,2026-02-26,"OK","https://www.foreignaffairs.com/rss.xml" +100,full,thinktanks,"CSIS",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:csis.org+when:7d&hl=en-US&gl=US&ceid=US:en" +101,full,thinktanks,"RAND",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:rand.org+when:7d&hl=en-US&gl=US&ceid=US:en" +102,full,thinktanks,"Brookings",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:brookings.edu+when:7d&hl=en-US&gl=US&ceid=US:en" +103,full,thinktanks,"Carnegie",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:carnegieendowment.org+when:7d&hl=en-US&gl=US&ceid=US:en" +104,full,thinktanks,"War on the Rocks",OK,2026-02-26,"OK","https://warontherocks.com/feed" +105,full,thinktanks,"AEI",OK,2026-02-26,"OK","https://www.aei.org/feed/" +106,full,thinktanks,"Responsible Statecraft",OK,2026-02-26,"OK","https://responsiblestatecraft.org/feed/" +107,full,thinktanks,"RUSI",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:rusi.org+when:3d&hl=en-US&gl=US&ceid=US:en" +108,full,thinktanks,"FPRI",OK,2026-02-26,"OK","https://www.fpri.org/feed/" +109,full,thinktanks,"Jamestown",OK,2026-02-25,"OK","https://jamestown.org/feed/" +110,full,crisis,"CrisisWatch",EMPTY,,"No dates found","https://www.crisisgroup.org/rss" +111,full,crisis,"IAEA",EMPTY,,"No dates found","https://www.iaea.org/feeds/topnews" +112,full,crisis,"WHO",OK,2026-02-25,"OK","https://www.who.int/rss-feeds/news-english.xml" +113,full,crisis,"UNHCR",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:unhcr.org+OR+UNHCR+refugees+when:3d&hl=en-US&gl=US&ceid=US:en" +114,full,africa,"Africa News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Africa+OR+Nigeria+OR+Kenya+OR+""South+Africa""+OR+Ethiopia)+when:2d&hl=en-US&gl=US&ceid=US:en" +115,full,africa,"Sahel Crisis",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Sahel+OR+Mali+OR+Niger+OR+""Burkina+Faso""+OR+Wagner)+when:3d&hl=en-US&gl=US&ceid=US:en" +116,full,africa,"News24",EMPTY,,"No dates found","https://feeds.capi24.com/v1/Search/articles/news24/Africa/rss" +117,full,africa,"BBC Africa",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/news/world/africa/rss.xml" +118,full,africa,"Jeune Afrique",OK,2026-02-26,"OK","https://www.jeuneafrique.com/feed/" +119,full,africa,"Africanews [en]",OK,2026-02-26,"OK","https://www.africanews.com/feed/rss" +120,full,africa,"Africanews [fr]",OK,2026-02-26,"OK","https://fr.africanews.com/feed/rss" +121,full,africa,"BBC Afrique",OK,2026-02-26,"OK","https://www.bbc.com/afrique/index.xml" +122,full,africa,"Premium Times",OK,2026-02-26,"OK","https://www.premiumtimesng.com/feed" +123,full,africa,"Vanguard Nigeria",OK,2026-02-26,"OK","https://www.vanguardngr.com/feed/" +124,full,africa,"Channels TV",OK,2026-02-26,"OK","https://www.channelstv.com/feed/" +125,full,africa,"Daily Trust",OK,2026-02-26,"OK","https://dailytrust.com/feed/" +126,full,africa,"ThisDay",OK,2026-02-26,"OK","https://www.thisdaylive.com/feed" +127,full,latam,"Latin America",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Brazil+OR+Mexico+OR+Argentina+OR+Venezuela+OR+Colombia)+when:2d&hl=en-US&gl=US&ceid=US:en" +128,full,latam,"BBC Latin America",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/news/world/latin_america/rss.xml" +129,full,latam,"Reuters LatAm",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+(Brazil+OR+Mexico+OR+Argentina)+when:3d&hl=en-US&gl=US&ceid=US:en" +130,full,latam,"Guardian Americas",OK,2026-02-26,"OK","https://www.theguardian.com/world/americas/rss" +131,full,latam,"Clarín",OK,2026-02-26,"OK","https://www.clarin.com/rss/lo-ultimo/" +132,full,latam,"O Globo",DEAD,,"HTTP 404","https://oglobo.globo.com/rss/top_noticias/" +133,full,latam,"Folha de S.Paulo",DEAD,,"fetch failed","https://feeds.folha.uol.com.br/emcimadahora/rss091.xml" +134,full,latam,"Brasil Paralelo",OK,2026-02-26,"OK","https://www.brasilparalelo.com.br/noticias/rss.xml" +135,full,latam,"El Tiempo",OK,2026-02-26,"OK","https://www.eltiempo.com/rss/mundo_latinoamerica.xml" +136,full,latam,"El Universal",DEAD,,"HTTP 404","https://www.eluniversal.com.mx/rss.xml" +137,full,latam,"La Silla Vacía",OK,2026-02-26,"OK","https://www.lasillavacia.com/rss" +138,full,latam,"Mexico News Daily",OK,2026-02-26,"OK","https://mexiconewsdaily.com/feed/" +139,full,latam,"Animal Político",DEAD,,"HTTP 404","https://animalpolitico.com/feed/" +140,full,latam,"Proceso",DEAD,,"HTTP 404","https://www.proceso.com.mx/feed/" +141,full,latam,"Milenio",DEAD,,"HTTP 404","https://www.milenio.com/rss" +142,full,latam,"Mexico Security",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Mexico+cartel+OR+Mexico+violence+OR+Mexico+troops+OR+narco+Mexico)+when:2d&hl=en-US&gl=US&ceid=US:en" +143,full,latam,"AP Mexico",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:apnews.com+Mexico+when:3d&hl=en-US&gl=US&ceid=US:en" +144,full,latam,"InSight Crime",OK,2026-02-26,"OK","https://insightcrime.org/feed/" +145,full,latam,"France 24 LatAm",OK,2026-02-26,"OK","https://www.france24.com/en/americas/rss" +146,full,asia,"Asia News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(China+OR+Japan+OR+Korea+OR+India+OR+ASEAN)+when:2d&hl=en-US&gl=US&ceid=US:en" +147,full,asia,"BBC Asia",OK,2026-02-26,"OK","https://feeds.bbci.co.uk/news/world/asia/rss.xml" +148,full,asia,"The Diplomat",OK,2026-02-26,"OK","https://thediplomat.com/feed/" +149,full,asia,"South China Morning Post",OK,2026-02-26,"OK","https://www.scmp.com/rss/91/feed/" +150,full,asia,"Reuters Asia",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+(China+OR+Japan+OR+Taiwan+OR+Korea)+when:3d&hl=en-US&gl=US&ceid=US:en" +151,full,asia,"Xinhua",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:xinhuanet.com+OR+Xinhua+when:1d&hl=en-US&gl=US&ceid=US:en" +152,full,asia,"Japan Today",OK,2026-02-26,"OK","https://japantoday.com/feed/atom" +153,full,asia,"Nikkei Asia",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:asia.nikkei.com+when:3d&hl=en-US&gl=US&ceid=US:en" +154,full,asia,"Asahi Shimbun",OK,2026-02-26,"OK","https://www.asahi.com/rss/asahi/newsheadlines.rdf" +155,full,asia,"The Hindu",OK,2026-02-26,"OK","https://www.thehindu.com/news/national/feeder/default.rss" +156,full,asia,"Indian Express",OK,2026-02-26,"OK","https://indianexpress.com/section/india/feed/" +157,full,asia,"India News Network",EMPTY,,"No dates found","https://www.indianewsnetwork.com/rss.en.diplomacy.xml" +158,full,asia,"CNA",OK,2026-02-26,"OK","https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml" +159,full,asia,"MIIT (China)",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:miit.gov.cn+when:7d&hl=zh-CN&gl=CN&ceid=CN:zh-Hans" +160,full,asia,"MOFCOM (China)",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:mofcom.gov.cn+when:7d&hl=zh-CN&gl=CN&ceid=CN:zh-Hans" +161,full,asia,"Bangkok Post",DEAD,,"HTTP 451","https://www.bangkokpost.com/rss" +162,full,asia,"Thai PBS",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:thaipbsworld.com+when:2d&hl=th&gl=TH&ceid=TH:th" +163,full,asia,"VnExpress",EMPTY,,"No dates found","https://vnexpress.net/rss" +164,full,asia,"Tuoi Tre News",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:tuoitrenews.vn+when:2d&hl=vi&gl=VN&ceid=VN:vi" +165,full,asia,"ABC News Australia",OK,2026-02-26,"OK","https://www.abc.net.au/news/feed/2942460/rss.xml" +166,full,asia,"Guardian Australia",OK,2026-02-26,"OK","https://www.theguardian.com/australia-news/rss" +167,full,asia,"Island Times (Palau)",OK,2026-02-24,"OK","https://islandtimes.org/feed/" +168,full,energy,"Oil & Gas",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(oil+price+OR+OPEC+OR+""natural+gas""+OR+pipeline+OR+LNG)+when:2d&hl=en-US&gl=US&ceid=US:en" +169,full,energy,"Nuclear Energy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""nuclear+energy""+OR+""nuclear+power""+OR+uranium+OR+IAEA)+when:3d&hl=en-US&gl=US&ceid=US:en" +170,full,energy,"Reuters Energy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+(oil+OR+gas+OR+energy+OR+OPEC)+when:3d&hl=en-US&gl=US&ceid=US:en" +171,full,energy,"Mining & Resources",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(lithium+OR+""rare+earth""+OR+cobalt+OR+mining)+when:3d&hl=en-US&gl=US&ceid=US:en" +172,tech,tech,"TechCrunch",OK,2026-02-26,"OK","https://techcrunch.com/feed/" +173,tech,tech,"ZDNet",OK,2026-02-26,"OK","https://www.zdnet.com/news/rss.xml" +174,tech,tech,"TechMeme",OK,2026-02-26,"OK","https://www.techmeme.com/feed.xml" +175,tech,tech,"Engadget",OK,2026-02-26,"OK","https://www.engadget.com/rss.xml" +176,tech,tech,"Fast Company",OK,2026-02-26,"OK","https://feeds.feedburner.com/fastcompany/headlines" +177,tech,ai,"AI News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(OpenAI+OR+Anthropic+OR+Google+AI+OR+""large+language+model""+OR+ChatGPT+OR+Claude+OR+""AI+model"")+when:2d&hl=en-US&gl=US&ceid=US:en" +178,tech,ai,"MIT Tech Review AI",OK,2026-02-26,"OK","https://www.technologyreview.com/topic/artificial-intelligence/feed" +179,tech,ai,"MIT Research",OK,2026-02-26,"OK","https://news.mit.edu/rss/research" +180,tech,ai,"ArXiv ML",OK,2026-02-26,"OK","https://export.arxiv.org/rss/cs.LG" +181,tech,ai,"AI Weekly",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""artificial+intelligence""+OR+""machine+learning""+when:3d&hl=en-US&gl=US&ceid=US:en" +182,tech,ai,"Anthropic News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Anthropic+Claude+AI+when:7d&hl=en-US&gl=US&ceid=US:en" +183,tech,ai,"OpenAI News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=OpenAI+ChatGPT+GPT-4+when:7d&hl=en-US&gl=US&ceid=US:en" +184,tech,startups,"TechCrunch Startups",OK,2026-02-26,"OK","https://techcrunch.com/category/startups/feed/" +185,tech,startups,"VentureBeat",OK,2026-02-26,"OK","https://venturebeat.com/feed/" +186,tech,startups,"Crunchbase News",OK,2026-02-26,"OK","https://news.crunchbase.com/feed/" +187,tech,startups,"SaaStr",OK,2026-02-26,"OK","https://www.saastr.com/feed/" +188,tech,startups,"AngelList News",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:angellist.com+OR+""AngelList""+funding+when:7d&hl=en-US&gl=US&ceid=US:en" +189,tech,startups,"TechCrunch Venture",OK,2026-02-26,"OK","https://techcrunch.com/category/venture/feed/" +190,tech,startups,"The Information",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:theinformation.com+startup+OR+funding+when:3d&hl=en-US&gl=US&ceid=US:en" +191,tech,startups,"Fortune Term Sheet",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""Term+Sheet""+venture+capital+OR+startup+when:7d&hl=en-US&gl=US&ceid=US:en" +192,tech,startups,"PitchBook News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:pitchbook.com+when:7d&hl=en-US&gl=US&ceid=US:en" +193,tech,startups,"CB Insights",OK,2026-02-18,"OK","https://www.cbinsights.com/research/feed/" +194,tech,vcblogs,"Y Combinator Blog",OK,2026-02-05,"OK","https://www.ycombinator.com/blog/rss/" +195,tech,vcblogs,"a16z Blog",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:a16z.com+OR+""Andreessen+Horowitz""+blog+when:14d&hl=en-US&gl=US&ceid=US:en" +196,tech,vcblogs,"Sequoia Blog",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:sequoiacap.com+when:7d&hl=en-US&gl=US&ceid=US:en" +197,tech,vcblogs,"Paul Graham Essays",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=""Paul+Graham""+essay+OR+blog+when:30d&hl=en-US&gl=US&ceid=US:en" +198,tech,vcblogs,"VC Insights",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""venture+capital""+insights+OR+""VC+trends""+OR+""startup+advice"")+when:7d&hl=en-US&gl=US&ceid=US:en" +199,tech,vcblogs,"Lenny\",OK,2026-02-26,"OK","https://www.lennysnewsletter.com/feed" +200,tech,vcblogs,"Stratechery",OK,2026-02-26,"OK","https://stratechery.com/feed/" +201,tech,regionalStartups,"EU Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:eu-startups.com+when:7d&hl=en-US&gl=US&ceid=US:en" +202,tech,regionalStartups,"Tech.eu",OK,2026-02-26,"OK","https://tech.eu/feed/" +203,tech,regionalStartups,"Sifted (Europe)",OK,2026-02-26,"OK","https://sifted.eu/feed" +204,tech,regionalStartups,"The Next Web",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:thenextweb.com+when:7d&hl=en-US&gl=US&ceid=US:en" +205,tech,regionalStartups,"Tech in Asia",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:techinasia.com+when:7d&hl=en-US&gl=US&ceid=US:en" +206,tech,regionalStartups,"KrASIA",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:kr-asia.com+OR+KrASIA+when:7d&hl=en-US&gl=US&ceid=US:en" +207,tech,regionalStartups,"SEA Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Singapore+OR+Indonesia+OR+Vietnam+OR+Thailand+OR+Malaysia)+startup+funding+when:7d&hl=en-US&gl=US&ceid=US:en" +208,tech,regionalStartups,"Asia VC News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Southeast+Asia""+OR+ASEAN)+venture+capital+OR+funding+when:7d&hl=en-US&gl=US&ceid=US:en" +209,tech,regionalStartups,"China Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=China+startup+funding+OR+""Chinese+startup""+when:7d&hl=en-US&gl=US&ceid=US:en" +210,tech,regionalStartups,"36Kr English",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:36kr.com+OR+""36Kr""+startup+china+when:7d&hl=en-US&gl=US&ceid=US:en" +211,tech,regionalStartups,"China Tech Giants",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Alibaba+OR+Tencent+OR+ByteDance+OR+Baidu+OR+JD.com+OR+Xiaomi+OR+Huawei)+when:3d&hl=en-US&gl=US&ceid=US:en" +212,tech,regionalStartups,"Japan Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Japan+startup+funding+OR+""Japanese+startup""+when:7d&hl=en-US&gl=US&ceid=US:en" +213,tech,regionalStartups,"Japan Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Japan+startup+OR+Japan+tech+OR+SoftBank+OR+Rakuten+OR+Sony)+funding+when:7d&hl=en-US&gl=US&ceid=US:en" +214,tech,regionalStartups,"Nikkei Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:asia.nikkei.com+technology+when:3d&hl=en-US&gl=US&ceid=US:en" +215,tech,regionalStartups,"Korea Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Korea+startup+OR+Korean+tech+OR+Samsung+OR+Kakao+OR+Naver+OR+Coupang)+when:7d&hl=en-US&gl=US&ceid=US:en" +216,tech,regionalStartups,"Korea Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Korea+startup+funding+OR+""Korean+unicorn""+when:7d&hl=en-US&gl=US&ceid=US:en" +217,tech,regionalStartups,"Inc42 (India)",OK,2026-02-26,"OK","https://inc42.com/feed/" +218,tech,regionalStartups,"YourStory",OK,2026-02-26,"OK","https://yourstory.com/feed" +219,tech,regionalStartups,"India Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=India+startup+funding+OR+""Indian+startup""+when:7d&hl=en-US&gl=US&ceid=US:en" +220,tech,regionalStartups,"India Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Flipkart+OR+Razorpay+OR+Zerodha+OR+Zomato+OR+Paytm+OR+PhonePe)+when:7d&hl=en-US&gl=US&ceid=US:en" +221,tech,regionalStartups,"SEA Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Grab+OR+GoTo+OR+Sea+Limited+OR+Shopee+OR+Tokopedia)+when:7d&hl=en-US&gl=US&ceid=US:en" +222,tech,regionalStartups,"Vietnam Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Vietnam+startup+OR+Vietnam+tech+when:7d&hl=en-US&gl=US&ceid=US:en" +223,tech,regionalStartups,"Indonesia Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Indonesia+startup+OR+Indonesia+tech+when:7d&hl=en-US&gl=US&ceid=US:en" +224,tech,regionalStartups,"Taiwan Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Taiwan+startup+OR+TSMC+OR+MediaTek+OR+Foxconn)+when:7d&hl=en-US&gl=US&ceid=US:en" +225,tech,regionalStartups,"LAVCA (LATAM)",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=site:lavca.org+when:7d&hl=en-US&gl=US&ceid=US:en" +226,tech,regionalStartups,"LATAM Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Latin+America""+startup+OR+LATAM+funding)+when:7d&hl=en-US&gl=US&ceid=US:en" +227,tech,regionalStartups,"Startups LATAM",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(startup+Brazil+OR+startup+Mexico+OR+startup+Argentina+OR+startup+Colombia+OR+startup+Chile)+when:7d&hl=en-US&gl=US&ceid=US:en" +228,tech,regionalStartups,"Brazil Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Nubank+OR+iFood+OR+Mercado+Libre+OR+Rappi+OR+VTEX)+when:7d&hl=en-US&gl=US&ceid=US:en" +229,tech,regionalStartups,"FinTech LATAM",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=fintech+(Brazil+OR+Mexico+OR+Argentina+OR+""Latin+America"")+when:7d&hl=en-US&gl=US&ceid=US:en" +230,tech,regionalStartups,"TechCabal (Africa)",OK,2026-02-26,"OK","https://techcabal.com/feed/" +231,tech,regionalStartups,"Disrupt Africa",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:disrupt-africa.com+when:7d&hl=en-US&gl=US&ceid=US:en" +232,tech,regionalStartups,"Africa Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Africa+startup+funding+OR+""African+startup""+when:7d&hl=en-US&gl=US&ceid=US:en" +233,tech,regionalStartups,"Africa Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Flutterwave+OR+Paystack+OR+Jumia+OR+Andela+OR+""Africa+startup"")+when:7d&hl=en-US&gl=US&ceid=US:en" +234,tech,regionalStartups,"MENA Startups",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(MENA+startup+OR+""Middle+East""+funding+OR+Gulf+startup)+when:7d&hl=en-US&gl=US&ceid=US:en" +235,tech,regionalStartups,"MENA Tech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(UAE+startup+OR+Saudi+tech+OR+Dubai+startup+OR+NEOM+tech)+when:7d&hl=en-US&gl=US&ceid=US:en" +236,tech,github,"GitHub Blog",OK,2026-02-24,"OK","https://github.blog/feed/" +237,tech,github,"GitHub Trending",EMPTY,,"No dates found","https://mshibanami.github.io/GitHubTrendingRSS/daily/all.xml" +238,tech,github,"Show HN",OK,2026-02-26,"OK","https://hnrss.org/show" +239,tech,github,"YC Launches",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Y+Combinator""+OR+""YC+launch""+OR+""YC+W25""+OR+""YC+S25"")+when:7d&hl=en-US&gl=US&ceid=US:en" +240,tech,github,"Dev Events",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""developer+conference""+OR+""tech+summit""+OR+""devcon""+OR+""developer+event"")+when:7d&hl=en-US&gl=US&ceid=US:en" +241,tech,github,"Open Source News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""open+source""+project+release+OR+launch+when:3d&hl=en-US&gl=US&ceid=US:en" +242,tech,ipo,"IPO News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(IPO+OR+""initial+public+offering""+OR+SPAC)+tech+when:7d&hl=en-US&gl=US&ceid=US:en" +243,tech,ipo,"Renaissance IPO",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:renaissancecapital.com+IPO+when:14d&hl=en-US&gl=US&ceid=US:en" +244,tech,ipo,"Tech IPO News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=tech+IPO+OR+""tech+company""+IPO+when:7d&hl=en-US&gl=US&ceid=US:en" +245,tech,funding,"SEC Filings",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=(S-1+OR+""IPO+filing""+OR+""SEC+filing"")+startup+when:7d&hl=en-US&gl=US&ceid=US:en" +246,tech,funding,"VC News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Series+A""+OR+""Series+B""+OR+""Series+C""+OR+""funding+round""+OR+""venture+capital"")+when:7d&hl=en-US&gl=US&ceid=US:en" +247,tech,funding,"Seed & Pre-Seed",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""seed+round""+OR+""pre-seed""+OR+""angel+round""+OR+""seed+funding"")+when:7d&hl=en-US&gl=US&ceid=US:en" +248,tech,funding,"Startup Funding",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""startup+funding""+OR+""raised+funding""+OR+""raised+$""+OR+""funding+announced"")+when:7d&hl=en-US&gl=US&ceid=US:en" +249,tech,producthunt,"Product Hunt",OK,2026-02-26,"OK","https://www.producthunt.com/feed" +250,tech,outages,"AWS Status",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=AWS+outage+OR+""Amazon+Web+Services""+down+when:1d&hl=en-US&gl=US&ceid=US:en" +251,tech,outages,"Cloud Outages",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Azure+OR+GCP+OR+Cloudflare+OR+Slack+OR+GitHub)+outage+OR+down+when:1d&hl=en-US&gl=US&ceid=US:en" +252,tech,security,"Krebs Security",OK,2026-02-20,"OK","https://krebsonsecurity.com/feed/" +253,tech,security,"The Hacker News",OK,2026-02-26,"OK","https://feeds.feedburner.com/TheHackersNews" +254,tech,security,"Dark Reading",OK,2026-02-26,"OK","https://www.darkreading.com/rss.xml" +255,tech,security,"Schneier",OK,2026-02-26,"OK","https://www.schneier.com/feed/" +256,tech,policy,"Politico Tech",OK,2026-02-26,"OK","https://rss.politico.com/technology.xml" +257,tech,policy,"AI Regulation",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=AI+regulation+OR+""artificial+intelligence""+law+OR+policy+when:7d&hl=en-US&gl=US&ceid=US:en" +258,tech,policy,"Tech Antitrust",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=tech+antitrust+OR+FTC+Google+OR+FTC+Apple+OR+FTC+Amazon+when:7d&hl=en-US&gl=US&ceid=US:en" +259,tech,policy,"EFF News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:eff.org+OR+""Electronic+Frontier+Foundation""+when:14d&hl=en-US&gl=US&ceid=US:en" +260,tech,policy,"EU Digital Policy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Digital+Services+Act""+OR+""Digital+Markets+Act""+OR+""EU+AI+Act""+OR+""GDPR"")+when:7d&hl=en-US&gl=US&ceid=US:en" +261,tech,policy,"Euractiv Digital",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:euractiv.com+digital+OR+tech+when:7d&hl=en-US&gl=US&ceid=US:en" +262,tech,policy,"EU Commission Digital",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:ec.europa.eu+digital+OR+technology+when:14d&hl=en-US&gl=US&ceid=US:en" +263,tech,policy,"China Tech Policy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(China+tech+regulation+OR+China+AI+policy+OR+MIIT+technology)+when:7d&hl=en-US&gl=US&ceid=US:en" +264,tech,policy,"UK Tech Policy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(UK+AI+safety+OR+""Online+Safety+Bill""+OR+UK+tech+regulation)+when:7d&hl=en-US&gl=US&ceid=US:en" +265,tech,policy,"India Tech Policy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(India+tech+regulation+OR+India+data+protection+OR+India+AI+policy)+when:7d&hl=en-US&gl=US&ceid=US:en" +266,tech,thinktanks,"Brookings Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:brookings.edu+technology+OR+AI+when:14d&hl=en-US&gl=US&ceid=US:en" +267,tech,thinktanks,"CSIS Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:csis.org+technology+OR+AI+when:14d&hl=en-US&gl=US&ceid=US:en" +268,tech,thinktanks,"MIT Tech Policy",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:techpolicypress.org+when:14d&hl=en-US&gl=US&ceid=US:en" +269,tech,thinktanks,"Stanford HAI",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:hai.stanford.edu+when:14d&hl=en-US&gl=US&ceid=US:en" +270,tech,thinktanks,"AI Now Institute",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:ainowinstitute.org+when:14d&hl=en-US&gl=US&ceid=US:en" +271,tech,thinktanks,"OECD Digital",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:oecd.org+digital+OR+AI+when:14d&hl=en-US&gl=US&ceid=US:en" +272,tech,thinktanks,"EU Tech Policy",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""EU+tech+policy""+OR+""European+digital""+OR+Bruegel+tech)+when:14d&hl=en-US&gl=US&ceid=US:en" +273,tech,thinktanks,"Chatham House Tech",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:chathamhouse.org+technology+OR+AI+when:14d&hl=en-US&gl=US&ceid=US:en" +274,tech,thinktanks,"ISEAS (Singapore)",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:iseas.edu.sg+technology+when:14d&hl=en-US&gl=US&ceid=US:en" +275,tech,thinktanks,"ORF Tech (India)",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(India+tech+policy+OR+ORF+technology+OR+""Observer+Research+Foundation""+tech)+when:14d&hl=en-US&gl=US&ceid=US:en" +276,tech,thinktanks,"RIETI (Japan)",OK,2026-02-19,"OK","https://news.google.com/rss/search?q=site:rieti.go.jp+technology+when:30d&hl=en-US&gl=US&ceid=US:en" +277,tech,thinktanks,"Asia Pacific Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Asia+Pacific""+tech+policy+OR+""Lowy+Institute""+technology)+when:14d&hl=en-US&gl=US&ceid=US:en" +278,tech,thinktanks,"China Tech Analysis",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""China+tech+strategy""+OR+""Chinese+AI""+OR+""China+semiconductor"")+analysis+when:7d&hl=en-US&gl=US&ceid=US:en" +279,tech,thinktanks,"DigiChina",EMPTY,,"No dates found","https://news.google.com/rss/search?q=site:digichina.stanford.edu+when:14d&hl=en-US&gl=US&ceid=US:en" +280,tech,finance,"CNBC Tech",OK,2026-02-26,"OK","https://www.cnbc.com/id/19854910/device/rss/rss.html" +281,tech,finance,"MarketWatch Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:marketwatch.com+technology+markets+when:2d&hl=en-US&gl=US&ceid=US:en" +282,tech,finance,"Yahoo Finance",OK,2026-02-25,"OK","https://finance.yahoo.com/rss/topstories" +283,tech,finance,"Seeking Alpha Tech",OK,2026-02-26,"OK","https://seekingalpha.com/market_currents.xml" +284,tech,hardware,"Tom's Hardware",OK,2026-02-26,"OK","https://www.tomshardware.com/feeds/all" +285,tech,hardware,"SemiAnalysis",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:semianalysis.com+when:7d&hl=en-US&gl=US&ceid=US:en" +286,tech,hardware,"Semiconductor News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=semiconductor+OR+chip+OR+TSMC+OR+NVIDIA+OR+Intel+when:3d&hl=en-US&gl=US&ceid=US:en" +287,tech,cloud,"InfoQ",OK,2026-02-26,"OK","https://feed.infoq.com/" +288,tech,cloud,"The New Stack",OK,2026-02-26,"OK","https://thenewstack.io/feed/" +289,tech,cloud,"DevOps.com",OK,2026-02-26,"OK","https://devops.com/feed/" +290,tech,dev,"Dev.to",OK,2026-02-26,"OK","https://dev.to/feed" +291,tech,dev,"Lobsters",OK,2026-02-26,"OK","https://lobste.rs/rss" +292,tech,dev,"Changelog",OK,2026-02-23,"OK","https://changelog.com/feed" +293,tech,layoffs,"Layoffs.fyi",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=tech+layoffs+when:7d&hl=en-US&gl=US&ceid=US:en" +294,tech,unicorns,"Unicorn News",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=(""unicorn+startup""+OR+""unicorn+valuation""+OR+""$1+billion+valuation"")+when:7d&hl=en-US&gl=US&ceid=US:en" +295,tech,unicorns,"CB Insights Unicorn",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=site:cbinsights.com+unicorn+when:14d&hl=en-US&gl=US&ceid=US:en" +296,tech,unicorns,"Decacorn News",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=(""decacorn""+OR+""$10+billion+valuation""+OR+""$10B+valuation"")+startup+when:14d&hl=en-US&gl=US&ceid=US:en" +297,tech,unicorns,"New Unicorns",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""becomes+unicorn""+OR+""joins+unicorn""+OR+""reaches+unicorn""+OR+""achieved+unicorn"")+when:14d&hl=en-US&gl=US&ceid=US:en" +298,tech,accelerators,"Techstars News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=Techstars+accelerator+when:14d&hl=en-US&gl=US&ceid=US:en" +299,tech,accelerators,"500 Global News",OK,2026-02-19,"OK","https://news.google.com/rss/search?q=""500+Global""+OR+""500+Startups""+accelerator+when:14d&hl=en-US&gl=US&ceid=US:en" +300,tech,accelerators,"Demo Day News",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=(""demo+day""+OR+""YC+batch""+OR+""accelerator+batch"")+startup+when:7d&hl=en-US&gl=US&ceid=US:en" +301,tech,accelerators,"Startup School",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""Startup+School""+OR+""YC+Startup+School""+when:14d&hl=en-US&gl=US&ceid=US:en" +302,tech,podcasts,"Acquired Episodes",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=""Acquired+podcast""+episode+when:14d&hl=en-US&gl=US&ceid=US:en" +303,tech,podcasts,"All-In Podcast",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=""All-In+podcast""+(Chamath+OR+Sacks+OR+Friedberg)+when:7d&hl=en-US&gl=US&ceid=US:en" +304,tech,podcasts,"a16z Insights",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""a16z""+OR+""Andreessen+Horowitz"")+podcast+OR+interview+when:14d&hl=en-US&gl=US&ceid=US:en" +305,tech,podcasts,"TWIST Episodes",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=""This+Week+in+Startups""+Jason+Calacanis+when:14d&hl=en-US&gl=US&ceid=US:en" +306,tech,podcasts,"20VC Episodes",EMPTY,,"No dates found","https://news.google.com/rss/search?q=""20+Minute+VC""+Harry+Stebbings+when:14d&hl=en-US&gl=US&ceid=US:en" +307,tech,podcasts,"Lex Fridman Tech",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=(""Lex+Fridman""+interview)+(AI+OR+tech+OR+startup+OR+CEO)+when:7d&hl=en-US&gl=US&ceid=US:en" +308,tech,podcasts,"Verge Shows",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Vergecast""+OR+""Decoder+podcast""+Verge)+when:14d&hl=en-US&gl=US&ceid=US:en" +309,tech,podcasts,"Hard Fork (NYT)",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""Hard+Fork""+podcast+NYT+when:14d&hl=en-US&gl=US&ceid=US:en" +310,tech,podcasts,"Pivot Podcast",EMPTY,,"No dates found","https://news.google.com/rss/search?q=""Pivot+podcast""+(Kara+Swisher+OR+Scott+Galloway)+when:14d&hl=en-US&gl=US&ceid=US:en" +311,tech,podcasts,"Tech Newsletters",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=(""Benedict+Evans""+OR+""Pragmatic+Engineer""+OR+Stratechery)+tech+when:14d&hl=en-US&gl=US&ceid=US:en" +312,tech,podcasts,"AI Podcasts",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""AI+podcast""+OR+""artificial+intelligence+podcast"")+episode+when:14d&hl=en-US&gl=US&ceid=US:en" +313,tech,podcasts,"AI Interviews",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(NVIDIA+OR+OpenAI+OR+Anthropic+OR+DeepMind)+interview+OR+podcast+when:14d&hl=en-US&gl=US&ceid=US:en" +314,tech,podcasts,"How I Built This",OK,2026-02-23,"OK","https://news.google.com/rss/search?q=""How+I+Built+This""+Guy+Raz+when:14d&hl=en-US&gl=US&ceid=US:en" +315,tech,podcasts,"Startup Podcasts",EMPTY,,"No dates found","https://news.google.com/rss/search?q=(""Masters+of+Scale""+OR+""The+Pitch+podcast""+OR+""startup+podcast"")+episode+when:14d&hl=en-US&gl=US&ceid=US:en" +316,finance,markets,"Seeking Alpha",OK,2026-02-26,"OK","https://seekingalpha.com/market_currents.xml" +317,finance,markets,"Reuters Markets",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:reuters.com+markets+stocks+when:1d&hl=en-US&gl=US&ceid=US:en" +318,finance,markets,"Bloomberg Markets",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:bloomberg.com+markets+when:1d&hl=en-US&gl=US&ceid=US:en" +319,finance,markets,"Investing.com News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:investing.com+markets+when:1d&hl=en-US&gl=US&ceid=US:en" +320,finance,forex,"Forex News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""forex""+OR+""currency""+OR+""FX+market"")+trading+when:1d&hl=en-US&gl=US&ceid=US:en" +321,finance,forex,"Dollar Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""dollar+index""+OR+DXY+OR+""US+dollar""+OR+""euro+dollar"")+when:2d&hl=en-US&gl=US&ceid=US:en" +322,finance,forex,"Central Bank Rates",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""central+bank""+OR+""interest+rate""+OR+""rate+decision""+OR+""monetary+policy"")+when:2d&hl=en-US&gl=US&ceid=US:en" +323,finance,bonds,"Bond Market",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""bond+market""+OR+""treasury+yields""+OR+""bond+yields""+OR+""fixed+income"")+when:2d&hl=en-US&gl=US&ceid=US:en" +324,finance,bonds,"Treasury Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""US+Treasury""+OR+""Treasury+auction""+OR+""10-year+yield""+OR+""2-year+yield"")+when:2d&hl=en-US&gl=US&ceid=US:en" +325,finance,bonds,"Corporate Bonds",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""corporate+bond""+OR+""high+yield""+OR+""investment+grade""+OR+""credit+spread"")+when:3d&hl=en-US&gl=US&ceid=US:en" +326,finance,commodities,"Oil & Gas",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(oil+price+OR+OPEC+OR+""natural+gas""+OR+""crude+oil""+OR+WTI+OR+Brent)+when:1d&hl=en-US&gl=US&ceid=US:en" +327,finance,commodities,"Gold & Metals",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(gold+price+OR+silver+price+OR+copper+OR+platinum+OR+""precious+metals"")+when:2d&hl=en-US&gl=US&ceid=US:en" +328,finance,commodities,"Agriculture",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(wheat+OR+corn+OR+soybeans+OR+coffee+OR+sugar)+price+OR+commodity+when:3d&hl=en-US&gl=US&ceid=US:en" +329,finance,commodities,"Commodity Trading",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""commodity+trading""+OR+""futures+market""+OR+CME+OR+NYMEX+OR+COMEX)+when:2d&hl=en-US&gl=US&ceid=US:en" +330,finance,crypto,"CoinDesk",OK,2026-02-26,"OK","https://www.coindesk.com/arc/outboundfeeds/rss/" +331,finance,crypto,"Cointelegraph",OK,2026-02-26,"OK","https://cointelegraph.com/rss" +332,finance,crypto,"The Block",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:theblock.co+when:1d&hl=en-US&gl=US&ceid=US:en" +333,finance,crypto,"Crypto News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(bitcoin+OR+ethereum+OR+crypto+OR+""digital+assets"")+when:1d&hl=en-US&gl=US&ceid=US:en" +334,finance,crypto,"DeFi News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(DeFi+OR+""decentralized+finance""+OR+DEX+OR+""yield+farming"")+when:3d&hl=en-US&gl=US&ceid=US:en" +335,finance,centralbanks,"ECB Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""European+Central+Bank""+OR+ECB+OR+Lagarde)+monetary+policy+when:3d&hl=en-US&gl=US&ceid=US:en" +336,finance,centralbanks,"BoJ Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Bank+of+Japan""+OR+BoJ)+monetary+policy+when:3d&hl=en-US&gl=US&ceid=US:en" +337,finance,centralbanks,"BoE Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Bank+of+England""+OR+BoE)+monetary+policy+when:3d&hl=en-US&gl=US&ceid=US:en" +338,finance,centralbanks,"PBoC Watch",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""People%27s+Bank+of+China""+OR+PBoC+OR+PBOC)+when:7d&hl=en-US&gl=US&ceid=US:en" +339,finance,centralbanks,"Global Central Banks",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""rate+hike""+OR+""rate+cut""+OR+""interest+rate+decision"")+central+bank+when:3d&hl=en-US&gl=US&ceid=US:en" +340,finance,economic,"Economic Data",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(CPI+OR+inflation+OR+GDP+OR+""jobs+report""+OR+""nonfarm+payrolls""+OR+PMI)+when:2d&hl=en-US&gl=US&ceid=US:en" +341,finance,economic,"Trade & Tariffs",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(tariff+OR+""trade+war""+OR+""trade+deficit""+OR+sanctions)+when:2d&hl=en-US&gl=US&ceid=US:en" +342,finance,economic,"Housing Market",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""housing+market""+OR+""home+prices""+OR+""mortgage+rates""+OR+REIT)+when:3d&hl=en-US&gl=US&ceid=US:en" +343,finance,ipo,"IPO News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(IPO+OR+""initial+public+offering""+OR+SPAC+OR+""direct+listing"")+when:3d&hl=en-US&gl=US&ceid=US:en" +344,finance,ipo,"Earnings Reports",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""earnings+report""+OR+""quarterly+earnings""+OR+""revenue+beat""+OR+""earnings+miss"")+when:2d&hl=en-US&gl=US&ceid=US:en" +345,finance,ipo,"M&A News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""merger""+OR+""acquisition""+OR+""takeover+bid""+OR+""buyout"")+billion+when:3d&hl=en-US&gl=US&ceid=US:en" +346,finance,derivatives,"Options Market",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""options+market""+OR+""options+trading""+OR+""put+call+ratio""+OR+VIX)+when:2d&hl=en-US&gl=US&ceid=US:en" +347,finance,derivatives,"Futures Trading",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""futures+trading""+OR+""S%26P+500+futures""+OR+""Nasdaq+futures"")+when:1d&hl=en-US&gl=US&ceid=US:en" +348,finance,fintech,"Fintech News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(fintech+OR+""payment+technology""+OR+""neobank""+OR+""digital+banking"")+when:3d&hl=en-US&gl=US&ceid=US:en" +349,finance,fintech,"Trading Tech",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""algorithmic+trading""+OR+""trading+platform""+OR+""quantitative+finance"")+when:7d&hl=en-US&gl=US&ceid=US:en" +350,finance,fintech,"Blockchain Finance",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""blockchain+finance""+OR+""tokenization""+OR+""digital+securities""+OR+CBDC)+when:7d&hl=en-US&gl=US&ceid=US:en" +351,finance,regulation,"Financial Regulation",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(SEC+OR+CFTC+OR+FINRA+OR+FCA)+regulation+OR+enforcement+when:3d&hl=en-US&gl=US&ceid=US:en" +352,finance,regulation,"Banking Rules",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(Basel+OR+""capital+requirements""+OR+""banking+regulation"")+when:7d&hl=en-US&gl=US&ceid=US:en" +353,finance,regulation,"Crypto Regulation",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(crypto+regulation+OR+""digital+asset""+regulation+OR+""stablecoin""+regulation)+when:7d&hl=en-US&gl=US&ceid=US:en" +354,finance,institutional,"Hedge Fund News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""hedge+fund""+OR+""Bridgewater""+OR+""Citadel""+OR+""Renaissance"")+when:7d&hl=en-US&gl=US&ceid=US:en" +355,finance,institutional,"Private Equity",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""private+equity""+OR+Blackstone+OR+KKR+OR+Apollo+OR+Carlyle)+when:3d&hl=en-US&gl=US&ceid=US:en" +356,finance,institutional,"Sovereign Wealth",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""sovereign+wealth+fund""+OR+""pension+fund""+OR+""institutional+investor"")+when:7d&hl=en-US&gl=US&ceid=US:en" +357,finance,analysis,"Market Outlook",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""market+outlook""+OR+""stock+market+forecast""+OR+""bull+market""+OR+""bear+market"")+when:3d&hl=en-US&gl=US&ceid=US:en" +358,finance,analysis,"Risk & Volatility",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(VIX+OR+""market+volatility""+OR+""risk+off""+OR+""market+correction"")+when:3d&hl=en-US&gl=US&ceid=US:en" +359,finance,analysis,"Bank Research",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Goldman+Sachs""+OR+""JPMorgan""+OR+""Morgan+Stanley"")+forecast+OR+outlook+when:3d&hl=en-US&gl=US&ceid=US:en" +360,finance,gccNews,"Arabian Business",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:arabianbusiness.com+(Saudi+Arabia+OR+UAE+OR+GCC)+when:7d&hl=en-US&gl=US&ceid=US:en" +361,finance,gccNews,"The National",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:thenationalnews.com+(Abu+Dhabi+OR+UAE+OR+Saudi)+when:7d&hl=en-US&gl=US&ceid=US:en" +362,finance,gccNews,"Arab News",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:arabnews.com+(Saudi+Arabia+OR+investment+OR+infrastructure)+when:7d&hl=en-US&gl=US&ceid=US:en" +363,finance,gccNews,"Gulf FDI",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(PIF+OR+""DP+World""+OR+Mubadala+OR+ADNOC+OR+Masdar+OR+""ACWA+Power"")+infrastructure+when:7d&hl=en-US&gl=US&ceid=US:en" +364,finance,gccNews,"Gulf Investments",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=(""Saudi+Arabia""+OR+""UAE""+OR+""Abu+Dhabi"")+investment+infrastructure+when:7d&hl=en-US&gl=US&ceid=US:en" +365,finance,gccNews,"Vision 2030",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=""Vision+2030""+(project+OR+investment+OR+announced)+when:14d&hl=en-US&gl=US&ceid=US:en" +366,happy,positive,"Good News Network",OK,2026-02-26,"OK","https://www.goodnewsnetwork.org/feed/" +367,happy,positive,"Positive.News",OK,2026-02-26,"OK","https://www.positive.news/feed/" +368,happy,positive,"Reasons to be Cheerful",OK,2026-02-26,"OK","https://reasonstobecheerful.world/feed/" +369,happy,positive,"Optimist Daily",OK,2026-02-26,"OK","https://www.optimistdaily.com/feed/" +370,happy,positive,"Upworthy",OK,2026-02-26,"OK","https://www.upworthy.com/feed/" +371,happy,positive,"DailyGood",OK,2026-02-23,"OK","https://www.dailygood.org/feed" +372,happy,positive,"Good Good Good",OK,2026-02-26,"OK","https://www.goodgoodgood.co/articles/rss.xml" +373,happy,positive,"GOOD Magazine",OK,2026-02-26,"OK","https://www.good.is/feed/" +374,happy,positive,"Sunny Skyz",OK,2026-02-26,"OK","https://www.sunnyskyz.com/rss_tebow.php" +375,happy,positive,"The Better India",OK,2026-02-26,"OK","https://thebetterindia.com/feed/" +376,happy,science,"GNN Science",OK,2026-02-26,"OK","https://www.goodnewsnetwork.org/category/news/science/feed/" +377,happy,science,"ScienceDaily",DEAD,,"Timeout","https://www.sciencedaily.com/rss/top.xml" +378,happy,science,"Nature News",OK,2026-02-26,"OK","https://feeds.nature.com/nature/rss/current" +379,happy,science,"Live Science",EMPTY,,"No dates found","https://www.livescience.com/feeds/all" +380,happy,science,"New Scientist",OK,2026-02-26,"OK","https://www.newscientist.com/feed/home/" +381,happy,science,"Singularity Hub",OK,2026-02-24,"OK","https://singularityhub.com/feed/" +382,happy,science,"Human Progress",OK,2026-02-26,"OK","https://humanprogress.org/feed/" +383,happy,science,"Greater Good (Berkeley)",EMPTY,,"No dates found","https://greatergood.berkeley.edu/rss" +384,happy,nature,"GNN Animals",OK,2026-02-26,"OK","https://www.goodnewsnetwork.org/category/news/animals/feed/" +385,happy,health,"GNN Health",OK,2026-02-25,"OK","https://www.goodnewsnetwork.org/category/news/health/feed/" +386,happy,inspiring,"GNN Heroes",OK,2026-02-25,"OK","https://www.goodnewsnetwork.org/category/news/inspiring/feed/" +387,intel,inspiring,"Defense One",OK,2026-02-26,"OK","https://www.defenseone.com/rss/all/" +388,intel,inspiring,"Breaking Defense",DEAD,,"HTTP 403","https://breakingdefense.com/feed/" +389,intel,inspiring,"The War Zone",OK,2026-02-26,"OK","https://www.twz.com/feed" +390,intel,inspiring,"Defense News",OK,2026-02-26,"OK","https://www.defensenews.com/arc/outboundfeeds/rss/?outputType=xml" +391,intel,inspiring,"Janes",OK,2026-02-25,"OK","https://news.google.com/rss/search?q=site:janes.com+when:3d&hl=en-US&gl=US&ceid=US:en" +392,intel,inspiring,"Military Times",OK,2026-02-26,"OK","https://www.militarytimes.com/arc/outboundfeeds/rss/?outputType=xml" +393,intel,inspiring,"Task & Purpose",OK,2026-02-26,"OK","https://taskandpurpose.com/feed/" +394,intel,inspiring,"USNI News",OK,2026-02-26,"OK","https://news.usni.org/feed" +395,intel,inspiring,"gCaptain",OK,2026-02-26,"OK","https://gcaptain.com/feed/" +396,intel,inspiring,"Oryx OSINT",STALE,2024-12-07,"Stale","https://www.oryxspioenkop.com/feeds/posts/default?alt=rss" +397,intel,inspiring,"UK MOD",OK,2026-02-26,"OK","https://www.gov.uk/government/organisations/ministry-of-defence.atom" +398,intel,inspiring,"CSIS",EMPTY,,"No dates found","https://www.csis.org/analysis?type=analysis" +399,intel,inspiring,"Chatham House",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:chathamhouse.org+when:7d&hl=en-US&gl=US&ceid=US:en" +400,intel,inspiring,"ECFR",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:ecfr.eu+when:7d&hl=en-US&gl=US&ceid=US:en" +401,intel,inspiring,"Middle East Institute",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:mei.edu+when:7d&hl=en-US&gl=US&ceid=US:en" +402,intel,inspiring,"RAND",DEAD,,"HTTP 404","https://www.rand.org/rss/all.xml" +403,intel,inspiring,"Brookings",EMPTY,,"No dates found","https://www.brookings.edu/feed/" +404,intel,inspiring,"Carnegie",EMPTY,,"No dates found","https://carnegieendowment.org/rss/" +405,intel,inspiring,"FAS",STALE,2023-02-14,"Stale","https://fas.org/feed/" +406,intel,inspiring,"NTI",DEAD,,"HTTP 403","https://www.nti.org/rss/" +407,intel,inspiring,"RUSI",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:rusi.org+when:7d&hl=en-US&gl=US&ceid=US:en" +408,intel,inspiring,"Wilson Center",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:wilsoncenter.org+when:7d&hl=en-US&gl=US&ceid=US:en" +409,intel,inspiring,"GMF",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:gmfus.org+when:7d&hl=en-US&gl=US&ceid=US:en" +410,intel,inspiring,"Stimson Center",OK,2026-02-26,"OK","https://www.stimson.org/feed/" +411,intel,inspiring,"CNAS",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:cnas.org+when:7d&hl=en-US&gl=US&ceid=US:en" +412,intel,inspiring,"Lowy Institute",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:lowyinstitute.org+when:7d&hl=en-US&gl=US&ceid=US:en" +413,intel,inspiring,"Arms Control Assn",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:armscontrol.org+when:7d&hl=en-US&gl=US&ceid=US:en" +414,intel,inspiring,"Bulletin of Atomic Scientists",OK,2026-02-26,"OK","https://news.google.com/rss/search?q=site:thebulletin.org+when:7d&hl=en-US&gl=US&ceid=US:en" +415,intel,inspiring,"Bellingcat",DEAD,,"fetch failed","https://www.bellingcat.com/feed/" +416,intel,inspiring,"Ransomware.live",OK,2026-02-26,"OK","https://www.ransomware.live/rss.xml" +417,intel,inspiring,"FAO News",OK,2026-02-25,"OK","https://www.fao.org/feeds/fao-newsroom-rss" +418,intel,inspiring,"FAO GIEWS",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=site:fao.org+GIEWS+food+security+when:30d&hl=en-US&gl=US&ceid=US:en" +419,intel,inspiring,"EU ISS",OK,2026-02-24,"OK","https://news.google.com/rss/search?q=site:iss.europa.eu+when:7d&hl=en-US&gl=US&ceid=US:en" +420,tech,vcblogs,"FwdStart Newsletter",SKIP,,"Local endpoint","/api/fwdstart" diff --git a/server/worldmonitor/news/v1/list-feed-digest.ts b/server/worldmonitor/news/v1/list-feed-digest.ts index 37cc34395..2d18050f9 100644 --- a/server/worldmonitor/news/v1/list-feed-digest.ts +++ b/server/worldmonitor/news/v1/list-feed-digest.ts @@ -188,7 +188,7 @@ export async function listFeedDigest( const digestCacheKey = `news:digest:v1:${variant}:${lang}`; try { - const cached = await cachedFetchJson(digestCacheKey, 300, async () => { + const cached = await cachedFetchJson(digestCacheKey, 900, async () => { return buildDigest(variant, lang); }); return cached ?? { categories: {}, feedStatuses: {}, generatedAt: new Date().toISOString() }; diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 6d85027a9..25875287e 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -29,7 +29,7 @@ } ], "security": { - "csp": "default-src 'self'; connect-src 'self' https: http://localhost:5173 http://127.0.0.1:* ws: wss: blob: data:; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline'; script-src 'self' 'wasm-unsafe-eval' https://www.youtube.com https://us-assets.i.posthog.com; worker-src 'self' blob:; font-src 'self' data: https:; media-src 'self' data: blob: https:; frame-src 'self' http://127.0.0.1:* http://localhost:* https://worldmonitor.app https://tech.worldmonitor.app https://www.youtube.com https://www.youtube-nocookie.com;" + "csp": "default-src 'self'; connect-src 'self' https: http://localhost:5173 http://127.0.0.1:* ws: wss: blob: data:; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline'; script-src 'self' 'wasm-unsafe-eval' https://www.youtube.com; worker-src 'self' blob:; font-src 'self' data: https:; media-src 'self' data: blob: https:; frame-src 'self' http://127.0.0.1:* http://localhost:* https://worldmonitor.app https://tech.worldmonitor.app https://www.youtube.com https://www.youtube-nocookie.com;" } }, "bundle": { diff --git a/src/app/data-loader.ts b/src/app/data-loader.ts index dcc0b21f0..5f5fb479f 100644 --- a/src/app/data-loader.ts +++ b/src/app/data-loader.ts @@ -77,7 +77,7 @@ import { fetchTelegramFeed } from '@/services/telegram-intel'; import { fetchOrefAlerts, startOrefPolling, stopOrefPolling, onOrefAlertsUpdate } from '@/services/oref-alerts'; import { enrichEventsWithExposure } from '@/services/population-exposure'; import { debounce, getCircuitBreakerCooldownInfo } from '@/utils'; -import { isFeatureAvailable } from '@/services/runtime-config'; +import { isFeatureAvailable, isFeatureEnabled } from '@/services/runtime-config'; import { getAiFlowSettings } from '@/services/ai-flow-settings'; import { t, getCurrentLanguage } from '@/services/i18n'; import { getHydratedData } from '@/services/bootstrap'; @@ -175,6 +175,12 @@ export class DataLoaderManager implements AppModule { public updateSearchIndex: () => void = () => {}; private digestBreaker = { state: 'closed' as 'closed' | 'open' | 'half-open', failures: 0, cooldownUntil: 0 }; + private readonly digestRequestTimeoutMs = 8000; + private readonly digestBreakerCooldownMs = 5 * 60 * 1000; + private readonly persistedDigestMaxAgeMs = 6 * 60 * 60 * 1000; + private readonly perFeedFallbackCategoryFeedLimit = 3; + private readonly perFeedFallbackIntelFeedLimit = 6; + private readonly perFeedFallbackBatchSize = 2; private lastGoodDigest: ListFeedDigestResponse | null = null; constructor(ctx: AppContext, callbacks: DataLoaderCallbacks) { @@ -201,7 +207,7 @@ export class DataLoaderManager implements AppModule { try { const resp = await fetch( `/api/news/v1/list-feed-digest?variant=${SITE_VARIANT}&lang=${getCurrentLanguage()}`, - { signal: AbortSignal.timeout(3000) }, + { signal: AbortSignal.timeout(this.digestRequestTimeoutMs) }, ); if (!resp.ok) throw new Error(`HTTP ${resp.status}`); const data = await resp.json() as ListFeedDigestResponse; @@ -216,7 +222,7 @@ export class DataLoaderManager implements AppModule { this.digestBreaker.failures++; if (this.digestBreaker.failures >= 2) { this.digestBreaker.state = 'open'; - this.digestBreaker.cooldownUntil = now + 60_000; + this.digestBreaker.cooldownUntil = now + this.digestBreakerCooldownMs; } return this.lastGoodDigest ?? await this.loadPersistedDigest(); } @@ -230,12 +236,27 @@ export class DataLoaderManager implements AppModule { try { const envelope = await getPersistentCache('digest:last-good'); if (!envelope) return null; - if (Date.now() - envelope.updatedAt > 30 * 60 * 1000) return null; + if (Date.now() - envelope.updatedAt > this.persistedDigestMaxAgeMs) return null; this.lastGoodDigest = envelope.data; return envelope.data; } catch { return null; } } + private isPerFeedFallbackEnabled(): boolean { + return isFeatureEnabled('newsPerFeedFallback'); + } + + private getStaleNewsItems(category: string): NewsItem[] { + const staleItems = this.ctx.newsByCategory[category]; + if (!Array.isArray(staleItems) || staleItems.length === 0) return []; + return [...staleItems].sort((a, b) => b.pubDate.getTime() - a.pubDate.getTime()); + } + + private selectLimitedFeeds(feeds: T[], maxFeeds: number): T[] { + if (feeds.length <= maxFeeds) return feeds; + return feeds.slice(0, maxFeeds); + } + private shouldShowIntelligenceNotifications(): boolean { return !this.ctx.isMobile && !!this.ctx.findingsBadge?.isPopupEnabled(); } @@ -540,10 +561,10 @@ export class DataLoaderManager implements AppModule { }); return []; } + const enabledNames = new Set(enabledFeeds.map(f => f.name)); // Digest branch: server already aggregated feeds — map proto items to client types if (digest?.categories && category in digest.categories) { - const enabledNames = new Set(enabledFeeds.map(f => f.name)); let items = (digest.categories[category]?.items ?? []) .map(protoItemToNewsItem) .filter(i => enabledNames.has(i.source)); @@ -584,7 +605,7 @@ export class DataLoaderManager implements AppModule { return items; } - // Per-feed fallback: fetch each feed individually (first load or digest unavailable) + // Fallback path when digest is unavailable: stale-first, then limited per-feed fan-out. const renderIntervalMs = 100; let lastRenderTime = 0; let renderTimeout: ReturnType | null = null; @@ -618,7 +639,36 @@ export class DataLoaderManager implements AppModule { } }; - const items = await fetchCategoryFeeds(enabledFeeds, { + const staleItems = this.getStaleNewsItems(category).filter(i => enabledNames.has(i.source)); + if (staleItems.length > 0) { + console.warn(`[News] Digest missing for "${category}", serving stale headlines (${staleItems.length})`); + this.renderNewsForCategory(category, staleItems); + this.ctx.statusPanel?.updateFeed(category.charAt(0).toUpperCase() + category.slice(1), { + status: 'ok', + itemCount: staleItems.length, + }); + return staleItems; + } + + if (!this.isPerFeedFallbackEnabled()) { + console.warn(`[News] Digest missing for "${category}", limited per-feed fallback disabled`); + this.renderNewsForCategory(category, []); + this.ctx.statusPanel?.updateFeed(category.charAt(0).toUpperCase() + category.slice(1), { + status: 'error', + errorMessage: 'Digest unavailable', + }); + return []; + } + + const fallbackFeeds = this.selectLimitedFeeds(enabledFeeds, this.perFeedFallbackCategoryFeedLimit); + if (fallbackFeeds.length < enabledFeeds.length) { + console.warn(`[News] Digest missing for "${category}", using limited per-feed fallback (${fallbackFeeds.length}/${enabledFeeds.length} feeds)`); + } else { + console.warn(`[News] Digest missing for "${category}", using per-feed fallback (${fallbackFeeds.length} feeds)`); + } + + const items = await fetchCategoryFeeds(fallbackFeeds, { + batchSize: this.perFeedFallbackBatchSize, onBatch: (partialItems) => { scheduleRender(partialItems); this.flashMapForNews(partialItems); @@ -636,7 +686,7 @@ export class DataLoaderManager implements AppModule { if (items.length === 0) { const failures = getFeedFailures(); - const failedFeeds = enabledFeeds.filter(f => failures.has(f.name)); + const failedFeeds = fallbackFeeds.filter(f => failures.has(f.name)); if (failedFeeds.length > 0) { const names = failedFeeds.map(f => f.name).join(', '); panel.showError(`${t('common.noNewsAvailable')} (${names} failed)`); @@ -714,6 +764,7 @@ export class DataLoaderManager implements AppModule { if (SITE_VARIANT === 'full') { const enabledIntelSources = INTEL_SOURCES.filter(f => !this.ctx.disabledSources.has(f.name)); + const enabledIntelNames = new Set(enabledIntelSources.map(f => f.name)); const intelPanel = this.ctx.newsPanels['intel']; if (enabledIntelSources.length === 0) { delete this.ctx.newsByCategory['intel']; @@ -721,10 +772,9 @@ export class DataLoaderManager implements AppModule { this.ctx.statusPanel?.updateFeed('Intel', { status: 'ok', itemCount: 0 }); } else if (digest?.categories && 'intel' in digest.categories) { // Digest branch for intel - const enabledNames = new Set(enabledIntelSources.map(f => f.name)); const intel = (digest.categories['intel']?.items ?? []) .map(protoItemToNewsItem) - .filter(i => enabledNames.has(i.source)); + .filter(i => enabledIntelNames.has(i.source)); checkBatchForBreakingAlerts(intel); this.renderNewsForCategory('intel', intel); if (intelPanel) { @@ -738,24 +788,50 @@ export class DataLoaderManager implements AppModule { collectedNews.push(...intel); this.flashMapForNews(intel); } else { - const intelResult = await Promise.allSettled([fetchCategoryFeeds(enabledIntelSources)]); - if (intelResult[0]?.status === 'fulfilled') { - const intel = intelResult[0].value; - checkBatchForBreakingAlerts(intel); - this.renderNewsForCategory('intel', intel); + const staleIntel = this.getStaleNewsItems('intel').filter(i => enabledIntelNames.has(i.source)); + if (staleIntel.length > 0) { + console.warn(`[News] Intel digest missing, serving stale headlines (${staleIntel.length})`); + this.renderNewsForCategory('intel', staleIntel); if (intelPanel) { try { - const baseline = await updateBaseline('news:intel', intel.length); - const deviation = calculateDeviation(intel.length, baseline); + const baseline = await updateBaseline('news:intel', staleIntel.length); + const deviation = calculateDeviation(staleIntel.length, baseline); intelPanel.setDeviation(deviation.zScore, deviation.percentChange, deviation.level); } catch (e) { console.warn('[Baseline] news:intel write failed:', e); } } - this.ctx.statusPanel?.updateFeed('Intel', { status: 'ok', itemCount: intel.length }); - collectedNews.push(...intel); - this.flashMapForNews(intel); - } else { + this.ctx.statusPanel?.updateFeed('Intel', { status: 'ok', itemCount: staleIntel.length }); + collectedNews.push(...staleIntel); + } else if (!this.isPerFeedFallbackEnabled()) { + console.warn('[News] Intel digest missing, limited per-feed fallback disabled'); delete this.ctx.newsByCategory['intel']; - console.error('[App] Intel feed failed:', intelResult[0]?.reason); + this.ctx.statusPanel?.updateFeed('Intel', { status: 'error', errorMessage: 'Digest unavailable' }); + } else { + const fallbackIntelFeeds = this.selectLimitedFeeds(enabledIntelSources, this.perFeedFallbackIntelFeedLimit); + if (fallbackIntelFeeds.length < enabledIntelSources.length) { + console.warn(`[News] Intel digest missing, using limited per-feed fallback (${fallbackIntelFeeds.length}/${enabledIntelSources.length} feeds)`); + } + + const intelResult = await Promise.allSettled([ + fetchCategoryFeeds(fallbackIntelFeeds, { batchSize: this.perFeedFallbackBatchSize }), + ]); + if (intelResult[0]?.status === 'fulfilled') { + const intel = intelResult[0].value; + checkBatchForBreakingAlerts(intel); + this.renderNewsForCategory('intel', intel); + if (intelPanel) { + try { + const baseline = await updateBaseline('news:intel', intel.length); + const deviation = calculateDeviation(intel.length, baseline); + intelPanel.setDeviation(deviation.zScore, deviation.percentChange, deviation.level); + } catch (e) { console.warn('[Baseline] news:intel write failed:', e); } + } + this.ctx.statusPanel?.updateFeed('Intel', { status: 'ok', itemCount: intel.length }); + collectedNews.push(...intel); + this.flashMapForNews(intel); + } else { + delete this.ctx.newsByCategory['intel']; + console.error('[App] Intel feed failed:', intelResult[0]?.reason); + } } } } diff --git a/src/config/variants/base.ts b/src/config/variants/base.ts index e47090a51..bfa9be935 100644 --- a/src/config/variants/base.ts +++ b/src/config/variants/base.ts @@ -8,7 +8,7 @@ export { AI_DATA_CENTERS } from '../ai-datacenters'; // Refresh intervals - shared across all variants export const REFRESH_INTERVALS = { - feeds: 7 * 60 * 1000, + feeds: 15 * 60 * 1000, markets: 8 * 60 * 1000, crypto: 8 * 60 * 1000, predictions: 10 * 60 * 1000, diff --git a/src/main.ts b/src/main.ts index 3d40140e6..a52da8a1b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -155,8 +155,6 @@ Sentry.init({ if (frames.length > 0 && frames.every(f => /^blob:/.test(f.filename ?? ''))) return null; // Suppress YouTube IFrame widget API internal errors if (frames.some(f => /www-widgetapi\.js/.test(f.filename ?? ''))) return null; - // Suppress Sentry SDK internal crashes (logs.js) - if (frames.some(f => /\/ingest\/static\/logs\.js/.test(f.filename ?? ''))) return null; return event; }, }); @@ -170,7 +168,6 @@ import { debugGetCells, getCellCount } from '@/services/geo-convergence'; import { initMetaTags } from '@/services/meta-tags'; import { installRuntimeFetchPatch, installWebApiRedirect } from '@/services/runtime'; import { loadDesktopSecrets } from '@/services/runtime-config'; -import { initAnalytics, trackApiKeysSnapshot } from '@/services/analytics'; import { applyStoredTheme } from '@/utils/theme-manager'; import { SITE_VARIANT } from '@/config/variant'; import { clearChunkReloadGuard, installChunkReloadGuard } from '@/bootstrap/chunk-reload'; @@ -181,9 +178,6 @@ const chunkReloadStorageKey = installChunkReloadGuard(__APP_VERSION__); // Initialize Vercel Analytics inject(); -// Initialize PostHog product analytics -void initAnalytics(); - // Initialize dynamic meta tags for sharing initMetaTags(); @@ -191,10 +185,7 @@ initMetaTags(); installRuntimeFetchPatch(); // In web production, route RPC calls through api.worldmonitor.app (Cloudflare edge). installWebApiRedirect(); -loadDesktopSecrets().then(async () => { - await initAnalytics(); - trackApiKeysSnapshot(); -}).catch(() => {}); +loadDesktopSecrets().catch(() => {}); // Apply stored theme preference before app initialization (safety net for inline script) applyStoredTheme(); diff --git a/src/services/analytics.ts b/src/services/analytics.ts index 96a2ff03d..8a506ff35 100644 --- a/src/services/analytics.ts +++ b/src/services/analytics.ts @@ -1,386 +1,123 @@ /** - * PostHog Analytics Service + * Analytics facade. * - * Always active when VITE_POSTHOG_KEY is set. No consent gate. - * All exports are no-ops when the key is absent (dev/local). - * - * Data safety: - * - Typed allowlists per event — unlisted properties silently dropped - * - sanitize_properties callback strips strings matching API key prefixes - * - No session recordings, no autocapture - * - distinct_id is a random UUID — pseudonymous, not identifiable + * PostHog has been removed from the application. + * Vercel Analytics remains initialized in src/main.ts. + * Event-level helpers are kept as no-ops to preserve existing call sites. */ -import { isDesktopRuntime } from './runtime'; -import { getRuntimeConfigSnapshot, type RuntimeSecretKey } from './runtime-config'; -import { SITE_VARIANT } from '@/config'; -import { isMobileDevice } from '@/utils'; -import { invokeTauri } from './tauri-bridge'; - -// ── Installation identity ── - -function getOrCreateInstallationId(): string { - const STORAGE_KEY = 'wm-installation-id'; - let id = localStorage.getItem(STORAGE_KEY); - if (!id) { - id = crypto.randomUUID(); - localStorage.setItem(STORAGE_KEY, id); - } - return id; -} - -// ── Stable property name map for secret keys ── - -const SECRET_ANALYTICS_NAMES: Record = { - GROQ_API_KEY: 'groq', - OPENROUTER_API_KEY: 'openrouter', - FRED_API_KEY: 'fred', - EIA_API_KEY: 'eia', - CLOUDFLARE_API_TOKEN: 'cloudflare', - ACLED_ACCESS_TOKEN: 'acled', - URLHAUS_AUTH_KEY: 'urlhaus', - OTX_API_KEY: 'otx', - ABUSEIPDB_API_KEY: 'abuseipdb', - WINGBITS_API_KEY: 'wingbits', - WS_RELAY_URL: 'ws_relay', - VITE_OPENSKY_RELAY_URL: 'opensky_relay', - OPENSKY_CLIENT_ID: 'opensky', - OPENSKY_CLIENT_SECRET: 'opensky_secret', - AISSTREAM_API_KEY: 'aisstream', - FINNHUB_API_KEY: 'finnhub', - NASA_FIRMS_API_KEY: 'nasa_firms', - UC_DP_KEY: 'uc_dp', - OLLAMA_API_URL: 'ollama_url', - OLLAMA_MODEL: 'ollama_model', - WORLDMONITOR_API_KEY: 'worldmonitor', - WTO_API_KEY: 'wto', - AVIATIONSTACK_API: 'aviationstack', - ICAO_API_KEY: 'icao', -}; - -// ── Typed event schemas (allowlisted properties per event) ── - -const HAS_KEYS = Object.values(SECRET_ANALYTICS_NAMES).map(n => `has_${n}`); - -const EVENT_SCHEMAS: Record> = { - // Phase 1 — core events - wm_app_loaded: new Set(['load_time_ms', 'panel_count']), - wm_panel_viewed: new Set(['panel_id']), - wm_summary_generated: new Set(['provider', 'model', 'cached']), - wm_summary_failed: new Set(['last_provider']), - wm_api_keys_configured: new Set([ - 'total_keys_configured', 'total_features_enabled', 'enabled_features', - 'ollama_model', 'platform', - ...HAS_KEYS, - ]), - // Phase 2 — plan-specified events - wm_panel_resized: new Set(['panel_id', 'new_span']), - wm_variant_switched: new Set(['from', 'to']), - wm_map_layer_toggled: new Set(['layer_id', 'enabled', 'source']), - wm_country_brief_opened: new Set(['country_code']), - wm_theme_changed: new Set(['theme']), - wm_language_changed: new Set(['language']), - wm_feature_toggled: new Set(['feature_id', 'enabled']), - wm_search_used: new Set(['query_length', 'result_count']), - // Phase 2 — additional interaction events - wm_map_view_changed: new Set(['view']), - wm_country_selected: new Set(['country_code', 'country_name', 'source']), - wm_search_result_selected: new Set(['result_type']), - wm_panel_toggled: new Set(['panel_id', 'enabled']), - wm_finding_clicked: new Set(['finding_id', 'finding_source', 'finding_type', 'priority']), - wm_update_shown: new Set(['current_version', 'remote_version']), - wm_update_clicked: new Set(['target_version']), - wm_update_dismissed: new Set(['target_version']), - wm_critical_banner_action: new Set(['action', 'theater_id']), - wm_download_clicked: new Set(['platform']), - wm_download_banner_dismissed: new Set([]), - wm_webcam_selected: new Set(['webcam_id', 'city', 'view_mode']), - wm_webcam_region_filtered: new Set(['region']), - wm_deeplink_opened: new Set(['deeplink_type', 'target']), -}; - -function sanitizeProps(event: string, raw: Record): Record { - const allowed = EVENT_SCHEMAS[event]; - if (!allowed) return {}; - const safe: Record = {}; - for (const [k, v] of Object.entries(raw)) { - if (allowed.has(k)) safe[k] = v; - } - return safe; -} - -// ── Defense-in-depth: strip values that look like API keys ── - -const API_KEY_PREFIXES = /^(sk-|gsk_|or-|Bearer )/; - -function deepStripSecrets(props: Record): Record { - const cleaned: Record = {}; - for (const [k, v] of Object.entries(props)) { - if (typeof v === 'string' && API_KEY_PREFIXES.test(v)) { - cleaned[k] = '[REDACTED]'; - } else { - cleaned[k] = v; - } - } - return cleaned; -} - -// ── PostHog instance management ── - -type PostHogInstance = { - init: (key: string, config: Record) => void; - register: (props: Record) => void; - capture: (event: string, props?: Record, options?: { transport?: 'XHR' | 'sendBeacon' }) => void; -}; - -let posthogInstance: PostHogInstance | null = null; -let initPromise: Promise | null = null; - -const POSTHOG_KEY = import.meta.env.VITE_POSTHOG_KEY as string | undefined; -const POSTHOG_HOST = isDesktopRuntime() - ? ((import.meta.env.VITE_POSTHOG_HOST as string | undefined) || 'https://us.i.posthog.com') - : '/ingest'; // Reverse proxy through own domain to bypass ad blockers - -// ── Public API ── - export async function initAnalytics(): Promise { - if (!POSTHOG_KEY) return; - if (initPromise) return initPromise; - - initPromise = (async () => { - try { - const mod = await import('posthog-js'); - const posthog = mod.default; - - posthog.init(POSTHOG_KEY, { - api_host: POSTHOG_HOST, - ui_host: 'https://us.posthog.com', - persistence: 'localStorage', - autocapture: false, - capture_pageview: false, // Manual capture below — auto-capture silently fails with bootstrap + SPA - capture_pageleave: true, - disable_session_recording: true, - bootstrap: { distinctID: getOrCreateInstallationId() }, - sanitize_properties: (props: Record) => deepStripSecrets(props), - }); - - // Register super properties (attached to every event) - const superProps: Record = { - platform: isDesktopRuntime() ? 'desktop' : 'web', - variant: SITE_VARIANT, - app_version: __APP_VERSION__, - is_mobile: isMobileDevice(), - screen_width: screen.width, - screen_height: screen.height, - viewport_width: innerWidth, - viewport_height: innerHeight, - is_big_screen: screen.width >= 2560 || screen.height >= 1440, - is_tv_mode: screen.width >= 3840, - device_pixel_ratio: devicePixelRatio, - browser_language: navigator.language, - local_hour: new Date().getHours(), - local_day: new Date().getDay(), - }; - - // Desktop additionally registers OS and arch - if (isDesktopRuntime()) { - try { - const info = await invokeTauri<{ os: string; arch: string }>('get_desktop_runtime_info'); - superProps.desktop_os = info.os; - superProps.desktop_arch = info.arch; - } catch { - // Tauri bridge may not be available yet - } - } - - posthog.register(superProps); - posthogInstance = posthog as unknown as PostHogInstance; - - // Fire $pageview manually after full init — auto capture_pageview: true - // fires during init() before super props are registered, and silently - // fails with bootstrap + SPA setups (posthog-js #386). - posthog.capture('$pageview'); - - // Flush any events queued while offline (desktop) - flushOfflineQueue(); - - // Re-flush when coming back online - if (isDesktopRuntime()) { - window.addEventListener('online', () => flushOfflineQueue()); - } - } catch (error) { - console.warn('[Analytics] Failed to initialize PostHog:', error); - } - })(); - - return initPromise; + // Intentionally no-op. } -// ── Offline event queue (desktop) ── - -const OFFLINE_QUEUE_KEY = 'wm-analytics-offline-queue'; -const OFFLINE_QUEUE_CAP = 200; - -function enqueueOffline(name: string, props: Record): void { - try { - const raw = localStorage.getItem(OFFLINE_QUEUE_KEY); - const queue: Array<{ name: string; props: Record; ts: number }> = raw ? JSON.parse(raw) : []; - queue.push({ name, props, ts: Date.now() }); - if (queue.length > OFFLINE_QUEUE_CAP) queue.splice(0, queue.length - OFFLINE_QUEUE_CAP); - localStorage.setItem(OFFLINE_QUEUE_KEY, JSON.stringify(queue)); - } catch { /* localStorage full or unavailable */ } +export function trackEvent(_name: string, _props?: Record): void { + // Intentionally no-op. } -function flushOfflineQueue(): void { - if (!posthogInstance) return; - try { - const raw = localStorage.getItem(OFFLINE_QUEUE_KEY); - if (!raw) return; - const queue: Array<{ name: string; props: Record }> = JSON.parse(raw); - localStorage.removeItem(OFFLINE_QUEUE_KEY); - for (const { name, props } of queue) { - posthogInstance.capture(name, props); - } - } catch { /* corrupt queue, discard */ } +export function trackEventBeforeUnload(_name: string, _props?: Record): void { + // Intentionally no-op. } -export function trackEvent(name: string, props?: Record): void { - const safeProps = props ? sanitizeProps(name, props) : {}; - if (!posthogInstance) { - if (isDesktopRuntime() && POSTHOG_KEY) enqueueOffline(name, safeProps); - return; - } - posthogInstance.capture(name, safeProps); -} - -/** Use sendBeacon transport for events fired just before page reload. */ -export function trackEventBeforeUnload(name: string, props?: Record): void { - if (!posthogInstance) return; - const safeProps = props ? sanitizeProps(name, props) : {}; - posthogInstance.capture(name, safeProps, { transport: 'sendBeacon' }); -} - -export function trackPanelView(panelId: string): void { - trackEvent('wm_panel_viewed', { panel_id: panelId }); +export function trackPanelView(_panelId: string): void { + // Intentionally no-op. } export function trackApiKeysSnapshot(): void { - const config = getRuntimeConfigSnapshot(); - const presence: Record = {}; - for (const [internalKey, analyticsName] of Object.entries(SECRET_ANALYTICS_NAMES)) { - const state = config.secrets[internalKey as RuntimeSecretKey]; - presence[`has_${analyticsName}`] = Boolean(state?.value); - } - - const enabledFeatures = Object.entries(config.featureToggles) - .filter(([, v]) => v).map(([k]) => k); - - trackEvent('wm_api_keys_configured', { - platform: isDesktopRuntime() ? 'desktop' : 'web', - total_keys_configured: Object.values(presence).filter(Boolean).length, - ...presence, - enabled_features: enabledFeatures, - total_features_enabled: enabledFeatures.length, - ollama_model: config.secrets.OLLAMA_MODEL?.value || 'none', - }); + // Intentionally no-op. } -export function trackLLMUsage(provider: string, model: string, cached: boolean): void { - trackEvent('wm_summary_generated', { provider, model, cached }); +export function trackLLMUsage(_provider: string, _model: string, _cached: boolean): void { + // Intentionally no-op. } -export function trackLLMFailure(lastProvider: string): void { - trackEvent('wm_summary_failed', { last_provider: lastProvider }); +export function trackLLMFailure(_lastProvider: string): void { + // Intentionally no-op. } -// ── Phase 2 helpers (plan-specified events) ── - -export function trackPanelResized(panelId: string, newSpan: number): void { - trackEvent('wm_panel_resized', { panel_id: panelId, new_span: newSpan }); +export function trackPanelResized(_panelId: string, _newSpan: number): void { + // Intentionally no-op. } -export function trackVariantSwitch(from: string, to: string): void { - trackEventBeforeUnload('wm_variant_switched', { from, to }); +export function trackVariantSwitch(_from: string, _to: string): void { + // Intentionally no-op. } -export function trackMapLayerToggle(layerId: string, enabled: boolean, source: 'user' | 'programmatic'): void { - trackEvent('wm_map_layer_toggled', { layer_id: layerId, enabled, source }); +export function trackMapLayerToggle(_layerId: string, _enabled: boolean, _source: 'user' | 'programmatic'): void { + // Intentionally no-op. } -export function trackCountryBriefOpened(countryCode: string): void { - trackEvent('wm_country_brief_opened', { country_code: countryCode }); +export function trackCountryBriefOpened(_countryCode: string): void { + // Intentionally no-op. } -export function trackThemeChanged(theme: string): void { - trackEventBeforeUnload('wm_theme_changed', { theme }); +export function trackThemeChanged(_theme: string): void { + // Intentionally no-op. } -export function trackLanguageChange(language: string): void { - trackEventBeforeUnload('wm_language_changed', { language }); +export function trackLanguageChange(_language: string): void { + // Intentionally no-op. } -export function trackFeatureToggle(featureId: string, enabled: boolean): void { - trackEvent('wm_feature_toggled', { feature_id: featureId, enabled }); +export function trackFeatureToggle(_featureId: string, _enabled: boolean): void { + // Intentionally no-op. } -export function trackSearchUsed(queryLength: number, resultCount: number): void { - trackEvent('wm_search_used', { query_length: queryLength, result_count: resultCount }); +export function trackSearchUsed(_queryLength: number, _resultCount: number): void { + // Intentionally no-op. } -// ── Phase 2 helpers (additional interaction events) ── - -export function trackMapViewChange(view: string): void { - trackEvent('wm_map_view_changed', { view }); +export function trackMapViewChange(_view: string): void { + // Intentionally no-op. } -export function trackCountrySelected(code: string, name: string, source: string): void { - trackEvent('wm_country_selected', { country_code: code, country_name: name, source }); +export function trackCountrySelected(_code: string, _name: string, _source: string): void { + // Intentionally no-op. } -export function trackSearchResultSelected(resultType: string): void { - trackEvent('wm_search_result_selected', { result_type: resultType }); +export function trackSearchResultSelected(_resultType: string): void { + // Intentionally no-op. } -export function trackPanelToggled(panelId: string, enabled: boolean): void { - trackEvent('wm_panel_toggled', { panel_id: panelId, enabled }); +export function trackPanelToggled(_panelId: string, _enabled: boolean): void { + // Intentionally no-op. } -export function trackFindingClicked(id: string, source: string, type: string, priority: string): void { - trackEvent('wm_finding_clicked', { finding_id: id, finding_source: source, finding_type: type, priority }); +export function trackFindingClicked(_id: string, _source: string, _type: string, _priority: string): void { + // Intentionally no-op. } -export function trackUpdateShown(current: string, remote: string): void { - trackEvent('wm_update_shown', { current_version: current, remote_version: remote }); +export function trackUpdateShown(_current: string, _remote: string): void { + // Intentionally no-op. } -export function trackUpdateClicked(version: string): void { - trackEvent('wm_update_clicked', { target_version: version }); +export function trackUpdateClicked(_version: string): void { + // Intentionally no-op. } -export function trackUpdateDismissed(version: string): void { - trackEvent('wm_update_dismissed', { target_version: version }); +export function trackUpdateDismissed(_version: string): void { + // Intentionally no-op. } -export function trackCriticalBannerAction(action: string, theaterId: string): void { - trackEvent('wm_critical_banner_action', { action, theater_id: theaterId }); +export function trackCriticalBannerAction(_action: string, _theaterId: string): void { + // Intentionally no-op. } -export function trackDownloadClicked(platform: string): void { - trackEvent('wm_download_clicked', { platform }); +export function trackDownloadClicked(_platform: string): void { + // Intentionally no-op. } export function trackDownloadBannerDismissed(): void { - trackEvent('wm_download_banner_dismissed'); + // Intentionally no-op. } -export function trackWebcamSelected(webcamId: string, city: string, viewMode: string): void { - trackEvent('wm_webcam_selected', { webcam_id: webcamId, city, view_mode: viewMode }); +export function trackWebcamSelected(_webcamId: string, _city: string, _viewMode: string): void { + // Intentionally no-op. } -export function trackWebcamRegionFiltered(region: string): void { - trackEvent('wm_webcam_region_filtered', { region }); +export function trackWebcamRegionFiltered(_region: string): void { + // Intentionally no-op. } -export function trackDeeplinkOpened(type: string, target: string): void { - trackEvent('wm_deeplink_opened', { deeplink_type: type, target }); +export function trackDeeplinkOpened(_type: string, _target: string): void { + // Intentionally no-op. } diff --git a/src/services/rss.ts b/src/services/rss.ts index 4dc473429..7b538c63f 100644 --- a/src/services/rss.ts +++ b/src/services/rss.ts @@ -15,7 +15,7 @@ const MAX_CACHE_ENTRIES = 100; const FEED_SCOPE_SEPARATOR = '::'; const feedFailures = new Map(); const feedCache = new Map(); -const CACHE_TTL = 10 * 60 * 1000; +const CACHE_TTL = 30 * 60 * 1000; function toSerializable(items: NewsItem[]): Array & { pubDate: string }> { return items.map(item => ({ ...item, pubDate: item.pubDate.toISOString() })); diff --git a/src/services/runtime-config.ts b/src/services/runtime-config.ts index dfa33486c..1408da3b9 100644 --- a/src/services/runtime-config.ts +++ b/src/services/runtime-config.ts @@ -45,6 +45,7 @@ export type RuntimeFeatureId = | 'aiOllama' | 'wtoTrade' | 'supplyChain' + | 'newsPerFeedFallback' | 'aviationStack' | 'icaoNotams'; @@ -93,6 +94,7 @@ const defaultToggles: Record = { aiOllama: true, wtoTrade: true, supplyChain: true, + newsPerFeedFallback: false, aviationStack: true, icaoNotams: true, }; @@ -219,6 +221,13 @@ export const RUNTIME_FEATURES: RuntimeFeatureDefinition[] = [ requiredSecrets: ['FRED_API_KEY'], fallback: 'Chokepoints and minerals always available; shipping requires FRED key.', }, + { + id: 'newsPerFeedFallback', + name: 'News per-feed fallback', + description: 'If digest aggregation is unavailable, use stale headlines first and optionally fetch a limited feed subset.', + requiredSecrets: [], + fallback: 'Stale headlines remain available; limited per-feed fallback is disabled.', + }, { id: 'aviationStack', name: 'AviationStack flight delays', diff --git a/src/services/runtime.ts b/src/services/runtime.ts index 9b169d8bb..3ecc2f9f4 100644 --- a/src/services/runtime.ts +++ b/src/services/runtime.ts @@ -1,4 +1,5 @@ const WS_API_URL = import.meta.env.VITE_WS_API_URL || ''; +const KEYED_CLOUD_API_PATTERN = /^\/api\/(?:[^/]+\/v1\/|bootstrap(?:\?|$)|rss-proxy(?:\?|$)|polymarket(?:\?|$)|ais-snapshot(?:\?|$))/; const DEFAULT_REMOTE_HOSTS: Record = { tech: WS_API_URL, @@ -323,7 +324,7 @@ export function installRuntimeFetchPatch(): void { const cloudUrl = `${getRemoteApiBaseUrl()}${target}`; if (debug) console.log(`[fetch] cloud fallback → ${cloudUrl}`); const cloudHeaders = new Headers(init?.headers); - if (/^\/api\/[^/]+\/v1\//.test(target)) { + if (KEYED_CLOUD_API_PATTERN.test(target)) { const { getRuntimeConfigSnapshot } = await import('@/services/runtime-config'); const wmKeyValue = getRuntimeConfigSnapshot().secrets['WORLDMONITOR_API_KEY']?.value; if (wmKeyValue) { @@ -385,7 +386,12 @@ export function installRuntimeFetchPatch(): void { (window as unknown as Record).__wmFetchPatched = true; } -const WEB_RPC_PATTERN = /^\/api\/[^/]+\/v1\//; +const WEB_REDIRECT_PATHS = [ + /^\/api\/[^/]+\/v1\//, + /^\/api\/rss-proxy(?:\?|$)/, + /^\/api\/polymarket(?:\?|$)/, + /^\/api\/ais-snapshot(?:\?|$)/, +]; const ALLOWED_REDIRECT_HOSTS = /^https:\/\/([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)*worldmonitor\.app(:\d+)?$/; function isAllowedRedirectTarget(url: string): boolean { @@ -408,18 +414,37 @@ export function installWebApiRedirect(): void { const nativeFetch = window.fetch.bind(window); const API_BASE = WS_API_URL; + const shouldRedirectPath = (pathWithQuery: string): boolean => WEB_REDIRECT_PATHS.some((pattern) => pattern.test(pathWithQuery)); + const shouldFallbackToOrigin = (status: number): boolean => status === 404 || status === 405 || status === 501; + const fetchWithRedirectFallback = async ( + redirectedInput: RequestInfo | URL, + originalInput: RequestInfo | URL, + originalInit?: RequestInit, + ): Promise => { + try { + const redirectedResponse = await nativeFetch(redirectedInput, originalInit); + if (!shouldFallbackToOrigin(redirectedResponse.status)) return redirectedResponse; + return nativeFetch(originalInput, originalInit); + } catch { + return nativeFetch(originalInput, originalInit); + } + }; window.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - if (typeof input === 'string' && WEB_RPC_PATTERN.test(input)) { - return nativeFetch(`${API_BASE}${input}`, init); + if (typeof input === 'string' && shouldRedirectPath(input)) { + return fetchWithRedirectFallback(`${API_BASE}${input}`, input, init); } - if (input instanceof URL && input.origin === window.location.origin && WEB_RPC_PATTERN.test(input.pathname)) { - return nativeFetch(new URL(`${API_BASE}${input.pathname}${input.search}`), init); + if (input instanceof URL && input.origin === window.location.origin && shouldRedirectPath(`${input.pathname}${input.search}`)) { + return fetchWithRedirectFallback(new URL(`${API_BASE}${input.pathname}${input.search}`), input, init); } if (input instanceof Request) { const u = new URL(input.url); - if (u.origin === window.location.origin && WEB_RPC_PATTERN.test(u.pathname)) { - return nativeFetch(new Request(`${API_BASE}${u.pathname}${u.search}`, input), init); + if (u.origin === window.location.origin && shouldRedirectPath(`${u.pathname}${u.search}`)) { + return fetchWithRedirectFallback( + new Request(`${API_BASE}${u.pathname}${u.search}`, input), + input.clone(), + init, + ); } } return nativeFetch(input, init); diff --git a/src/services/settings-constants.ts b/src/services/settings-constants.ts index 80bd51f31..ff931409e 100644 --- a/src/services/settings-constants.ts +++ b/src/services/settings-constants.ts @@ -90,6 +90,6 @@ export const SETTINGS_CATEGORIES: SettingsCategory[] = [ { id: 'tracking', label: 'Tracking & Sensing', - features: ['aisRelay', 'openskyRelay', 'wingbitsEnrichment', 'nasaFirms', 'aviationStack', 'icaoNotams'], + features: ['aisRelay', 'openskyRelay', 'wingbitsEnrichment', 'nasaFirms', 'aviationStack', 'icaoNotams', 'newsPerFeedFallback'], }, ]; diff --git a/vercel.json b/vercel.json index 365f51cce..17195d05d 100644 --- a/vercel.json +++ b/vercel.json @@ -1,9 +1,5 @@ { "ignoreCommand": "if [ -z \"$VERCEL_GIT_PREVIOUS_SHA\" ]; then exit 1; fi; git cat-file -e $VERCEL_GIT_PREVIOUS_SHA 2>/dev/null || exit 1; git diff --quiet $VERCEL_GIT_PREVIOUS_SHA HEAD -- ':!*.md' ':!.planning' ':!docs/' ':!e2e/' ':!scripts/' ':!.github/'", - "rewrites": [ - { "source": "/ingest/static/:path(.*)", "destination": "https://us-assets.i.posthog.com/static/:path" }, - { "source": "/ingest/:path(.*)", "destination": "https://us.i.posthog.com/:path" } - ], "headers": [ { "source": "/(.*)", @@ -13,7 +9,7 @@ { "key": "Strict-Transport-Security", "value": "max-age=63072000; includeSubDomains; preload" }, { "key": "Referrer-Policy", "value": "strict-origin-when-cross-origin" }, { "key": "Permissions-Policy", "value": "camera=(), microphone=(), geolocation=()" }, - { "key": "Content-Security-Policy", "value": "default-src 'self'; connect-src 'self' https: http://localhost:5173 ws: wss: blob: data:; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://www.youtube.com https://static.cloudflareinsights.com https://vercel.live https://us-assets.i.posthog.com; worker-src 'self' blob:; font-src 'self' data: https:; media-src 'self' data: blob: https:; frame-src 'self' https://worldmonitor.app https://tech.worldmonitor.app https://happy.worldmonitor.app https://www.youtube.com https://www.youtube-nocookie.com; frame-ancestors 'self'; base-uri 'self'; object-src 'none'; form-action 'self'" } + { "key": "Content-Security-Policy", "value": "default-src 'self'; connect-src 'self' https: http://localhost:5173 ws: wss: blob: data:; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://www.youtube.com https://static.cloudflareinsights.com https://vercel.live; worker-src 'self' blob:; font-src 'self' data: https:; media-src 'self' data: blob: https:; frame-src 'self' https://worldmonitor.app https://tech.worldmonitor.app https://happy.worldmonitor.app https://www.youtube.com https://www.youtube-nocookie.com; frame-ancestors 'self'; base-uri 'self'; object-src 'none'; form-action 'self'" } ] }, { diff --git a/vite.config.ts b/vite.config.ts index 78aedcd51..25f194802 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -732,18 +732,6 @@ export default defineConfig({ handler: 'NetworkOnly', method: 'POST', }, - { - urlPattern: ({ url, sameOrigin }: { url: URL; sameOrigin: boolean }) => - sameOrigin && /^\/ingest\//.test(url.pathname), - handler: 'NetworkOnly', - method: 'GET', - }, - { - urlPattern: ({ url, sameOrigin }: { url: URL; sameOrigin: boolean }) => - sameOrigin && /^\/ingest\//.test(url.pathname), - handler: 'NetworkOnly', - method: 'POST', - }, { urlPattern: ({ url, sameOrigin }: { url: URL; sameOrigin: boolean }) => sameOrigin && /^\/rss\//.test(url.pathname), diff --git a/web-app-overview.jpg b/web-app-overview.jpg new file mode 100644 index 000000000..6427a2cf8 Binary files /dev/null and b/web-app-overview.jpg differ