mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
9d94ad36aa
* feat(climate+health):add shared air quality seed and mirrored health/climate RPCs * feat(climate+health):add shared air quality seed and mirrored health/climate RPCs * fix(air-quality): address review findings — TTL, seed-health, FAST_KEYS, shared meta - Raise CACHE_TTL from 3600 to 10800 (3× the 1h cron cadence; gold standard) - Add health:air-quality to api/seed-health.js SEED_DOMAINS so monitoring dashboard tracks freshness - Remove climateAirQuality and healthAirQuality from FAST_KEYS (large station payloads; load in slow batch) - Point climateAirQuality SEED_META to same meta key as healthAirQuality (same seeder run, one source of truth) * fix(bootstrap): move air quality keys to SLOW tier — large station payloads avoid critical-path batch * fix(air-quality): fix malformed OpenAQ URL and remove from bootstrap until panel exists - Drop deprecated first URL attempt (parameters=pm25, order_by=lastUpdated, sort=desc); use correct v3 params (parameters_id=2, sort_order=desc) directly — eliminates guaranteed 4xx retry cycle per page on 20-page crawl - Remove climateAirQuality and healthAirQuality from BOOTSTRAP_CACHE_KEYS, SLOW_KEYS, and BOOTSTRAP_TIERS — no panel consumes these yet; adding thousands of station records to every startup bootstrap is pure payload bloat - Remove normalizeAirQualityPayload helpers from bootstrap.js (no longer called) - Update service wrappers to fetch via RPC directly; re-add bootstrap hydration when a panel actually needs it * fix(air-quality): raise lock TTL to 3600s to cover 20-page crawl worst case 2 OpenAQ calls × 20 pages × (30s timeout × 3 attempts) = 3600s max runtime. Previous 600s TTL allowed concurrent cron runs on any degraded upstream. --------- Co-authored-by: Elie Habib <elie.habib@gmail.com>
53 lines
1.7 KiB
Bash
Executable File
53 lines
1.7 KiB
Bash
Executable File
#!/bin/sh
|
|
# Run all seed scripts against the local Redis REST proxy.
|
|
# Usage: ./scripts/run-seeders.sh
|
|
#
|
|
# Requires the worldmonitor stack to be running (uvx podman-compose up -d).
|
|
# The Redis REST proxy listens on localhost:8079 by default.
|
|
|
|
UPSTASH_REDIS_REST_URL="${UPSTASH_REDIS_REST_URL:-http://localhost:8079}"
|
|
UPSTASH_REDIS_REST_TOKEN="${UPSTASH_REDIS_REST_TOKEN:-wm-local-token}"
|
|
export UPSTASH_REDIS_REST_URL UPSTASH_REDIS_REST_TOKEN
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
# Source API keys from docker-compose.override.yml if present.
|
|
# These keys are configured for the container but seeders run on the host.
|
|
OVERRIDE="$PROJECT_DIR/docker-compose.override.yml"
|
|
if [ -f "$OVERRIDE" ]; then
|
|
_env_tmp=$(mktemp)
|
|
grep -E '^\s+[A-Z_]+:' "$OVERRIDE" \
|
|
| grep -v '#' \
|
|
| sed 's/^\s*//' \
|
|
| sed 's/: */=/' \
|
|
| sed "s/[\"']//g" \
|
|
| grep -E '^(NASA_FIRMS|GROQ|AISSTREAM|FRED|FINNHUB|EIA|ACLED_ACCESS_TOKEN|ACLED_EMAIL|ACLED_PASSWORD|CLOUDFLARE|AVIATIONSTACK|OPENAQ_API_KEY|WAQI_API_KEY|OPENROUTER_API_KEY|LLM_API_URL|LLM_API_KEY|LLM_MODEL|OLLAMA_API_URL|OLLAMA_MODEL)' \
|
|
| sed 's/^/export /' > "$_env_tmp"
|
|
. "$_env_tmp"
|
|
rm -f "$_env_tmp"
|
|
fi
|
|
ok=0 fail=0 skip=0
|
|
|
|
for f in "$SCRIPT_DIR"/seed-*.mjs; do
|
|
name="$(basename "$f")"
|
|
printf "→ %s ... " "$name"
|
|
output=$(node "$f" 2>&1)
|
|
rc=$?
|
|
last=$(echo "$output" | tail -1)
|
|
|
|
if echo "$last" | grep -qi "skip\|not set\|missing.*key\|not found"; then
|
|
printf "SKIP (%s)\n" "$last"
|
|
skip=$((skip + 1))
|
|
elif [ $rc -eq 0 ]; then
|
|
printf "OK\n"
|
|
ok=$((ok + 1))
|
|
else
|
|
printf "FAIL (%s)\n" "$last"
|
|
fail=$((fail + 1))
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "Done: $ok ok, $skip skipped, $fail failed"
|