Files
worldmonitor/docker/redis-rest-proxy.mjs
Jon Torrez f4183f99c7 feat: self-hosted Docker stack (#1521)
* feat: self-hosted Docker stack with nginx, Redis REST proxy, and seeders

Multi-stage Docker build: esbuild TS handler compilation, vite frontend
build, nginx + Node.js API under supervisord. Upstash-compatible Redis
REST proxy with command allowlist for security. AIS relay WebSocket
sidecar. Seeder wrapper script with auto-sourced env vars from
docker-compose.override.yml. Self-hosting guide with architecture
diagram, API key setup, and troubleshooting.

Security: Redis proxy command allowlist (blocks FLUSHALL/CONFIG/EVAL),
nginx security headers (X-Content-Type-Options, X-Frame-Options,
Referrer-Policy), non-root container user.

* feat(docker): add Docker secrets support for API keys

Entrypoint reads /run/secrets/* files and exports as env vars at
startup. Secrets take priority over environment block values and
stay out of docker inspect / process metadata.

Both methods (env vars and secrets) work simultaneously.

* fix(docker): point supervisord at templated nginx config

The entrypoint runs envsubst on nginx.conf.template and writes
the result to /tmp/nginx.conf (with LOCAL_API_PORT substituted
and listening on port 8080 for non-root). But supervisord was
still launching nginx with /etc/nginx/nginx.conf — the default
Alpine config that listens on port 80, which fails with
"Permission denied" under the non-root appuser.

* fix(docker): remove KEYS from Redis allowlist, fix nginx header inheritance, add LLM vars to seeders

- Remove KEYS from redis-rest-proxy allowlist (O(N) blocking, Redis DoS risk)
- Move security headers into each nginx location block to prevent add_header
  inheritance suppression
- Add LLM_API_URL / LLM_API_KEY / LLM_MODEL to run-seeders.sh grep filter
  so LLM API keys set in docker-compose.override.yml are forwarded to seed scripts

* fix(docker): add path-based POST to Redis proxy, expand allowlist, add missing seeder secrets

- Add POST /{command}/{args...} handler to redis-rest-proxy so Upstash-style
  path POSTs work (setCachedJson uses POST /set/<key>/<value>/EX/<ttl>)
- Expand allowlist: HLEN, LTRIM (seed-military-bases, seed-forecasts),
  ZREVRANGE (premium-stock-store), ZRANDMEMBER (seed-military-bases)
- Add ACLED_EMAIL, ACLED_PASSWORD, OPENROUTER_API_KEY, OLLAMA_API_URL,
  OLLAMA_MODEL to run-seeders.sh so override keys reach host-run seeders

---------

Co-authored-by: Elie Habib <elie.habib@gmail.com>
2026-03-19 12:07:20 +04:00

194 lines
6.2 KiB
JavaScript

#!/usr/bin/env node
/**
* Upstash-compatible Redis REST proxy.
* Translates REST URL paths to raw Redis commands via redis npm package.
*
* Supports:
* GET /{command}/{arg1}/{arg2}/... → Redis command
* POST / → JSON body ["COMMAND", "arg1", ...]
* POST /pipeline → JSON body [["CMD1",...], ["CMD2",...]]
* POST /multi-exec → JSON body [["CMD1",...], ["CMD2",...]]
*
* Env:
* REDIS_URL - Redis connection string (default: redis://redis:6379)
* SRH_TOKEN - Bearer token for auth (default: none)
* PORT - Listen port (default: 80)
*/
import http from 'node:http';
import crypto from 'node:crypto';
import { createClient } from 'redis';
const REDIS_URL = process.env.SRH_CONNECTION_STRING || process.env.REDIS_URL || 'redis://redis:6379';
const TOKEN = process.env.SRH_TOKEN || '';
const PORT = parseInt(process.env.PORT || '80', 10);
const client = createClient({ url: REDIS_URL });
client.on('error', (err) => console.error('Redis error:', err.message));
await client.connect();
console.log(`Connected to Redis at ${REDIS_URL}`);
function checkAuth(req) {
if (!TOKEN) return true;
const auth = req.headers.authorization || '';
const prefix = 'Bearer ';
if (!auth.startsWith(prefix)) return false;
const provided = auth.slice(prefix.length);
if (provided.length !== TOKEN.length) return false;
return crypto.timingSafeEqual(Buffer.from(provided), Buffer.from(TOKEN));
}
// Command safety: allowlist of expected Redis commands.
// Blocks dangerous operations like FLUSHALL, CONFIG SET, EVAL, DEBUG, SLAVEOF.
const ALLOWED_COMMANDS = new Set([
'GET', 'SET', 'DEL', 'MGET', 'MSET', 'SCAN',
'TTL', 'EXPIRE', 'PEXPIRE', 'EXISTS', 'TYPE',
'HGET', 'HSET', 'HDEL', 'HGETALL', 'HMGET', 'HMSET', 'HKEYS', 'HVALS', 'HEXISTS', 'HLEN',
'LPUSH', 'RPUSH', 'LPOP', 'RPOP', 'LRANGE', 'LLEN', 'LTRIM',
'SADD', 'SREM', 'SMEMBERS', 'SISMEMBER', 'SCARD',
'ZADD', 'ZREM', 'ZRANGE', 'ZRANGEBYSCORE', 'ZREVRANGE', 'ZSCORE', 'ZCARD', 'ZRANDMEMBER',
'GEOADD', 'GEOSEARCH', 'GEOPOS', 'GEODIST',
'INCR', 'DECR', 'INCRBY', 'DECRBY',
'PING', 'ECHO', 'INFO', 'DBSIZE',
'PUBLISH', 'SUBSCRIBE',
'SETNX', 'SETEX', 'PSETEX', 'GETSET',
'APPEND', 'STRLEN',
]);
async function runCommand(args) {
const cmd = args[0].toUpperCase();
if (!ALLOWED_COMMANDS.has(cmd)) {
throw new Error(`Command not allowed: ${cmd}`);
}
const cmdArgs = args.slice(1);
return client.sendCommand([cmd, ...cmdArgs.map(String)]);
}
const MAX_BODY_BYTES = 1024 * 1024; // 1 MB
async function readBody(req) {
const chunks = [];
let totalLength = 0;
for await (const chunk of req) {
totalLength += chunk.length;
if (totalLength > MAX_BODY_BYTES) {
req.destroy();
throw new Error('Request body too large');
}
chunks.push(chunk);
}
return Buffer.concat(chunks).toString();
}
const server = http.createServer(async (req, res) => {
res.setHeader('content-type', 'application/json');
if (!checkAuth(req)) {
res.writeHead(401);
res.end(JSON.stringify({ error: 'Unauthorized' }));
return;
}
try {
// POST / — single command
if (req.method === 'POST' && (req.url === '/' || req.url === '')) {
const body = JSON.parse(await readBody(req));
const result = await runCommand(body);
res.writeHead(200);
res.end(JSON.stringify({ result }));
return;
}
// POST /pipeline — batch commands
if (req.method === 'POST' && req.url === '/pipeline') {
const commands = JSON.parse(await readBody(req));
const results = [];
for (const cmd of commands) {
try {
const result = await runCommand(cmd);
results.push({ result });
} catch (err) {
results.push({ error: err.message });
}
}
res.writeHead(200);
res.end(JSON.stringify(results));
return;
}
// POST /multi-exec — transaction
if (req.method === 'POST' && req.url === '/multi-exec') {
const commands = JSON.parse(await readBody(req));
const multi = client.multi();
for (const cmd of commands) {
const cmdName = cmd[0].toUpperCase();
if (!ALLOWED_COMMANDS.has(cmdName)) {
res.writeHead(403);
res.end(JSON.stringify({ error: `Command not allowed: ${cmdName}` }));
return;
}
multi.sendCommand(cmd.map(String));
}
const results = await multi.exec();
res.writeHead(200);
res.end(JSON.stringify(results.map((r) => ({ result: r }))));
return;
}
// GET / — welcome
if (req.method === 'GET' && (req.url === '/' || req.url === '')) {
res.writeHead(200);
res.end('"Welcome to Serverless Redis HTTP!"');
return;
}
// GET /{command}/{args...} — REST style
if (req.method === 'GET') {
const pathname = new URL(req.url, 'http://localhost').pathname;
const parts = pathname.slice(1).split('/').map(decodeURIComponent);
if (parts.length === 0 || !parts[0]) {
res.writeHead(400);
res.end(JSON.stringify({ error: 'No command specified' }));
return;
}
const result = await runCommand(parts);
res.writeHead(200);
res.end(JSON.stringify({ result }));
return;
}
// POST /{command}/{args...} — Upstash-compatible path-based POST
// Used by setCachedJson(): POST /set/<key>/<value>/EX/<ttl>
if (req.method === 'POST') {
const pathname = new URL(req.url, 'http://localhost').pathname;
const parts = pathname.slice(1).split('/').map(decodeURIComponent);
if (parts.length === 0 || !parts[0]) {
res.writeHead(400);
res.end(JSON.stringify({ error: 'No command specified' }));
return;
}
const result = await runCommand(parts);
res.writeHead(200);
res.end(JSON.stringify({ result }));
return;
}
// OPTIONS
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
res.writeHead(404);
res.end(JSON.stringify({ error: 'Not found' }));
} catch (err) {
res.writeHead(500);
res.end(JSON.stringify({ error: err.message }));
}
});
server.listen(PORT, '0.0.0.0', () => {
console.log(`Redis REST proxy listening on 0.0.0.0:${PORT}`);
});