mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(sentiment): add AAII investor sentiment survey (#2930)
* feat(sentiment): add AAII investor sentiment survey Weekly bull/bear/neutral sentiment from AAII (1987-present). Shows current reading, bull-bear spread, and 52-week historical chart. Seeder fetches from AAII CSV, stores last 52 weeks in Redis. * fix(aaii): wire panel loading + mark fallback data explicitly * fix(aaii): keep panel live across refreshes + surface in health monitoring - fetchData now falls back to /api/bootstrap?keys=aaiiSentiment on refresh (getHydratedData is one-shot and returns undefined after the first read, causing a permanent spinner on hourly refresh) - Shows an error state with auto-retry when both hydrated and bootstrap-fetch miss, matching the WsbTickerScannerPanel pattern - Registered aaiiSentiment in api/health.js BOOTSTRAP_KEYS and api/seed-health.js SEED_DOMAINS so rollout failures and fallback-only operation are observable in the monitoring dashboards * fix(sentiment): handle BIFF8 SST trailing bytes and use UTC for AAII Thursday calc Two P2 greptile fixes from PR #2930 review: 1. BIFF8 SST parser was reading the rich-text run count (cRun, flags & 0x08) and extended-string size (cbExtRst, flags & 0x04) to advance past those header fields, but never skipped the trailing bytes AFTER the char data: 4 * cRun formatting-run bytes and cbExtRst ext-rst bytes. If any string before the column header was rich-text formatted, every subsequent SST entry parsed from the wrong offset, silently breaking XLS extraction and falling back to HTML scraping. 2. parseHtmlSentiment() computed last-Thursday via today.getDay() + setDate(today.getDate() - daysToThursday), both local-TZ-dependent. On Railway (non-UTC TZ) the inferred Thursday could drift by a day, causing the HTML-derived row to mismatch the XLS historical rows. Switched to getUTCDay() + Date.UTC() for TZ-stable arithmetic.
This commit is contained in:
Elie Habib
2
api/bootstrap.js
vendored
2
api/bootstrap.js
vendored
@@ -96,6 +96,7 @@ const BOOTSTRAP_CACHE_KEYS = {
|
||||
oilStocksAnalysis: 'energy:oil-stocks-analysis:v1',
|
||||
lngVulnerability: 'energy:lng-vulnerability:v1',
|
||||
sprPolicies: 'energy:spr-policies:v1',
|
||||
aaiiSentiment: 'market:aaii-sentiment:v1',
|
||||
};
|
||||
|
||||
const SLOW_KEYS = new Set([
|
||||
@@ -135,6 +136,7 @@ const SLOW_KEYS = new Set([
|
||||
'oilStocksAnalysis',
|
||||
'lngVulnerability',
|
||||
'sprPolicies',
|
||||
'aaiiSentiment',
|
||||
]);
|
||||
const FAST_KEYS = new Set([
|
||||
'earthquakes', 'outages', 'serviceStatuses', 'ddosAttacks', 'trafficAnomalies', 'macroSignals', 'chokepoints',
|
||||
|
||||
@@ -87,6 +87,7 @@ const BOOTSTRAP_KEYS = {
|
||||
vpdTrackerHistorical: 'health:vpd-tracker:historical:v1',
|
||||
electricityPrices: 'energy:electricity:v1:index',
|
||||
gasStorageCountries: 'energy:gas-storage:v1:_countries',
|
||||
aaiiSentiment: 'market:aaii-sentiment:v1',
|
||||
};
|
||||
|
||||
const STANDALONE_KEYS = {
|
||||
@@ -289,6 +290,7 @@ const SEED_META = {
|
||||
lngVulnerability: { key: 'seed-meta:energy:jodi-gas', maxStaleMin: 60 * 24 * 40 }, // written by jodi-gas seeder afterPublish; shares seed-meta key
|
||||
chokepointBaselines: { key: 'seed-meta:energy:chokepoint-baselines', maxStaleMin: 60 * 24 * 400 }, // 400 days
|
||||
sprPolicies: { key: 'seed-meta:energy:spr-policies', maxStaleMin: 60 * 24 * 400 }, // 400 days; static registry, same cadence as chokepoint baselines
|
||||
aaiiSentiment: { key: 'seed-meta:market:aaii-sentiment', maxStaleMin: 20160 }, // weekly cron; 20160min = 14 days = 2x weekly cadence
|
||||
portwatchChokepointsRef: { key: 'seed-meta:portwatch:chokepoints-ref', maxStaleMin: 60 * 24 * 2 }, // daily cron; 2d = 2× interval
|
||||
chokepointFlows: { key: 'seed-meta:energy:chokepoint-flows', maxStaleMin: 720 }, // 6h cron; 720min = 2x interval
|
||||
emberElectricity: { key: 'seed-meta:energy:ember', maxStaleMin: 2880 }, // daily cron (08:00 UTC); 2880min = 48h = 2x interval
|
||||
|
||||
@@ -77,6 +77,7 @@ const SEED_DOMAINS = {
|
||||
'energy:spine': { key: 'seed-meta:energy:spine', intervalMin: 1440 }, // daily cron (0 6 * * *); intervalMin = maxStaleMin / 2 (2880 / 2)
|
||||
'energy:ember': { key: 'seed-meta:energy:ember', intervalMin: 1440 }, // daily cron (0 8 * * *); intervalMin = maxStaleMin / 2 (2880 / 2)
|
||||
'energy:spr-policies': { key: 'seed-meta:energy:spr-policies', intervalMin: 288000 }, // annual static registry; intervalMin = health.js maxStaleMin / 2 (576000 / 2)
|
||||
'market:aaii-sentiment': { key: 'seed-meta:market:aaii-sentiment', intervalMin: 10080 }, // weekly cron; intervalMin = maxStaleMin / 2 (20160 / 2)
|
||||
};
|
||||
|
||||
async function getMetaBatch(keys) {
|
||||
|
||||
404
scripts/seed-aaii-sentiment.mjs
Normal file
404
scripts/seed-aaii-sentiment.mjs
Normal file
@@ -0,0 +1,404 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs';
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const AAII_KEY = 'market:aaii-sentiment:v1';
|
||||
const AAII_TTL = 604800; // 7 days (weekly data)
|
||||
|
||||
const AAII_XLS_URL = 'https://www.aaii.com/files/surveys/sentiment.xls';
|
||||
const AAII_HTML_URL = 'https://www.aaii.com/sentimentsurvey';
|
||||
|
||||
export function parseXlsRows(buffer) {
|
||||
const rows = [];
|
||||
const bytes = new Uint8Array(buffer);
|
||||
const len = bytes.length;
|
||||
|
||||
const strings = [];
|
||||
let i = 0;
|
||||
while (i < len - 4) {
|
||||
const recType = bytes[i] | (bytes[i + 1] << 8);
|
||||
const recLen = bytes[i + 2] | (bytes[i + 3] << 8);
|
||||
if (recLen > 100000 || recLen < 0) { i++; continue; }
|
||||
|
||||
// SST record (shared string table)
|
||||
if (recType === 0x00FC && recLen > 8) {
|
||||
let pos = i + 4 + 8; // skip total/unique counts
|
||||
while (pos < i + 4 + recLen && strings.length < 10000) {
|
||||
if (pos + 3 > len) break;
|
||||
const charCount = bytes[pos] | (bytes[pos + 1] << 8);
|
||||
const flags = bytes[pos + 2];
|
||||
pos += 3;
|
||||
let cRun = 0;
|
||||
let cbExtRst = 0;
|
||||
if (flags & 0x08) {
|
||||
// rich text: cRun (u16) = number of formatting runs (4 bytes each) after char data
|
||||
cRun = bytes[pos] | (bytes[pos + 1] << 8);
|
||||
pos += 2;
|
||||
}
|
||||
if (flags & 0x04) {
|
||||
// extended string: cbExtRst (u32) = byte length of ext-rst block after char data
|
||||
cbExtRst = bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24);
|
||||
pos += 4;
|
||||
}
|
||||
if (flags & 0x01) {
|
||||
// UTF-16
|
||||
const strBytes = charCount * 2;
|
||||
if (pos + strBytes > len) break;
|
||||
let s = '';
|
||||
for (let j = 0; j < charCount; j++) {
|
||||
s += String.fromCharCode(bytes[pos + j * 2] | (bytes[pos + j * 2 + 1] << 8));
|
||||
}
|
||||
strings.push(s);
|
||||
pos += strBytes;
|
||||
} else {
|
||||
if (pos + charCount > len) break;
|
||||
let s = '';
|
||||
for (let j = 0; j < charCount; j++) s += String.fromCharCode(bytes[pos + j]);
|
||||
strings.push(s);
|
||||
pos += charCount;
|
||||
}
|
||||
// Skip trailing formatting-run and ext-rst bytes (BIFF8 spec)
|
||||
if (flags & 0x08) pos += 4 * cRun;
|
||||
if (flags & 0x04) pos += cbExtRst;
|
||||
}
|
||||
}
|
||||
i += 4 + recLen;
|
||||
}
|
||||
|
||||
// Extract NUMBER records (type 0x0203) and RK records (type 0x027E)
|
||||
// and LABEL/SST refs for dates
|
||||
const cells = new Map(); // "row,col" -> value
|
||||
i = 0;
|
||||
while (i < len - 4) {
|
||||
const recType = bytes[i] | (bytes[i + 1] << 8);
|
||||
const recLen = bytes[i + 2] | (bytes[i + 3] << 8);
|
||||
if (recLen > 100000 || recLen < 0) { i++; continue; }
|
||||
|
||||
if (recType === 0x0203 && recLen >= 14) { // NUMBER
|
||||
const row = bytes[i + 4] | (bytes[i + 5] << 8);
|
||||
const col = bytes[i + 6] | (bytes[i + 7] << 8);
|
||||
const buf8 = new ArrayBuffer(8);
|
||||
const view = new DataView(buf8);
|
||||
for (let j = 0; j < 8; j++) view.setUint8(j, bytes[i + 10 + j]);
|
||||
const val = view.getFloat64(0, true);
|
||||
cells.set(`${row},${col}`, val);
|
||||
} else if (recType === 0x027E && recLen >= 10) { // RK
|
||||
const row = bytes[i + 4] | (bytes[i + 5] << 8);
|
||||
const col = bytes[i + 6] | (bytes[i + 7] << 8);
|
||||
const rkVal = bytes[i + 10] | (bytes[i + 11] << 8) | (bytes[i + 12] << 16) | (bytes[i + 13] << 24);
|
||||
let val;
|
||||
if (rkVal & 0x02) {
|
||||
val = (rkVal >> 2);
|
||||
} else {
|
||||
const buf8 = new ArrayBuffer(8);
|
||||
const view = new DataView(buf8);
|
||||
view.setInt32(4, rkVal & 0xFFFFFFFC, true);
|
||||
val = view.getFloat64(0, true);
|
||||
}
|
||||
if (rkVal & 0x01) val /= 100;
|
||||
cells.set(`${row},${col}`, val);
|
||||
} else if (recType === 0x00FD && recLen >= 10) { // LABELSST
|
||||
const row = bytes[i + 4] | (bytes[i + 5] << 8);
|
||||
const col = bytes[i + 6] | (bytes[i + 7] << 8);
|
||||
const sstIdx = bytes[i + 10] | (bytes[i + 11] << 8) | (bytes[i + 12] << 16) | (bytes[i + 13] << 24);
|
||||
if (sstIdx < strings.length) {
|
||||
cells.set(`${row},${col}`, strings[sstIdx]);
|
||||
}
|
||||
}
|
||||
i += 4 + recLen;
|
||||
}
|
||||
|
||||
if (cells.size === 0) return rows;
|
||||
|
||||
// Find max row/col
|
||||
let maxRow = 0, maxCol = 0;
|
||||
for (const key of cells.keys()) {
|
||||
const [r, c] = key.split(',').map(Number);
|
||||
if (r > maxRow) maxRow = r;
|
||||
if (c > maxCol) maxCol = c;
|
||||
}
|
||||
|
||||
// Build row arrays (first 10 columns, first 2000 rows max)
|
||||
const limit = Math.min(maxRow + 1, 2000);
|
||||
const colLimit = Math.min(maxCol + 1, 10);
|
||||
for (let r = 0; r < limit; r++) {
|
||||
const row = [];
|
||||
for (let c = 0; c < colLimit; c++) {
|
||||
row.push(cells.get(`${r},${c}`) ?? null);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function excelDateToISO(serial) {
|
||||
if (typeof serial !== 'number' || serial < 1) return null;
|
||||
// Excel serial: 1 = Jan 1, 1900. Lotus 1-2-3 bug: serial 60 = fake Feb 29, 1900.
|
||||
// For serial > 59: real days from Jan 1, 1900 = serial - 2
|
||||
// For serial <= 59: real days from Jan 1, 1900 = serial - 1
|
||||
const daysFromJan1 = serial > 59 ? serial - 2 : serial - 1;
|
||||
const d = new Date(Date.UTC(1900, 0, 1 + daysFromJan1));
|
||||
const y = d.getUTCFullYear();
|
||||
const m = String(d.getUTCMonth() + 1).padStart(2, '0');
|
||||
const day = String(d.getUTCDate()).padStart(2, '0');
|
||||
return `${y}-${m}-${day}`;
|
||||
}
|
||||
|
||||
export function extractSentimentData(rows) {
|
||||
// Find header row containing "Bullish" / "Bearish" / "Neutral"
|
||||
let headerIdx = -1;
|
||||
let bullCol = -1, neutralCol = -1, bearCol = -1, dateCol = -1, spreadCol = -1, sp500CloseCol = -1;
|
||||
|
||||
for (let r = 0; r < Math.min(rows.length, 20); r++) {
|
||||
const row = rows[r];
|
||||
for (let c = 0; c < row.length; c++) {
|
||||
const v = String(row[c] ?? '').toLowerCase().trim();
|
||||
if (v === 'bullish') { bullCol = c; headerIdx = r; }
|
||||
if (v === 'neutral') { neutralCol = c; }
|
||||
if (v === 'bearish') { bearCol = c; }
|
||||
if (v.includes('bull-bear') || v.includes('spread')) { spreadCol = c; }
|
||||
if (v.includes('close') || v.includes('s&p') || v.includes('sp 500')) { sp500CloseCol = c; }
|
||||
}
|
||||
if (headerIdx === r) {
|
||||
dateCol = 0; // date is always first column
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (headerIdx < 0 || bullCol < 0 || bearCol < 0) return [];
|
||||
|
||||
const data = [];
|
||||
for (let r = headerIdx + 1; r < rows.length; r++) {
|
||||
const row = rows[r];
|
||||
const rawDate = row[dateCol];
|
||||
const bull = typeof row[bullCol] === 'number' ? row[bullCol] : null;
|
||||
const bear = typeof row[bearCol] === 'number' ? row[bearCol] : null;
|
||||
const neutral = neutralCol >= 0 && typeof row[neutralCol] === 'number' ? row[neutralCol] : null;
|
||||
|
||||
if (bull == null || bear == null) continue;
|
||||
|
||||
let date;
|
||||
if (typeof rawDate === 'number' && rawDate > 30000) {
|
||||
date = excelDateToISO(rawDate);
|
||||
} else if (typeof rawDate === 'string') {
|
||||
const parsed = new Date(rawDate);
|
||||
if (!isNaN(parsed.getTime())) {
|
||||
date = parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
if (!date) continue;
|
||||
|
||||
// Convert fractions to percentages if needed
|
||||
const bullPct = bull > 1 ? bull : bull * 100;
|
||||
const bearPct = bear > 1 ? bear : bear * 100;
|
||||
const neutralPct = neutral != null ? (neutral > 1 ? neutral : neutral * 100) : +(100 - bullPct - bearPct).toFixed(1);
|
||||
const spread = +(bullPct - bearPct).toFixed(1);
|
||||
|
||||
data.push({
|
||||
date,
|
||||
bullish: +bullPct.toFixed(1),
|
||||
bearish: +bearPct.toFixed(1),
|
||||
neutral: +neutralPct.toFixed(1),
|
||||
spread,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by date descending (most recent first)
|
||||
data.sort((a, b) => b.date.localeCompare(a.date));
|
||||
return data;
|
||||
}
|
||||
|
||||
export function parseHtmlSentiment(html) {
|
||||
const rows = [];
|
||||
// Match table rows with percentage data
|
||||
// The AAII page has tableTxt cells: Bullish%, Neutral%, Bearish%
|
||||
const pcts = [...html.matchAll(/<td[^>]*class="tableTxt"[^>]*>([\d.]+)%/g)]
|
||||
.map(m => parseFloat(m[1]));
|
||||
|
||||
if (pcts.length >= 3) {
|
||||
// AAII publishes on Thursdays; find the most recent Thursday in UTC
|
||||
// (local-TZ arithmetic can drift by a day on Railway when TZ != UTC)
|
||||
const nowUTC = new Date();
|
||||
const dayOfWeek = nowUTC.getUTCDay();
|
||||
const daysToThursday = (dayOfWeek >= 4) ? dayOfWeek - 4 : dayOfWeek + 3;
|
||||
const tsThursday = Date.UTC(nowUTC.getUTCFullYear(), nowUTC.getUTCMonth(), nowUTC.getUTCDate() - daysToThursday);
|
||||
const date = new Date(tsThursday).toISOString().slice(0, 10);
|
||||
|
||||
rows.push({
|
||||
date,
|
||||
bullish: pcts[0],
|
||||
neutral: pcts[1],
|
||||
bearish: pcts[2],
|
||||
spread: +(pcts[0] - pcts[2]).toFixed(1),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
const FALLBACK_DATA = [
|
||||
{ date: '2026-04-03', bullish: 35.7, bearish: 43.0, neutral: 21.3, spread: -7.3 },
|
||||
{ date: '2026-03-27', bullish: 22.4, bearish: 55.8, neutral: 21.8, spread: -33.4 },
|
||||
{ date: '2026-03-20', bullish: 19.2, bearish: 57.1, neutral: 23.7, spread: -37.9 },
|
||||
{ date: '2026-03-13', bullish: 20.5, bearish: 54.5, neutral: 25.0, spread: -34.0 },
|
||||
{ date: '2026-03-06', bullish: 19.4, bearish: 59.2, neutral: 21.4, spread: -39.8 },
|
||||
{ date: '2026-02-27', bullish: 22.8, bearish: 52.2, neutral: 25.0, spread: -29.4 },
|
||||
{ date: '2026-02-20', bullish: 31.3, bearish: 44.1, neutral: 24.6, spread: -12.8 },
|
||||
{ date: '2026-02-13', bullish: 36.1, bearish: 41.0, neutral: 22.9, spread: -4.9 },
|
||||
{ date: '2026-02-06', bullish: 29.2, bearish: 40.9, neutral: 29.9, spread: -11.7 },
|
||||
{ date: '2026-01-30', bullish: 33.3, bearish: 37.5, neutral: 29.2, spread: -4.2 },
|
||||
{ date: '2026-01-23', bullish: 25.4, bearish: 40.6, neutral: 34.0, spread: -15.2 },
|
||||
{ date: '2026-01-16', bullish: 34.7, bearish: 29.4, neutral: 35.9, spread: 5.3 },
|
||||
{ date: '2026-01-09', bullish: 38.4, bearish: 34.0, neutral: 27.6, spread: 4.4 },
|
||||
{ date: '2026-01-02', bullish: 43.1, bearish: 25.3, neutral: 31.6, spread: 17.8 },
|
||||
{ date: '2025-12-26', bullish: 37.9, bearish: 34.1, neutral: 28.0, spread: 3.8 },
|
||||
{ date: '2025-12-19', bullish: 40.2, bearish: 30.4, neutral: 29.4, spread: 9.8 },
|
||||
{ date: '2025-12-12', bullish: 48.3, bearish: 23.7, neutral: 28.0, spread: 24.6 },
|
||||
{ date: '2025-12-05', bullish: 45.5, bearish: 27.5, neutral: 27.0, spread: 18.0 },
|
||||
{ date: '2025-11-28', bullish: 49.8, bearish: 22.1, neutral: 28.1, spread: 27.7 },
|
||||
{ date: '2025-11-21', bullish: 47.3, bearish: 25.7, neutral: 27.0, spread: 21.6 },
|
||||
{ date: '2025-11-14', bullish: 50.8, bearish: 20.3, neutral: 28.9, spread: 30.5 },
|
||||
{ date: '2025-11-07', bullish: 49.8, bearish: 22.1, neutral: 28.1, spread: 27.7 },
|
||||
{ date: '2025-10-31', bullish: 37.7, bearish: 31.8, neutral: 30.5, spread: 5.9 },
|
||||
{ date: '2025-10-24', bullish: 40.6, bearish: 28.2, neutral: 31.2, spread: 12.4 },
|
||||
{ date: '2025-10-17', bullish: 45.5, bearish: 25.6, neutral: 28.9, spread: 19.9 },
|
||||
{ date: '2025-10-10', bullish: 49.0, bearish: 24.6, neutral: 26.4, spread: 24.4 },
|
||||
{ date: '2025-10-03', bullish: 45.3, bearish: 25.2, neutral: 29.5, spread: 20.1 },
|
||||
{ date: '2025-09-26', bullish: 42.2, bearish: 27.0, neutral: 30.8, spread: 15.2 },
|
||||
{ date: '2025-09-19', bullish: 46.3, bearish: 24.5, neutral: 29.2, spread: 21.8 },
|
||||
{ date: '2025-09-12', bullish: 44.4, bearish: 26.1, neutral: 29.5, spread: 18.3 },
|
||||
{ date: '2025-09-05', bullish: 38.6, bearish: 28.4, neutral: 33.0, spread: 10.2 },
|
||||
{ date: '2025-08-29', bullish: 41.2, bearish: 27.0, neutral: 31.8, spread: 14.2 },
|
||||
{ date: '2025-08-22', bullish: 40.0, bearish: 30.1, neutral: 29.9, spread: 9.9 },
|
||||
{ date: '2025-08-15', bullish: 41.1, bearish: 26.8, neutral: 32.1, spread: 14.3 },
|
||||
{ date: '2025-08-08', bullish: 44.7, bearish: 29.1, neutral: 26.2, spread: 15.6 },
|
||||
{ date: '2025-08-01', bullish: 33.7, bearish: 37.5, neutral: 28.8, spread: -3.8 },
|
||||
{ date: '2025-07-25', bullish: 36.0, bearish: 29.4, neutral: 34.6, spread: 6.6 },
|
||||
{ date: '2025-07-18', bullish: 40.3, bearish: 27.4, neutral: 32.3, spread: 12.9 },
|
||||
{ date: '2025-07-11', bullish: 42.5, bearish: 25.0, neutral: 32.5, spread: 17.5 },
|
||||
{ date: '2025-07-04', bullish: 46.2, bearish: 27.6, neutral: 26.2, spread: 18.6 },
|
||||
{ date: '2025-06-27', bullish: 40.9, bearish: 31.1, neutral: 28.0, spread: 9.8 },
|
||||
{ date: '2025-06-20', bullish: 44.6, bearish: 26.5, neutral: 28.9, spread: 18.1 },
|
||||
{ date: '2025-06-13', bullish: 44.0, bearish: 25.6, neutral: 30.4, spread: 18.4 },
|
||||
{ date: '2025-06-06', bullish: 41.0, bearish: 28.0, neutral: 31.0, spread: 13.0 },
|
||||
{ date: '2025-05-30', bullish: 41.3, bearish: 32.3, neutral: 26.4, spread: 9.0 },
|
||||
{ date: '2025-05-23', bullish: 36.1, bearish: 33.3, neutral: 30.6, spread: 2.8 },
|
||||
{ date: '2025-05-16', bullish: 39.1, bearish: 31.0, neutral: 29.9, spread: 8.1 },
|
||||
{ date: '2025-05-09', bullish: 36.0, bearish: 37.5, neutral: 26.5, spread: -1.5 },
|
||||
{ date: '2025-05-02', bullish: 28.5, bearish: 44.7, neutral: 26.8, spread: -16.2 },
|
||||
{ date: '2025-04-25', bullish: 25.3, bearish: 52.2, neutral: 22.5, spread: -26.9 },
|
||||
{ date: '2025-04-18', bullish: 21.8, bearish: 55.6, neutral: 22.6, spread: -33.8 },
|
||||
{ date: '2025-04-11', bullish: 28.5, bearish: 52.1, neutral: 19.4, spread: -23.6 },
|
||||
];
|
||||
|
||||
async function fetchAaiiSentiment() {
|
||||
let data = [];
|
||||
let source = 'fallback';
|
||||
|
||||
// Strategy 1: Fetch the XLS file and parse it
|
||||
try {
|
||||
console.log(' Attempting XLS download...');
|
||||
const resp = await fetch(AAII_XLS_URL, {
|
||||
headers: { 'User-Agent': CHROME_UA, Accept: 'application/vnd.ms-excel,*/*' },
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (resp.ok) {
|
||||
const buffer = await resp.arrayBuffer();
|
||||
console.log(` XLS downloaded: ${(buffer.byteLength / 1024).toFixed(0)} KB`);
|
||||
const rows = parseXlsRows(buffer);
|
||||
console.log(` XLS parsed: ${rows.length} raw rows`);
|
||||
if (rows.length > 10) {
|
||||
data = extractSentimentData(rows);
|
||||
if (data.length > 0) {
|
||||
source = 'xls';
|
||||
console.log(` XLS extracted: ${data.length} sentiment rows`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.warn(` XLS fetch: HTTP ${resp.status}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(` XLS fetch failed: ${e.message}`);
|
||||
}
|
||||
|
||||
// Strategy 2: Scrape the HTML page for current reading
|
||||
if (data.length === 0) {
|
||||
try {
|
||||
console.log(' Attempting HTML scrape...');
|
||||
const resp = await fetch(AAII_HTML_URL, {
|
||||
headers: { 'User-Agent': CHROME_UA, Accept: 'text/html,application/xhtml+xml' },
|
||||
signal: AbortSignal.timeout(8_000),
|
||||
});
|
||||
if (resp.ok) {
|
||||
const html = await resp.text();
|
||||
data = parseHtmlSentiment(html);
|
||||
if (data.length > 0) {
|
||||
source = 'html';
|
||||
console.log(` HTML scraped: ${data.length} rows`);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(` HTML scrape failed: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: Use fallback data
|
||||
const isFallback = data.length === 0;
|
||||
if (isFallback) {
|
||||
console.log(' Using fallback data');
|
||||
data = FALLBACK_DATA;
|
||||
source = 'fallback';
|
||||
}
|
||||
|
||||
// Keep last 52 weeks
|
||||
const weeks = data.slice(0, 52);
|
||||
const latest = weeks[0];
|
||||
const prev = weeks.length > 1 ? weeks[1] : null;
|
||||
const historicalAvg = { bullish: 37.5, bearish: 31.0, neutral: 31.5 };
|
||||
|
||||
// Compute rolling averages
|
||||
const last8 = weeks.slice(0, 8);
|
||||
const avg8w = last8.length > 0 ? {
|
||||
bullish: +(last8.reduce((s, w) => s + w.bullish, 0) / last8.length).toFixed(1),
|
||||
bearish: +(last8.reduce((s, w) => s + w.bearish, 0) / last8.length).toFixed(1),
|
||||
neutral: +(last8.reduce((s, w) => s + w.neutral, 0) / last8.length).toFixed(1),
|
||||
spread: +(last8.reduce((s, w) => s + w.spread, 0) / last8.length).toFixed(1),
|
||||
} : null;
|
||||
|
||||
const extremeSpreads = weeks.filter(w => w.spread <= -20).length;
|
||||
const bullishExtremes = weeks.filter(w => w.bullish >= 50).length;
|
||||
const bearishExtremes = weeks.filter(w => w.bearish >= 50).length;
|
||||
|
||||
return {
|
||||
seededAt: isFallback ? new Date(latest.date + 'T12:00:00Z').toISOString() : new Date().toISOString(),
|
||||
fallback: isFallback,
|
||||
source,
|
||||
latest,
|
||||
previous: prev,
|
||||
avg8w,
|
||||
historicalAvg,
|
||||
extremes: {
|
||||
spreadBelow20: extremeSpreads,
|
||||
bullishAbove50: bullishExtremes,
|
||||
bearishAbove50: bearishExtremes,
|
||||
},
|
||||
weeks,
|
||||
};
|
||||
}
|
||||
|
||||
function validate(data) {
|
||||
return data?.latest?.bullish != null && data?.weeks?.length > 0;
|
||||
}
|
||||
|
||||
const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^.*[\\/]/, ''));
|
||||
if (isMain) {
|
||||
runSeed('market', 'aaii-sentiment', AAII_KEY, fetchAaiiSentiment, {
|
||||
validateFn: validate,
|
||||
ttlSeconds: AAII_TTL,
|
||||
recordCount: (data) => data?.weeks?.length ?? 0,
|
||||
sourceVersion: 'aaii-xls-html-v1',
|
||||
}).catch((err) => {
|
||||
console.error('FATAL:', err.message || err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -181,6 +181,7 @@ export const BOOTSTRAP_CACHE_KEYS: Record<string, string> = {
|
||||
oilStocksAnalysis: 'energy:oil-stocks-analysis:v1',
|
||||
lngVulnerability: 'energy:lng-vulnerability:v1',
|
||||
sprPolicies: 'energy:spr-policies:v1',
|
||||
aaiiSentiment: 'market:aaii-sentiment:v1',
|
||||
};
|
||||
|
||||
export const PORTWATCH_PORT_ACTIVITY_KEY_PREFIX = 'supply_chain:portwatch-ports:v1:';
|
||||
@@ -237,6 +238,7 @@ export const BOOTSTRAP_TIERS: Record<string, 'slow' | 'fast'> = {
|
||||
oilStocksAnalysis: 'slow',
|
||||
lngVulnerability: 'slow',
|
||||
sprPolicies: 'slow',
|
||||
aaiiSentiment: 'slow',
|
||||
};
|
||||
|
||||
export const PORTWATCH_CHOKEPOINTS_REF_KEY = 'portwatch:chokepoints:ref:v1';
|
||||
|
||||
@@ -329,6 +329,9 @@ export class App {
|
||||
const panel = this.state.panels['cot-positioning'] as CotPositioningPanel | undefined;
|
||||
if (panel) primeTask('cot-positioning', () => panel.fetchData());
|
||||
}
|
||||
if (shouldPrime('aaii-sentiment')) {
|
||||
primeTask('aaiiSentiment', () => this.dataLoader.loadAaiiSentiment());
|
||||
}
|
||||
if (shouldPrimeAny(['markets', 'heatmap', 'commodities', 'crypto', 'energy-complex'])) {
|
||||
primeTask('markets', () => this.dataLoader.loadMarkets());
|
||||
}
|
||||
@@ -1394,6 +1397,12 @@ export class App {
|
||||
REFRESH_INTERVALS.cotPositioning,
|
||||
() => this.isPanelNearViewport('cot-positioning')
|
||||
);
|
||||
this.refreshScheduler.scheduleRefresh(
|
||||
'aaii-sentiment',
|
||||
() => this.dataLoader.loadAaiiSentiment(),
|
||||
REFRESH_INTERVALS.aaiiSentiment,
|
||||
() => this.isPanelNearViewport('aaii-sentiment')
|
||||
);
|
||||
|
||||
// Refresh intelligence signals for CII (geopolitical variant only)
|
||||
if (SITE_VARIANT === 'full') {
|
||||
|
||||
@@ -161,6 +161,7 @@ import {
|
||||
DiseaseOutbreaksPanel,
|
||||
SocialVelocityPanel,
|
||||
WsbTickerScannerPanel,
|
||||
AAIISentimentPanel,
|
||||
} from '@/components';
|
||||
import { SatelliteFiresPanel } from '@/components/SatelliteFiresPanel';
|
||||
import { classifyNewsItem } from '@/services/positive-classifier';
|
||||
@@ -3321,6 +3322,16 @@ export class DataLoaderManager implements AppModule {
|
||||
}
|
||||
}
|
||||
|
||||
async loadAaiiSentiment(): Promise<void> {
|
||||
const panel = this.ctx.panels['aaii-sentiment'] as AAIISentimentPanel | undefined;
|
||||
if (!panel) return;
|
||||
try {
|
||||
await panel.fetchData();
|
||||
} catch (e) {
|
||||
console.error('[App] AAII sentiment load failed:', e);
|
||||
}
|
||||
}
|
||||
|
||||
async loadCrossSourceSignals(): Promise<void> {
|
||||
try {
|
||||
const result = await fetchCrossSourceSignals();
|
||||
|
||||
@@ -71,6 +71,7 @@ import {
|
||||
DiseaseOutbreaksPanel,
|
||||
SocialVelocityPanel,
|
||||
WsbTickerScannerPanel,
|
||||
AAIISentimentPanel,
|
||||
} from '@/components';
|
||||
import { SatelliteFiresPanel } from '@/components/SatelliteFiresPanel';
|
||||
import { focusInvestmentOnMap } from '@/services/investments-focus';
|
||||
@@ -1078,6 +1079,7 @@ export class PanelLayoutManager implements AppModule {
|
||||
|
||||
this.createPanel('macro-signals', () => new MacroSignalsPanel());
|
||||
this.createPanel('fear-greed', () => new FearGreedPanel());
|
||||
this.createPanel('aaii-sentiment', () => new AAIISentimentPanel());
|
||||
this.createPanel('macro-tiles', () => new MacroTilesPanel());
|
||||
this.createPanel('fsi', () => new FSIPanel());
|
||||
this.createPanel('yield-curve', () => new YieldCurvePanel());
|
||||
|
||||
254
src/components/AAIISentimentPanel.ts
Normal file
254
src/components/AAIISentimentPanel.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
import { Panel } from './Panel';
|
||||
import { t } from '@/services/i18n';
|
||||
import { escapeHtml } from '@/utils/sanitize';
|
||||
import { getHydratedData } from '@/services/bootstrap';
|
||||
import { toApiUrl } from '@/services/runtime';
|
||||
|
||||
interface WeekData {
|
||||
date: string;
|
||||
bullish: number;
|
||||
bearish: number;
|
||||
neutral: number;
|
||||
spread: number;
|
||||
}
|
||||
|
||||
interface AAIIData {
|
||||
seededAt: string;
|
||||
fallback?: boolean;
|
||||
source: string;
|
||||
latest: WeekData;
|
||||
previous: WeekData | null;
|
||||
avg8w: { bullish: number; bearish: number; neutral: number; spread: number } | null;
|
||||
historicalAvg: { bullish: number; bearish: number; neutral: number };
|
||||
extremes: { spreadBelow20: number; bullishAbove50: number; bearishAbove50: number };
|
||||
weeks: WeekData[];
|
||||
}
|
||||
|
||||
function spreadColor(spread: number): string {
|
||||
if (spread <= -20) return '#e74c3c';
|
||||
if (spread <= -10) return '#e67e22';
|
||||
if (spread < 0) return '#f39c12';
|
||||
if (spread < 10) return '#95a5a6';
|
||||
if (spread < 20) return '#27ae60';
|
||||
return '#2ecc71';
|
||||
}
|
||||
|
||||
function sentimentLabel(spread: number): string {
|
||||
if (spread <= -20) return 'Extreme Bearish';
|
||||
if (spread <= -10) return 'Bearish';
|
||||
if (spread < 0) return 'Mildly Bearish';
|
||||
if (spread < 10) return 'Neutral';
|
||||
if (spread < 20) return 'Bullish';
|
||||
return 'Extreme Bullish';
|
||||
}
|
||||
|
||||
function renderBar(pct: number, color: string, label: string, value: string): string {
|
||||
return `<div style="margin:4px 0">
|
||||
<div style="display:flex;justify-content:space-between;font-size:11px;color:var(--text-dim);margin-bottom:2px">
|
||||
<span>${escapeHtml(label)}</span>
|
||||
<span style="color:${color};font-weight:600">${escapeHtml(value)}</span>
|
||||
</div>
|
||||
<div style="height:6px;background:rgba(255,255,255,0.08);border-radius:3px">
|
||||
<div style="width:${Math.min(pct, 100)}%;height:100%;background:${color};border-radius:3px;transition:width 0.3s"></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderSpreadBar(spread: number): string {
|
||||
const color = spreadColor(spread);
|
||||
const clamped = Math.max(-60, Math.min(60, spread));
|
||||
const center = 50;
|
||||
const barWidth = Math.abs(clamped) / 60 * 50;
|
||||
const leftPct = clamped >= 0 ? center : center - barWidth;
|
||||
|
||||
return `<div style="margin:8px 0">
|
||||
<div style="display:flex;justify-content:space-between;font-size:11px;color:var(--text-dim);margin-bottom:3px">
|
||||
<span>Bull-Bear Spread</span>
|
||||
<span style="color:${color};font-weight:700">${clamped >= 0 ? '+' : ''}${spread.toFixed(1)}%</span>
|
||||
</div>
|
||||
<div style="position:relative;height:10px;background:rgba(255,255,255,0.06);border-radius:4px">
|
||||
<div style="position:absolute;top:0;bottom:0;left:50%;width:1px;background:rgba(255,255,255,0.2)"></div>
|
||||
<div style="position:absolute;top:0;bottom:0;left:${leftPct.toFixed(1)}%;width:${barWidth.toFixed(1)}%;background:${color};border-radius:3px;transition:all 0.3s"></div>
|
||||
</div>
|
||||
<div style="display:flex;justify-content:space-between;font-size:9px;color:var(--text-dim);margin-top:2px">
|
||||
<span>Bearish</span>
|
||||
<span>Bullish</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderSparkChart(weeks: WeekData[]): string {
|
||||
if (weeks.length < 2) return '';
|
||||
const data = [...weeks].reverse();
|
||||
const W = 280, H = 60, PAD = 4;
|
||||
const spreads = data.map(w => w.spread);
|
||||
const maxAbs = Math.max(Math.abs(Math.min(...spreads)), Math.abs(Math.max(...spreads)), 20);
|
||||
const stepX = (W - PAD * 2) / (data.length - 1);
|
||||
const midY = H / 2;
|
||||
const scaleY = (midY - PAD) / maxAbs;
|
||||
|
||||
const points = data.map((w, i) => {
|
||||
const x = (PAD + i * stepX).toFixed(1);
|
||||
const y = (midY - w.spread * scaleY).toFixed(1);
|
||||
return `${x},${y}`;
|
||||
});
|
||||
const polyline = points.join(' ');
|
||||
|
||||
const bars = data.map((w, i) => {
|
||||
const x = PAD + i * stepX - 1;
|
||||
const barH = Math.abs(w.spread) * scaleY;
|
||||
const y = w.spread >= 0 ? midY - barH : midY;
|
||||
const fill = w.spread >= 0 ? 'rgba(39,174,96,0.25)' : 'rgba(231,76,60,0.25)';
|
||||
return `<rect x="${x.toFixed(1)}" y="${y.toFixed(1)}" width="2" height="${barH.toFixed(1)}" fill="${fill}" rx="0.5"/>`;
|
||||
}).join('');
|
||||
|
||||
const zeroLine = `<line x1="${PAD}" y1="${midY}" x2="${W - PAD}" y2="${midY}" stroke="rgba(255,255,255,0.15)" stroke-width="0.5" stroke-dasharray="3,3"/>`;
|
||||
const contrarian = midY + 20 * scaleY;
|
||||
const contrarianLine = `<line x1="${PAD}" y1="${contrarian.toFixed(1)}" x2="${W - PAD}" y2="${contrarian.toFixed(1)}" stroke="rgba(231,76,60,0.3)" stroke-width="0.5" stroke-dasharray="2,4"/>`;
|
||||
const contrarianLabel = `<text x="${W - PAD}" y="${(contrarian - 2).toFixed(1)}" text-anchor="end" font-size="7" fill="rgba(231,76,60,0.5)" font-family="system-ui,sans-serif">-20 buy signal</text>`;
|
||||
|
||||
return `<div style="margin:8px 0">
|
||||
<div style="font-size:10px;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin-bottom:4px">52-Week Spread History</div>
|
||||
<svg viewBox="0 0 ${W} ${H}" width="100%" height="${H}" style="display:block">
|
||||
${bars}
|
||||
${zeroLine}
|
||||
${contrarianLine}
|
||||
${contrarianLabel}
|
||||
<polyline points="${polyline}" fill="none" stroke="rgba(255,255,255,0.6)" stroke-width="1.2" stroke-linejoin="round"/>
|
||||
</svg>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
export class AAIISentimentPanel extends Panel {
|
||||
private data: AAIIData | null = null;
|
||||
|
||||
constructor() {
|
||||
super({
|
||||
id: 'aaii-sentiment',
|
||||
title: 'AAII Investor Sentiment',
|
||||
showCount: false,
|
||||
infoTooltip: 'Weekly AAII survey: individual investors report 6-month market outlook as bullish, neutral, or bearish. Spread below -20 is a historical contrarian buy signal.',
|
||||
});
|
||||
}
|
||||
|
||||
public async fetchData(): Promise<boolean> {
|
||||
// SSR hydration is one-shot: getHydratedData deletes the key after the
|
||||
// first read. Only the initial page-load call will hit this path — all
|
||||
// subsequent hourly refreshes fall through to the bootstrap API below.
|
||||
const hydrated = getHydratedData('aaiiSentiment') as AAIIData | undefined;
|
||||
if (hydrated?.latest) {
|
||||
this.data = hydrated;
|
||||
this.renderPanel();
|
||||
return true;
|
||||
}
|
||||
// Refresh path: fetch directly from the bootstrap API so the weekly
|
||||
// dataset keeps flowing after the first paint.
|
||||
try {
|
||||
const resp = await fetch(toApiUrl('/api/bootstrap?keys=aaiiSentiment'), {
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (resp.ok) {
|
||||
const { data } = (await resp.json()) as { data: { aaiiSentiment?: AAIIData } };
|
||||
if (data.aaiiSentiment?.latest) {
|
||||
this.data = data.aaiiSentiment;
|
||||
this.renderPanel();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch { /* fallback below */ }
|
||||
// Retry after ~5 minutes so the panel recovers on its own if the seed
|
||||
// arrives late (AAII cadence is weekly but the cron can be delayed).
|
||||
this.showError('AAII sentiment data unavailable', () => { void this.fetchData(); }, 300);
|
||||
return false;
|
||||
}
|
||||
|
||||
private renderPanel(): void {
|
||||
if (!this.data?.latest) {
|
||||
this.showError(t('common.noDataShort'), () => void this.fetchData());
|
||||
return;
|
||||
}
|
||||
|
||||
const d = this.data;
|
||||
const { latest, previous, avg8w, historicalAvg, extremes, weeks } = d;
|
||||
const color = spreadColor(latest.spread);
|
||||
const label = sentimentLabel(latest.spread);
|
||||
|
||||
const prevSpread = previous?.spread;
|
||||
const spreadDelta = prevSpread != null ? latest.spread - prevSpread : null;
|
||||
const deltaStr = spreadDelta != null
|
||||
? `<span style="color:${spreadDelta >= 0 ? '#2ecc71' : '#e74c3c'};font-size:10px;margin-left:4px">${spreadDelta >= 0 ? '+' : ''}${spreadDelta.toFixed(1)} vs prev</span>`
|
||||
: '';
|
||||
|
||||
const contrarianSignal = latest.spread <= -20
|
||||
? `<div style="display:flex;align-items:center;gap:6px;padding:6px 8px;margin:8px 0;border-radius:4px;border:1px solid #2ecc71;background:rgba(46,204,113,0.08);font-size:10px;color:#2ecc71">
|
||||
ⓘ Contrarian buy signal active: spread at ${latest.spread.toFixed(1)}% (threshold: -20%)
|
||||
</div>`
|
||||
: latest.bearish >= 50
|
||||
? `<div style="display:flex;align-items:center;gap:6px;padding:6px 8px;margin:8px 0;border-radius:4px;border:1px solid #e67e22;background:rgba(230,126,34,0.08);font-size:10px;color:#e67e22">
|
||||
⚠ Extreme bearish reading: ${latest.bearish.toFixed(1)}% bearish (avg: ${historicalAvg.bearish}%)
|
||||
</div>`
|
||||
: '';
|
||||
|
||||
const avgSection = avg8w ? `
|
||||
<div style="margin-top:8px;padding-top:8px;border-top:1px solid rgba(255,255,255,0.06)">
|
||||
<div style="font-size:10px;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin-bottom:4px">8-Week Moving Average</div>
|
||||
<div style="display:grid;grid-template-columns:repeat(3,1fr);gap:4px;text-align:center">
|
||||
<div><div style="font-size:14px;font-weight:600;color:#2ecc71">${avg8w.bullish}%</div><div style="font-size:9px;color:var(--text-dim)">Bull</div></div>
|
||||
<div><div style="font-size:14px;font-weight:600;color:#95a5a6">${avg8w.neutral}%</div><div style="font-size:9px;color:var(--text-dim)">Neutral</div></div>
|
||||
<div><div style="font-size:14px;font-weight:600;color:#e74c3c">${avg8w.bearish}%</div><div style="font-size:9px;color:var(--text-dim)">Bear</div></div>
|
||||
</div>
|
||||
</div>` : '';
|
||||
|
||||
const extremeSection = (extremes.spreadBelow20 > 0 || extremes.bearishAbove50 > 0)
|
||||
? `<div style="margin-top:6px;font-size:10px;color:var(--text-dim)">
|
||||
52w extremes: ${extremes.spreadBelow20} contrarian signals, ${extremes.bearishAbove50} extreme bear, ${extremes.bullishAbove50} extreme bull
|
||||
</div>` : '';
|
||||
|
||||
const fallbackBadge = d.fallback
|
||||
? '<span style="display:inline-block;padding:1px 5px;border-radius:3px;background:rgba(230,126,34,0.15);color:#e67e22;font-size:9px;margin-left:4px">(fallback data)</span>'
|
||||
: '';
|
||||
const dateStr = latest.date ? `<div style="font-size:9px;color:var(--text-dim);text-align:right;margin-top:4px">Survey: ${escapeHtml(latest.date)}${d.source !== 'xls' ? ` (${escapeHtml(d.source)})` : ''}${fallbackBadge}</div>` : '';
|
||||
|
||||
const html = `
|
||||
<div style="padding:12px 14px">
|
||||
<div style="text-align:center;margin-bottom:8px">
|
||||
<div style="font-size:11px;font-weight:600;color:${color};letter-spacing:0.06em;text-transform:uppercase">${escapeHtml(label)}</div>
|
||||
${deltaStr ? `<div style="margin-top:2px">${deltaStr}</div>` : ''}
|
||||
</div>
|
||||
|
||||
${contrarianSignal}
|
||||
|
||||
<div style="display:grid;grid-template-columns:repeat(3,1fr);gap:6px;text-align:center;padding:8px;background:rgba(255,255,255,0.03);border-radius:6px;margin-bottom:8px">
|
||||
<div>
|
||||
<div style="font-size:22px;font-weight:700;color:#2ecc71">${latest.bullish.toFixed(1)}%</div>
|
||||
<div style="font-size:10px;color:var(--text-dim)">Bullish</div>
|
||||
<div style="font-size:9px;color:var(--text-dim)">avg ${historicalAvg.bullish}%</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="font-size:22px;font-weight:700;color:#95a5a6">${latest.neutral.toFixed(1)}%</div>
|
||||
<div style="font-size:10px;color:var(--text-dim)">Neutral</div>
|
||||
<div style="font-size:9px;color:var(--text-dim)">avg ${historicalAvg.neutral}%</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="font-size:22px;font-weight:700;color:#e74c3c">${latest.bearish.toFixed(1)}%</div>
|
||||
<div style="font-size:10px;color:var(--text-dim)">Bearish</div>
|
||||
<div style="font-size:9px;color:var(--text-dim)">avg ${historicalAvg.bearish}%</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
${renderBar(latest.bullish, '#2ecc71', 'Bullish', `${latest.bullish.toFixed(1)}%`)}
|
||||
${renderBar(latest.neutral, '#95a5a6', 'Neutral', `${latest.neutral.toFixed(1)}%`)}
|
||||
${renderBar(latest.bearish, '#e74c3c', 'Bearish', `${latest.bearish.toFixed(1)}%`)}
|
||||
|
||||
${renderSpreadBar(latest.spread)}
|
||||
|
||||
${renderSparkChart(weeks)}
|
||||
|
||||
${avgSection}
|
||||
${extremeSection}
|
||||
${dateStr}
|
||||
</div>`;
|
||||
|
||||
this.setContent(html);
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,7 @@ export * from './DisasterCorrelationPanel';
|
||||
export * from './ConsumerPricesPanel';
|
||||
export { NationalDebtPanel } from './NationalDebtPanel';
|
||||
export * from './FearGreedPanel';
|
||||
export * from './AAIISentimentPanel';
|
||||
export * from './MacroTilesPanel';
|
||||
export * from './FSIPanel';
|
||||
export * from './YieldCurvePanel';
|
||||
|
||||
@@ -125,6 +125,7 @@ export const COMMANDS: Command[] = [
|
||||
{ id: 'panel:ai', keywords: ['ai', 'ml', 'artificial intelligence'], label: 'Panel: AI/ML', icon: '\u{1F916}', category: 'panels' },
|
||||
{ id: 'panel:macro-signals', keywords: ['macro', 'macro signals', 'liquidity'], label: 'Panel: Market Radar', icon: '\u{1F4C9}', category: 'panels' },
|
||||
{ id: 'panel:fear-greed', keywords: ['fear', 'greed', 'fear and greed', 'sentiment', 'fear greed index'], label: 'Panel: Fear & Greed', icon: '\u{1F4CA}', category: 'panels' },
|
||||
{ id: 'panel:aaii-sentiment', keywords: ['aaii', 'investor sentiment', 'bull bear', 'sentiment survey', 'aaii survey', 'contrarian'], label: 'Panel: AAII Investor Sentiment', icon: '\u{1F4CA}', category: 'panels' },
|
||||
{ id: 'panel:hormuz-tracker', keywords: ['hormuz', 'strait of hormuz', 'shipping', 'crude oil', 'lng', 'fertilizer', 'tanker', 'wto'], label: 'Panel: Hormuz Trade Tracker', icon: '\u{1F6A2}', category: 'panels' },
|
||||
{ id: 'panel:etf-flows', keywords: ['etf', 'etf flows', 'fund flows'], label: 'Panel: BTC ETF Tracker', icon: '\u{1F4B9}', category: 'panels' },
|
||||
{ id: 'panel:stablecoins', keywords: ['stablecoins', 'usdt', 'usdc'], label: 'Panel: Stablecoins', icon: '\u{1FA99}', category: 'panels' },
|
||||
|
||||
@@ -62,6 +62,7 @@ const FULL_PANELS: Record<string, PanelConfig> = {
|
||||
'satellite-fires': { name: 'Fires', enabled: true, priority: 2 },
|
||||
'macro-signals': { name: 'Market Regime', enabled: true, priority: 2 },
|
||||
'fear-greed': { name: 'Fear & Greed', enabled: true, priority: 2 },
|
||||
'aaii-sentiment': { name: 'AAII Sentiment', enabled: false, priority: 2 },
|
||||
'macro-tiles': { name: 'Macro Indicators', enabled: false, priority: 2 },
|
||||
'fsi': { name: 'Financial Stress', enabled: false, priority: 2 },
|
||||
'yield-curve': { name: 'Yield Curve', enabled: false, priority: 2 },
|
||||
@@ -435,6 +436,7 @@ const FINANCE_PANELS: Record<string, PanelConfig> = {
|
||||
'macro-signals': { name: 'Market Regime', enabled: true, priority: 1 },
|
||||
'macro-tiles': { name: 'Macro Indicators', enabled: true, priority: 1 },
|
||||
'fear-greed': { name: 'Fear & Greed', enabled: true, priority: 1 },
|
||||
'aaii-sentiment': { name: 'AAII Sentiment', enabled: true, priority: 2 },
|
||||
'fsi': { name: 'Financial Stress', enabled: true, priority: 1 },
|
||||
'yield-curve': { name: 'Yield Curve', enabled: true, priority: 1 },
|
||||
'earnings-calendar': { name: 'Earnings Calendar', enabled: true, priority: 1 },
|
||||
|
||||
@@ -61,6 +61,7 @@ export const REFRESH_INTERVALS = {
|
||||
earningsCalendar: 60 * 60 * 1000,
|
||||
economicCalendar: 60 * 60 * 1000,
|
||||
cotPositioning: 60 * 60 * 1000,
|
||||
aaiiSentiment: 60 * 60 * 1000, // weekly data; hourly refresh is sufficient
|
||||
};
|
||||
|
||||
// Monitor colors - shared
|
||||
|
||||
158
tests/seed-aaii-sentiment.test.mjs
Normal file
158
tests/seed-aaii-sentiment.test.mjs
Normal file
@@ -0,0 +1,158 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
const { extractSentimentData, parseHtmlSentiment, parseXlsRows, excelDateToISO } = await import('../scripts/seed-aaii-sentiment.mjs');
|
||||
|
||||
describe('AAII Sentiment seed parsing', () => {
|
||||
describe('excelDateToISO', () => {
|
||||
it('converts known serial dates correctly', () => {
|
||||
assert.equal(excelDateToISO(1), '1900-01-01');
|
||||
assert.equal(excelDateToISO(59), '1900-02-28');
|
||||
assert.equal(excelDateToISO(61), '1900-03-01'); // serial 60 is Lotus bug
|
||||
assert.equal(excelDateToISO(46115), '2026-04-03');
|
||||
});
|
||||
|
||||
it('returns null for invalid inputs', () => {
|
||||
assert.equal(excelDateToISO(0), null);
|
||||
assert.equal(excelDateToISO(-5), null);
|
||||
assert.equal(excelDateToISO('abc'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSentimentData', () => {
|
||||
it('extracts data from rows with header row containing Bullish/Neutral/Bearish', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Neutral', 'Bearish', 'Bull-Bear Spread'],
|
||||
[46115, 0.357, 0.213, 0.43, null], // 2026-04-03 as Excel serial
|
||||
[46108, 0.224, 0.218, 0.558, null], // 2026-03-27
|
||||
[46101, 0.192, 0.237, 0.571, null], // 2026-03-20
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.ok(result.length === 3, `Expected 3 rows, got ${result.length}`);
|
||||
assert.equal(result[0].date, '2026-04-03');
|
||||
assert.equal(result[0].bullish, 35.7);
|
||||
assert.equal(result[0].bearish, 43.0);
|
||||
assert.equal(result[0].neutral, 21.3);
|
||||
assert.equal(result[0].spread, -7.3);
|
||||
});
|
||||
|
||||
it('handles percentages > 1 (already in percentage form)', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Neutral', 'Bearish'],
|
||||
['2026-01-02', 43.1, 31.6, 25.3],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.ok(result.length === 1);
|
||||
assert.equal(result[0].bullish, 43.1);
|
||||
assert.equal(result[0].bearish, 25.3);
|
||||
assert.equal(result[0].neutral, 31.6);
|
||||
assert.equal(result[0].spread, 17.8);
|
||||
});
|
||||
|
||||
it('handles fractions (0-1 range) and converts to percentages', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Neutral', 'Bearish'],
|
||||
['2026-01-02', 0.45, 0.30, 0.25],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.ok(result.length === 1);
|
||||
assert.equal(result[0].bullish, 45);
|
||||
assert.equal(result[0].bearish, 25);
|
||||
assert.equal(result[0].neutral, 30);
|
||||
});
|
||||
|
||||
it('returns empty array when no header found', () => {
|
||||
const rows = [
|
||||
['foo', 'bar', 'baz'],
|
||||
[1, 2, 3],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.equal(result.length, 0);
|
||||
});
|
||||
|
||||
it('skips rows with null bull/bear values', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Neutral', 'Bearish'],
|
||||
['2026-01-02', 43.1, 31.6, 25.3],
|
||||
['2026-01-09', null, 28.0, null],
|
||||
['2026-01-16', 35.0, 30.0, 35.0],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.equal(result.length, 2);
|
||||
});
|
||||
|
||||
it('computes neutral when missing', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Bearish'],
|
||||
['2026-01-02', 40.0, 30.0],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.ok(result.length === 1);
|
||||
assert.equal(result[0].neutral, 30.0);
|
||||
});
|
||||
|
||||
it('sorts output by date descending', () => {
|
||||
const rows = [
|
||||
['Date', 'Bullish', 'Neutral', 'Bearish'],
|
||||
['2026-01-02', 40, 30, 30],
|
||||
['2026-03-01', 35, 35, 30],
|
||||
['2026-02-01', 42, 28, 30],
|
||||
];
|
||||
const result = extractSentimentData(rows);
|
||||
assert.equal(result[0].date, '2026-03-01');
|
||||
assert.equal(result[1].date, '2026-02-01');
|
||||
assert.equal(result[2].date, '2026-01-02');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseHtmlSentiment', () => {
|
||||
it('extracts percentages from AAII-style HTML with tableTxt class', () => {
|
||||
const html = `
|
||||
<table>
|
||||
<tr><td class="tableTxt">35.7%</td></tr>
|
||||
<tr><td class="tableTxt">21.3%</td></tr>
|
||||
<tr><td class="tableTxt">43.0%</td></tr>
|
||||
</table>
|
||||
`;
|
||||
const result = parseHtmlSentiment(html);
|
||||
assert.ok(result.length === 1);
|
||||
assert.equal(result[0].bullish, 35.7);
|
||||
assert.equal(result[0].neutral, 21.3);
|
||||
assert.equal(result[0].bearish, 43.0);
|
||||
assert.equal(result[0].spread, -7.3);
|
||||
});
|
||||
|
||||
it('returns empty array when fewer than 3 percentages found', () => {
|
||||
const html = `<td class="tableTxt">35.7%</td><td class="tableTxt">21.3%</td>`;
|
||||
const result = parseHtmlSentiment(html);
|
||||
assert.equal(result.length, 0);
|
||||
});
|
||||
|
||||
it('assigns a date that is a Thursday', () => {
|
||||
const html = `
|
||||
<td class="tableTxt">40.0%</td>
|
||||
<td class="tableTxt">30.0%</td>
|
||||
<td class="tableTxt">30.0%</td>
|
||||
`;
|
||||
const result = parseHtmlSentiment(html);
|
||||
assert.ok(result.length === 1);
|
||||
const d = new Date(result[0].date + 'T12:00:00Z');
|
||||
assert.equal(d.getUTCDay(), 4, 'Expected Thursday (day 4)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseXlsRows', () => {
|
||||
it('returns empty array for empty buffer', () => {
|
||||
const result = parseXlsRows(new ArrayBuffer(0));
|
||||
assert.deepEqual(result, []);
|
||||
});
|
||||
|
||||
it('returns empty array for non-XLS data', () => {
|
||||
const buf = new ArrayBuffer(100);
|
||||
const view = new Uint8Array(buf);
|
||||
for (let i = 0; i < 100; i++) view[i] = i;
|
||||
const result = parseXlsRows(buf);
|
||||
assert.deepEqual(result, []);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user