fix(breadth): use ET trading day, daily cache tier, distinguish null from 0%

Addresses greptile P2 review on PR #2932:

- scripts/seed-market-breadth.mjs: stamp history rows with the ET trading
  day via Intl.DateTimeFormat('en-CA', { timeZone: 'America/New_York' }).
  Railway cron fires 9 PM ET (01:00-02:00 UTC next calendar day), so UTC
  dates were labelling every session one day ahead. en-CA produces ISO
  YYYY-MM-DD; America/New_York handles DST automatically.

- server/gateway.ts: move /api/market/v1/get-market-breadth-history from
  'slow' (s-maxage=1800) to 'daily' (s-maxage=86400, swr=14400). Seeder
  runs once per day so the short tier wasted Redis reads and edge
  re-fetches over the 23h gap.

- proto/worldmonitor/market/v1/get_market_breadth_history.proto: mark
  pct_above_{20d,50d,200d} and current_pct_above_* as `optional double`
  so the wire format and generated TypeScript types carry `number |
  undefined`. Without this, a missing Barchart reading and a legitimate
  0% breadth (severe market dislocation) rendered identically. The
  handler (get-market-breadth-history.ts) now passes nulls through as
  undefined via nullToUndefined() instead of casting through loose
  intermediate types, and no longer coerces with `?? 0`. Regenerated via
  `make generate` (buf 1.64.0).

- tests/market-breadth.test.mjs: update assertions to match the new
  nullToUndefined path and add coverage for the proto `optional`
  modifiers.

docs/api/MarketService.openapi.yaml

@@ -644,6 +644,32 @@ paths:
                         application/json:
                             schema:
                                 $ref: '#/components/schemas/Error'
+    /api/market/v1/get-market-breadth-history:
+        get:
+            tags:
+                - MarketService
+            summary: GetMarketBreadthHistory
+            description: GetMarketBreadthHistory retrieves historical % of S&P 500 stocks above 20/50/200-day SMAs.
+            operationId: GetMarketBreadthHistory
+            responses:
+                "200":
+                    description: Successful response
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/GetMarketBreadthHistoryResponse'
+                "400":
+                    description: Validation error
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/ValidationError'
+                default:
+                    description: Error response
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/Error'
 components:
     schemas:
         Error:
@@ -1806,3 +1832,43 @@ components:
                     type: string
                 transactionDate:
                     type: string
+        GetMarketBreadthHistoryRequest:
+            type: object
+        GetMarketBreadthHistoryResponse:
+            type: object
+            properties:
+                currentPctAbove20d:
+                    type: number
+                    format: double
+                currentPctAbove50d:
+                    type: number
+                    format: double
+                currentPctAbove200d:
+                    type: number
+                    format: double
+                updatedAt:
+                    type: string
+                history:
+                    type: array
+                    items:
+                        $ref: '#/components/schemas/BreadthSnapshot'
+                unavailable:
+                    type: boolean
+        BreadthSnapshot:
+            type: object
+            properties:
+                date:
+                    type: string
+                pctAbove20d:
+                    type: number
+                    format: double
+                    description: |-
+                        Optional so a missing/failed Barchart reading serializes as JSON null
+                         instead of collapsing to 0, which would render identically to a real 0%
+                         reading (severe market dislocation with no S&P stocks above SMA).
+                pctAbove50d:
+                    type: number
+                    format: double
+                pctAbove200d:
+                    type: number
+                    format: double

proto/worldmonitor/market/v1/get_market_breadth_history.proto

@@ -8,15 +8,18 @@ message GetMarketBreadthHistoryRequest {}
 
 message BreadthSnapshot {
   string date = 1;
-  double pct_above_20d = 2;
-  double pct_above_50d = 3;
-  double pct_above_200d = 4;
+  // Optional so a missing/failed Barchart reading serializes as JSON null
+  // instead of collapsing to 0, which would render identically to a real 0%
+  // reading (severe market dislocation with no S&P stocks above SMA).
+  optional double pct_above_20d = 2;
+  optional double pct_above_50d = 3;
+  optional double pct_above_200d = 4;
 }
 
 message GetMarketBreadthHistoryResponse {
-  double current_pct_above_20d = 1;
-  double current_pct_above_50d = 2;
-  double current_pct_above_200d = 3;
+  optional double current_pct_above_20d = 1;
+  optional double current_pct_above_50d = 2;
+  optional double current_pct_above_200d = 3;
   string updated_at = 4;
   repeated BreadthSnapshot history = 5;
   bool unavailable = 6;

scripts/seed-market-breadth.mjs

@@ -75,7 +75,11 @@ async function fetchAll() {
 
   const existing = await readExistingHistory();
   const history = existing?.history ?? [];
-  const today = new Date().toISOString().slice(0, 10);
+  // ET trading day: Railway cron fires at 9 PM ET which is 01:00-02:00 UTC on
+  // the NEXT calendar day, so UTC date would stamp today's session with
+  // tomorrow's date. en-CA locale returns ISO YYYY-MM-DD; America/New_York
+  // handles DST automatically.
+  const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'America/New_York' }).format(new Date());
 
   const lastEntry = history.at(-1);
   if (lastEntry?.date === today) {

server/gateway.ts

@@ -67,7 +67,7 @@ const RPC_CACHE_TIER: Record<string, CacheTier> = {
   '/api/market/v1/list-stablecoin-markets': 'medium',
   '/api/market/v1/get-sector-summary': 'medium',
   '/api/market/v1/get-fear-greed-index': 'slow',
-  '/api/market/v1/get-market-breadth-history': 'slow',
+  '/api/market/v1/get-market-breadth-history': 'daily',
   '/api/market/v1/list-gulf-quotes': 'medium',
   '/api/market/v1/analyze-stock': 'slow',
   '/api/market/v1/get-stock-analysis-history': 'medium',

server/worldmonitor/market/v1/get-market-breadth-history.ts

@@ -25,36 +25,18 @@ interface SeedPayload {
   history: SeedEntry[];
 }
 
-// Proto3 doubles cannot encode "absent" — we ship JSON null instead so the UI
-// can distinguish a real 0% reading from a missing/failed one. The generated
-// response interface types these as `number`, but the JSON transport preserves
-// null values at runtime; casts below accommodate the type/runtime gap.
-type BreadthSnapshotLoose = Omit<BreadthSnapshot, 'pctAbove20d' | 'pctAbove50d' | 'pctAbove200d'> & {
-  pctAbove20d: number | null;
-  pctAbove50d: number | null;
-  pctAbove200d: number | null;
-};
-type BreadthResponseLoose = Omit<
-  GetMarketBreadthHistoryResponse,
-  'currentPctAbove20d' | 'currentPctAbove50d' | 'currentPctAbove200d' | 'history'
-> & {
-  currentPctAbove20d: number | null;
-  currentPctAbove50d: number | null;
-  currentPctAbove200d: number | null;
-  history: BreadthSnapshotLoose[];
-};
-
 function emptyUnavailable(): GetMarketBreadthHistoryResponse {
   return {
-    currentPctAbove20d: 0,
-    currentPctAbove50d: 0,
-    currentPctAbove200d: 0,
     updatedAt: '',
     history: [],
     unavailable: true,
   };
 }
 
+function nullToUndefined(v: number | null | undefined): number | undefined {
+  return v == null ? undefined : v;
+}
+
 export async function getMarketBreadthHistory(
   _ctx: ServerContext,
   _req: GetMarketBreadthHistoryRequest,
@@ -65,25 +47,24 @@ export async function getMarketBreadthHistory(
       return emptyUnavailable();
     }
 
-    // Preserve null for current + history readings so a partial seed failure
-    // (one Barchart symbol returning null) can be distinguished from a real
-    // 0% breadth reading in the UI. Panel treats null as "missing".
-    const history: BreadthSnapshotLoose[] = raw.history.map((e) => ({
+    // Preserve missing readings as undefined (proto `optional` → JSON omits
+    // the field) so a partial seed failure can be distinguished from a real
+    // 0% breadth reading in the UI. Panel treats undefined as "missing".
+    const history: BreadthSnapshot[] = raw.history.map((e) => ({
       date: e.date,
-      pctAbove20d: e.pctAbove20d ?? null,
-      pctAbove50d: e.pctAbove50d ?? null,
-      pctAbove200d: e.pctAbove200d ?? null,
+      pctAbove20d: nullToUndefined(e.pctAbove20d),
+      pctAbove50d: nullToUndefined(e.pctAbove50d),
+      pctAbove200d: nullToUndefined(e.pctAbove200d),
     }));
 
-    const loose: BreadthResponseLoose = {
-      currentPctAbove20d: raw.current.pctAbove20d ?? null,
-      currentPctAbove50d: raw.current.pctAbove50d ?? null,
-      currentPctAbove200d: raw.current.pctAbove200d ?? null,
+    return {
+      currentPctAbove20d: nullToUndefined(raw.current.pctAbove20d),
+      currentPctAbove50d: nullToUndefined(raw.current.pctAbove50d),
+      currentPctAbove200d: nullToUndefined(raw.current.pctAbove200d),
       updatedAt: raw.updatedAt ?? '',
       history,
       unavailable: false,
     };
-    return loose as unknown as GetMarketBreadthHistoryResponse;
   } catch {
     return emptyUnavailable();
   }

src/generated/client/worldmonitor/market/v1/service_client.ts

@@ -484,6 +484,25 @@ export interface InsiderTransaction {
   transactionDate: string;
 }
 
+export interface GetMarketBreadthHistoryRequest {
+}
+
+export interface GetMarketBreadthHistoryResponse {
+  currentPctAbove20d?: number;
+  currentPctAbove50d?: number;
+  currentPctAbove200d?: number;
+  updatedAt: string;
+  history: BreadthSnapshot[];
+  unavailable: boolean;
+}
+
+export interface BreadthSnapshot {
+  date: string;
+  pctAbove20d?: number;
+  pctAbove50d?: number;
+  pctAbove200d?: number;
+}
+
 export interface FieldViolation {
   field: string;
   description: string;
@@ -1024,6 +1043,29 @@ export class MarketServiceClient {
     return await resp.json() as GetInsiderTransactionsResponse;
   }
 
+  async getMarketBreadthHistory(req: GetMarketBreadthHistoryRequest, options?: MarketServiceCallOptions): Promise<GetMarketBreadthHistoryResponse> {
+    let path = "/api/market/v1/get-market-breadth-history";
+    const url = this.baseURL + path;
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      ...this.defaultHeaders,
+      ...options?.headers,
+    };
+
+    const resp = await this.fetchFn(url, {
+      method: "GET",
+      headers,
+      signal: options?.signal,
+    });
+
+    if (!resp.ok) {
+      return this.handleError(resp);
+    }
+
+    return await resp.json() as GetMarketBreadthHistoryResponse;
+  }
+
   private async handleError(resp: Response): Promise<never> {
     const body = await resp.text();
     if (resp.status === 400) {

src/generated/server/worldmonitor/market/v1/service_server.ts

@@ -484,6 +484,25 @@ export interface InsiderTransaction {
   transactionDate: string;
 }
 
+export interface GetMarketBreadthHistoryRequest {
+}
+
+export interface GetMarketBreadthHistoryResponse {
+  currentPctAbove20d?: number;
+  currentPctAbove50d?: number;
+  currentPctAbove200d?: number;
+  updatedAt: string;
+  history: BreadthSnapshot[];
+  unavailable: boolean;
+}
+
+export interface BreadthSnapshot {
+  date: string;
+  pctAbove20d?: number;
+  pctAbove50d?: number;
+  pctAbove200d?: number;
+}
+
 export interface FieldViolation {
   field: string;
   description: string;
@@ -549,6 +568,7 @@ export interface MarketServiceHandler {
   listEarningsCalendar(ctx: ServerContext, req: ListEarningsCalendarRequest): Promise<ListEarningsCalendarResponse>;
   getCotPositioning(ctx: ServerContext, req: GetCotPositioningRequest): Promise<GetCotPositioningResponse>;
   getInsiderTransactions(ctx: ServerContext, req: GetInsiderTransactionsRequest): Promise<GetInsiderTransactionsResponse>;
+  getMarketBreadthHistory(ctx: ServerContext, req: GetMarketBreadthHistoryRequest): Promise<GetMarketBreadthHistoryResponse>;
 }
 
 export function createMarketServiceRoutes(
@@ -1424,6 +1444,43 @@ export function createMarketServiceRoutes(
         }
       },
     },
+    {
+      method: "GET",
+      path: "/api/market/v1/get-market-breadth-history",
+      handler: async (req: Request): Promise<Response> => {
+        try {
+          const pathParams: Record<string, string> = {};
+          const body = {} as GetMarketBreadthHistoryRequest;
+
+          const ctx: ServerContext = {
+            request: req,
+            pathParams,
+            headers: Object.fromEntries(req.headers.entries()),
+          };
+
+          const result = await handler.getMarketBreadthHistory(ctx, body);
+          return new Response(JSON.stringify(result as GetMarketBreadthHistoryResponse), {
+            status: 200,
+            headers: { "Content-Type": "application/json" },
+          });
+        } catch (err: unknown) {
+          if (err instanceof ValidationError) {
+            return new Response(JSON.stringify({ violations: err.violations }), {
+              status: 400,
+              headers: { "Content-Type": "application/json" },
+            });
+          }
+          if (options?.onError) {
+            return options.onError(err, req);
+          }
+          const message = err instanceof Error ? err.message : String(err);
+          return new Response(JSON.stringify({ message }), {
+            status: 500,
+            headers: { "Content-Type": "application/json" },
+          });
+        }
+      },
+    },
   ];
 }
 

tests/market-breadth.test.mjs

@@ -98,6 +98,15 @@ describe('Market breadth proto', () => {
     assert.match(protoSrc, /message BreadthSnapshot/);
   });
 
+  it('marks pct_above_* fields optional so null != 0 at the wire level', () => {
+    assert.match(protoSrc, /optional double pct_above_20d/);
+    assert.match(protoSrc, /optional double pct_above_50d/);
+    assert.match(protoSrc, /optional double pct_above_200d/);
+    assert.match(protoSrc, /optional double current_pct_above_20d/);
+    assert.match(protoSrc, /optional double current_pct_above_50d/);
+    assert.match(protoSrc, /optional double current_pct_above_200d/);
+  });
+
   it('is imported in service.proto', () => {
     assert.match(serviceSrc, /get_market_breadth_history\.proto/);
   });
@@ -149,8 +158,11 @@ describe('Market breadth null-vs-zero handling', () => {
     assert.doesNotMatch(handlerSrc, /raw\.current\.pctAbove20d\s*\?\?\s*0/);
     assert.doesNotMatch(handlerSrc, /raw\.current\.pctAbove50d\s*\?\?\s*0/);
     assert.doesNotMatch(handlerSrc, /raw\.current\.pctAbove200d\s*\?\?\s*0/);
-    // The loose interface carries number | null
-    assert.match(handlerSrc, /currentPctAbove20d:\s*number\s*\|\s*null/);
+    // Missing readings flow through nullToUndefined so proto `optional`
+    // serializes as JSON undefined (field omitted), not 0.
+    assert.match(handlerSrc, /nullToUndefined\(raw\.current\.pctAbove20d\)/);
+    assert.match(handlerSrc, /nullToUndefined\(raw\.current\.pctAbove50d\)/);
+    assert.match(handlerSrc, /nullToUndefined\(raw\.current\.pctAbove200d\)/);
   });
 
   it('panel type distinguishes null from number for current readings', () => {