From 79473341d68821846de8523b35cc39157cec2050 Mon Sep 17 00:00:00 2001 From: gp-somni-labs Date: Wed, 22 Apr 2026 19:33:10 -0500 Subject: [PATCH] internal/outpost: serialize websocket writes to prevent panic (#21728) The outpost API controller shares a single *websocket.Conn across multiple goroutines: the event-handler loop, the 10s health ticker (SendEventHello), the shutdown path (WriteMessage close), initEvent writing the hello frame on (re)connect, and RAC session handlers that also invoke SendEventHello. gorilla/websocket explicitly documents that concurrent WriteMessage/WriteJSON calls are unsafe and will panic with "concurrent write to websocket connection", which takes the outpost (and embedded-outpost authentik-server) pod down. Fix by adding a sync.Mutex on APIController guarding every write path on eventConn (initEvent hello, Shutdown close message, SendEventHello). Reads (ReadJSON in startEventHandler) are left unsynchronized as gorilla permits a single concurrent reader alongside a writer. Minimal, localized change: no API changes, no behavior changes, writes are already infrequent so lock contention is negligible. Refs #11090 Co-authored-by: curiosity --- internal/outpost/ak/api.go | 2 ++ internal/outpost/ak/api_event.go | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/internal/outpost/ak/api.go b/internal/outpost/ak/api.go index 8cb5c4830a..083fd66c9c 100644 --- a/internal/outpost/ak/api.go +++ b/internal/outpost/ak/api.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "runtime" + "sync" "syscall" "time" @@ -45,6 +46,7 @@ type APIController struct { reloadOffset time.Duration eventConn *websocket.Conn + eventConnMu sync.Mutex lastWsReconnect time.Time wsIsReconnecting bool eventHandlers []EventHandler diff --git a/internal/outpost/ak/api_event.go b/internal/outpost/ak/api_event.go index 3bc64664f3..b9dd0901a6 100644 --- a/internal/outpost/ak/api_event.go +++ b/internal/outpost/ak/api_event.go @@ -77,7 +77,12 @@ func (ac *APIController) initEvent(outpostUUID string, attempt int) error { Instruction: EventKindHello, Args: ac.getEventPingArgs(), } + // Serialize this write against concurrent SendEventHello callers (health + // ticker, RAC handlers) sharing the same *websocket.Conn. Gorilla's Conn + // does not permit concurrent writes. + ac.eventConnMu.Lock() err = ws.WriteJSON(msg) + ac.eventConnMu.Unlock() if err != nil { ac.logger.WithField("logger", "authentik.outpost.events").WithError(err).Warning("Failed to hello to authentik") return err @@ -91,7 +96,9 @@ func (ac *APIController) initEvent(outpostUUID string, attempt int) error { func (ac *APIController) Shutdown() { // Cleanly close the connection by sending a close message and then // waiting (with timeout) for the server to close the connection. + ac.eventConnMu.Lock() err := ac.eventConn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + ac.eventConnMu.Unlock() if err != nil { ac.logger.WithError(err).Warning("failed to write close message") return @@ -252,6 +259,10 @@ func (a *APIController) SendEventHello(args map[string]any) error { Instruction: EventKindHello, Args: allArgs, } + // Gorilla *websocket.Conn does not permit concurrent writes. This method + // is invoked from the health ticker and from RAC session handlers. + a.eventConnMu.Lock() err := a.eventConn.WriteJSON(aliveMsg) + a.eventConnMu.Unlock() return err }