From b5b7b228eda012ca754806f5698cf0abf99de12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 28 Feb 2022 12:07:10 +0100 Subject: [PATCH] Worker: capture panics in 'active' state and cleanly sign off before dying --- internal/worker/state_awake.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/worker/state_awake.go b/internal/worker/state_awake.go index f22c37a8..164b7c90 100644 --- a/internal/worker/state_awake.go +++ b/internal/worker/state_awake.go @@ -37,17 +37,28 @@ const ( func (w *Worker) gotoStateAwake(ctx context.Context) { w.stateMutex.Lock() - defer w.stateMutex.Unlock() - w.state = api.WorkerStatusAwake + w.stateMutex.Unlock() w.doneWg.Add(2) w.ackStateChange(ctx, w.state) + go w.runStateAwake(ctx) } // runStateAwake fetches a task and executes it, in an endless loop. func (w *Worker) runStateAwake(ctx context.Context) { + defer func() { + err := recover() + if err != nil { + w.SignOff(ctx) + log.Panic(). + Interface("panic", err). + Str("workerStatus", string(w.state)). + Msg("panic, so signed off and going to stop") + } + }() + defer w.doneWg.Done() defer log.Debug().Msg("stopping state 'awake'")