Manager: perform database integrity check every hour

Perform a database integrity check every hour. This check was already
performed at startup, in the main goroutine.

cmd/flamenco-manager/main.go

@@ -139,12 +139,6 @@ func runFlamencoManager() bool {
 	persist := openDB(*configService)
 	defer persist.Close()
 
-	// Disabled for now. `VACUUM` locks the database, which means that other
-	// queries can fail with a "database is locked (5) (SQLITE_BUSY)" error. This
-	// situation should be handled gracefully before reinstating the vacuum loop.
-	//
-	// go persist.PeriodicMaintenanceLoop(mainCtx)
-
 	timeService := clock.New()
 	compiler, err := job_compilers.Load(timeService)
 	if err != nil {
@@ -196,6 +190,14 @@ func runFlamencoManager() bool {
 		lastRender.Run(mainCtx)
 	}()
 
+	// Run a periodic integrity check on the database.
+	// When that check fails, the entire application should shut down.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		persist.PeriodicIntegrityCheck(mainCtx, mainCtxCancel)
+	}()
+
 	// Start the web server.
 	wg.Add(1)
 	go func() {

internal/manager/persistence/db.go

@@ -15,8 +15,6 @@ import (
 	"github.com/glebarez/sqlite"
 )
 
-const checkPeriod = 1 * time.Hour
-
 // DB provides the database interface.
 type DB struct {
 	gormDB *gorm.DB

internal/manager/persistence/integrity.go

@@ -12,7 +12,10 @@ import (
 
 var ErrIntegrity = errors.New("database integrity check failed")
 
-const integrityCheckTimeout = 2 * time.Second
+const (
+	integrityCheckTimeout = 2 * time.Second
+	integrityCheckPeriod  = 1 * time.Hour
+)
 
 type PragmaIntegrityCheckResult struct {
 	Description string `gorm:"column:integrity_check"`
@@ -25,6 +28,27 @@ type PragmaForeignKeyCheckResult struct {
 	FKID   int    `gorm:"column:fkid"`
 }
 
+// PeriodicIntegrityCheck periodically checks the database integrity.
+// This function only returns when the context is done.
+func (db *DB) PeriodicIntegrityCheck(ctx context.Context, onErrorCallback func()) {
+	log.Debug().Msg("database: periodic integrity check loop starting")
+	defer log.Debug().Msg("database: periodic integrity check loop stopping")
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(integrityCheckPeriod):
+		}
+
+		ok := db.performIntegrityCheck(ctx)
+		if !ok {
+			log.Error().Msg("database: periodic integrity check failed")
+			onErrorCallback()
+		}
+	}
+}
+
 // performIntegrityCheck uses a few 'pragma' SQL statements to do some integrity checking.
 // Returns true on OK, false if there was an issue. Issues are always logged.
 func (db *DB) performIntegrityCheck(ctx context.Context) (ok bool) {