diff --git a/cmd/flamenco-manager/main.go b/cmd/flamenco-manager/main.go index 7e54b24d..572559ec 100644 --- a/cmd/flamenco-manager/main.go +++ b/cmd/flamenco-manager/main.go @@ -139,12 +139,6 @@ func runFlamencoManager() bool { persist := openDB(*configService) defer persist.Close() - // Disabled for now. `VACUUM` locks the database, which means that other - // queries can fail with a "database is locked (5) (SQLITE_BUSY)" error. This - // situation should be handled gracefully before reinstating the vacuum loop. - // - // go persist.PeriodicMaintenanceLoop(mainCtx) - timeService := clock.New() compiler, err := job_compilers.Load(timeService) if err != nil { @@ -196,6 +190,14 @@ func runFlamencoManager() bool { lastRender.Run(mainCtx) }() + // Run a periodic integrity check on the database. + // When that check fails, the entire application should shut down. + wg.Add(1) + go func() { + defer wg.Done() + persist.PeriodicIntegrityCheck(mainCtx, mainCtxCancel) + }() + // Start the web server. wg.Add(1) go func() { diff --git a/internal/manager/persistence/db.go b/internal/manager/persistence/db.go index 1701c8fc..0debe254 100644 --- a/internal/manager/persistence/db.go +++ b/internal/manager/persistence/db.go @@ -15,8 +15,6 @@ import ( "github.com/glebarez/sqlite" ) -const checkPeriod = 1 * time.Hour - // DB provides the database interface. type DB struct { gormDB *gorm.DB diff --git a/internal/manager/persistence/integrity.go b/internal/manager/persistence/integrity.go index 6f2f15c9..2686e7fb 100644 --- a/internal/manager/persistence/integrity.go +++ b/internal/manager/persistence/integrity.go @@ -12,7 +12,10 @@ import ( var ErrIntegrity = errors.New("database integrity check failed") -const integrityCheckTimeout = 2 * time.Second +const ( + integrityCheckTimeout = 2 * time.Second + integrityCheckPeriod = 1 * time.Hour +) type PragmaIntegrityCheckResult struct { Description string `gorm:"column:integrity_check"` @@ -25,6 +28,27 @@ type PragmaForeignKeyCheckResult struct { FKID int `gorm:"column:fkid"` } +// PeriodicIntegrityCheck periodically checks the database integrity. +// This function only returns when the context is done. +func (db *DB) PeriodicIntegrityCheck(ctx context.Context, onErrorCallback func()) { + log.Debug().Msg("database: periodic integrity check loop starting") + defer log.Debug().Msg("database: periodic integrity check loop stopping") + + for { + select { + case <-ctx.Done(): + return + case <-time.After(integrityCheckPeriod): + } + + ok := db.performIntegrityCheck(ctx) + if !ok { + log.Error().Msg("database: periodic integrity check failed") + onErrorCallback() + } + } +} + // performIntegrityCheck uses a few 'pragma' SQL statements to do some integrity checking. // Returns true on OK, false if there was an issue. Issues are always logged. func (db *DB) performIntegrityCheck(ctx context.Context) (ok bool) {