From 15e37458202c16cc68272e447106053bbc48894e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Thu, 24 Nov 2022 17:18:06 +0100 Subject: [PATCH] Manager: SQLite WAL journal + NORMAL sync mode Run `PRAGMA journal_mode = WAL` and `PRAGMA synchronous = normal` when connecting to the SQLite database. This enables the write-ahead-log journal mode, which makes it safe to enable "normal" synchronisation (instead of the default "full" synchronisation). --- CHANGELOG.md | 2 +- internal/manager/persistence/db.go | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d19d51..9b3e678b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ bugs in actually-released versions. - Fix issue where workers would switch immediately on a state change request, even if it was of the "after task is finished" kind. - Add-on: Do a "pre-submission check" before sending files to the farm. This should provide submission errors earlier in the process, without waiting for files to be collected. - Worker: better handling of long lines from Blender/FFmpeg, splitting them up at character boundaries. - +- Manager: change SQLite parameters to have write-through-log journalling and less filesystem synchronisation. This reduces I/O load on the Manager. ## 3.1 - released 2022-10-18 diff --git a/internal/manager/persistence/db.go b/internal/manager/persistence/db.go index 58dc2001..53f10e85 100644 --- a/internal/manager/persistence/db.go +++ b/internal/manager/persistence/db.go @@ -97,6 +97,17 @@ func openDBWithConfig(dsn string, config *gorm.Config) (*DB, error) { log.Error().Msg("SQLite database does not want to enable foreign keys, this may cause data loss") } + // Write-ahead-log journal may improve writing speed. + log.Trace().Msg("enabling SQLite write-ahead-log journal mode") + if tx := gormDB.Exec("PRAGMA journal_mode = WAL"); tx.Error != nil { + return nil, fmt.Errorf("enabling SQLite write-ahead-log journal mode: %w", tx.Error) + } + // Switching from 'full' (default) to 'normal' sync may improve writing speed. + log.Trace().Msg("enabling SQLite 'normal' synchronisation") + if tx := gormDB.Exec("PRAGMA synchronous = normal"); tx.Error != nil { + return nil, fmt.Errorf("enabling SQLite 'normal' sync mode: %w", tx.Error) + } + db := DB{ gormDB: gormDB, }