diff --git a/internal/worker/persistence/db.go b/internal/worker/persistence/db.go index 07519e89..ff033875 100644 --- a/internal/worker/persistence/db.go +++ b/internal/worker/persistence/db.go @@ -44,7 +44,7 @@ func OpenDB(ctx context.Context, dsn string) (*DB, error) { // Perfom some maintenance at startup. db.vacuum() - if err := db.migrate(); err != nil { + if err := db.migrate(ctx); err != nil { return nil, err } log.Debug().Msg("database automigration succesful") diff --git a/internal/worker/persistence/db_migration.go b/internal/worker/persistence/db_migration.go index fc8b5e02..8b506d92 100644 --- a/internal/worker/persistence/db_migration.go +++ b/internal/worker/persistence/db_migration.go @@ -3,15 +3,69 @@ package persistence // SPDX-License-Identifier: GPL-3.0-or-later import ( + "context" + "embed" "fmt" + "strings" + + goose "github.com/pressly/goose/v3" + "github.com/rs/zerolog/log" ) -func (db *DB) migrate() error { - err := db.gormDB.AutoMigrate( - &TaskUpdate{}, - ) - if err != nil { - return fmt.Errorf("failed to automigrate database: %v", err) +//go:embed migrations/*.sql +var embedMigrations embed.FS + +func (db *DB) migrate(ctx context.Context) error { + // Set up Goose. + gooseLogger := GooseLogger{} + goose.SetLogger(&gooseLogger) + goose.SetBaseFS(embedMigrations) + if err := goose.SetDialect("sqlite3"); err != nil { + log.Fatal().AnErr("cause", err).Msg("could not tell Goose to use sqlite3") } + + // Hook up Goose to the database. + lowLevelDB, err := db.gormDB.DB() + if err != nil { + log.Fatal().AnErr("cause", err).Msg("GORM would not give us its low-level interface") + } + + // Disable foreign key constraints during the migrations. This is necessary + // for SQLite to do column renames / drops, as that requires creating a new + // table with the new schema, copying the data, dropping the old table, and + // moving the new one in its place. That table drop shouldn't trigger 'ON + // DELETE' actions on foreign keys. + // + // Since migration is 99% schema changes, and very little to no manipulation + // of data, foreign keys are disabled here instead of in the migration SQL + // files, so that it can't be forgotten. + + if err := db.pragmaForeignKeys(false); err != nil { + log.Fatal().AnErr("cause", err).Msg("could not disable foreign key constraints before performing database migrations, please report a bug at https://flamenco.blender.org/get-involved") + } + + // Run Goose. + log.Debug().Msg("migrating database with Goose") + if err := goose.UpContext(ctx, lowLevelDB, "migrations"); err != nil { + log.Fatal().AnErr("cause", err).Msg("could not migrate database to the latest version") + } + + // Re-enable foreign key checks. + if err := db.pragmaForeignKeys(true); err != nil { + log.Fatal().AnErr("cause", err).Msg("could not re-enable foreign key constraints after performing database migrations, please report a bug at https://flamenco.blender.org/get-involved") + } + return nil } + +type GooseLogger struct{} + +func (gl *GooseLogger) Fatalf(format string, v ...interface{}) { + msg := fmt.Sprintf(format, v...) + log.Fatal().Msg(strings.TrimSpace(msg)) +} + +func (gl *GooseLogger) Printf(format string, v ...interface{}) { + msg := fmt.Sprintf(format, v...) + log.Debug().Msg(strings.TrimSpace(msg)) +} diff --git a/internal/worker/persistence/migrations/0001_worker_v3_3.sql b/internal/worker/persistence/migrations/0001_worker_v3_3.sql new file mode 100644 index 00000000..c47c1c11 --- /dev/null +++ b/internal/worker/persistence/migrations/0001_worker_v3_3.sql @@ -0,0 +1,16 @@ +-- This is the initial Goose migration for Flamenco Worker. It recreates the +-- schema that might exist already, hence the "IF NOT EXISTS" clauses. +-- +-- WARNING: the 'Down' step will erase the entire database. +-- +-- +goose Up +CREATE TABLE IF NOT EXISTS `task_updates` ( + `id` integer, + `created_at` datetime, + `task_id` varchar(36) DEFAULT "", + `payload` BLOB, + PRIMARY KEY (`id`) +); + +-- +goose Down +DROP TABLE `task_updates`; diff --git a/internal/worker/persistence/migrations/README.md b/internal/worker/persistence/migrations/README.md new file mode 100644 index 00000000..9a127137 --- /dev/null +++ b/internal/worker/persistence/migrations/README.md @@ -0,0 +1,21 @@ +# SQL Migrations + +The files here are run by [Goose][goose], the database migration tool. + +[goose]: https://pkg.go.dev/github.com/pressly/goose/v3 + +These are embedded into the Flamenco Worker executable, and automatically run on +startup. + +## Foreign Key Constraints + +Foreign Key constraints (FKCs) are optional in SQLite, and always enabled by +Flamenco Worker. These are temporarily disabled during database migration +itself. This means you can replace a table like this, without `ON DELETE` +effects running. + +```sql +INSERT INTO `temp_table` SELECT * FROM `actual_table`; +DROP TABLE `actual_table`; +ALTER TABLE `temp_table` RENAME TO `actual_table`; +```