Worker: perform database migrations with Goose

Replace the GORM auto-migration with Goose. The latter uses hand-written
SQL files to migrate the database with understandable, explicit queries.

internal/worker/persistence/db.go

@@ -44,7 +44,7 @@ func OpenDB(ctx context.Context, dsn string) (*DB, error) {
 	// Perfom some maintenance at startup.
 	db.vacuum()
 
-	if err := db.migrate(); err != nil {
+	if err := db.migrate(ctx); err != nil {
 		return nil, err
 	}
 	log.Debug().Msg("database automigration succesful")

internal/worker/persistence/db_migration.go

@@ -3,15 +3,69 @@ package persistence
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 import (
+	"context"
+	"embed"
 	"fmt"
+	"strings"
+
+	goose "github.com/pressly/goose/v3"
+	"github.com/rs/zerolog/log"
 )
 
-func (db *DB) migrate() error {
+//go:embed migrations/*.sql
-	err := db.gormDB.AutoMigrate(
+var embedMigrations embed.FS
-		&TaskUpdate{},
+
-	)
+func (db *DB) migrate(ctx context.Context) error {
-	if err != nil {
+	// Set up Goose.
-		return fmt.Errorf("failed to automigrate database: %v", err)
+	gooseLogger := GooseLogger{}
+	goose.SetLogger(&gooseLogger)
+	goose.SetBaseFS(embedMigrations)
+	if err := goose.SetDialect("sqlite3"); err != nil {
+		log.Fatal().AnErr("cause", err).Msg("could not tell Goose to use sqlite3")
 	}
+
+	// Hook up Goose to the database.
+	lowLevelDB, err := db.gormDB.DB()
+	if err != nil {
+		log.Fatal().AnErr("cause", err).Msg("GORM would not give us its low-level interface")
+	}
+
+	// Disable foreign key constraints during the migrations. This is necessary
+	// for SQLite to do column renames / drops, as that requires creating a new
+	// table with the new schema, copying the data, dropping the old table, and
+	// moving the new one in its place. That table drop shouldn't trigger 'ON
+	// DELETE' actions on foreign keys.
+	//
+	// Since migration is 99% schema changes, and very little to no manipulation
+	// of data, foreign keys are disabled here instead of in the migration SQL
+	// files, so that it can't be forgotten.
+
+	if err := db.pragmaForeignKeys(false); err != nil {
+		log.Fatal().AnErr("cause", err).Msg("could not disable foreign key constraints before performing database migrations, please report a bug at https://flamenco.blender.org/get-involved")
+	}
+
+	// Run Goose.
+	log.Debug().Msg("migrating database with Goose")
+	if err := goose.UpContext(ctx, lowLevelDB, "migrations"); err != nil {
+		log.Fatal().AnErr("cause", err).Msg("could not migrate database to the latest version")
+	}
+
+	// Re-enable foreign key checks.
+	if err := db.pragmaForeignKeys(true); err != nil {
+		log.Fatal().AnErr("cause", err).Msg("could not re-enable foreign key constraints after performing database migrations, please report a bug at https://flamenco.blender.org/get-involved")
+	}
+
 	return nil
 }
+
+type GooseLogger struct{}
+
+func (gl *GooseLogger) Fatalf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	log.Fatal().Msg(strings.TrimSpace(msg))
+}
+
+func (gl *GooseLogger) Printf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	log.Debug().Msg(strings.TrimSpace(msg))
+}

internal/worker/persistence/migrations/0001_worker_v3_3.sql

@@ -0,0 +1,16 @@
+-- This is the initial Goose migration for Flamenco Worker. It recreates the
+-- schema that might exist already, hence the "IF NOT EXISTS" clauses.
+--
+-- WARNING: the 'Down' step will erase the entire database.
+--
+-- +goose Up
+CREATE TABLE IF NOT EXISTS `task_updates` (
+  `id` integer,
+  `created_at` datetime,
+  `task_id` varchar(36) DEFAULT "",
+  `payload` BLOB,
+  PRIMARY KEY (`id`)
+);
+
+-- +goose Down
+DROP TABLE `task_updates`;

internal/worker/persistence/migrations/README.md

@@ -0,0 +1,21 @@
+# SQL Migrations
+
+The files here are run by [Goose][goose], the database migration tool.
+
+[goose]: https://pkg.go.dev/github.com/pressly/goose/v3
+
+These are embedded into the Flamenco Worker executable, and automatically run on
+startup.
+
+## Foreign Key Constraints
+
+Foreign Key constraints (FKCs) are optional in SQLite, and always enabled by
+Flamenco Worker. These are temporarily disabled during database migration
+itself. This means you can replace a table like this, without `ON DELETE`
+effects running.
+
+```sql
+INSERT INTO `temp_table` SELECT * FROM `actual_table`;
+DROP TABLE `actual_table`;
+ALTER TABLE `temp_table` RENAME TO `actual_table`;
+```