blender/flamenco
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Manager: remove GORM annotations and last dependencies

Browse Source 
Remove GORM struct annotations/tags and references to GORM types.

Ref: #104305
This commit is contained in:
Sybren A. Stüvel 2024-09-26 23:03:43 +02:00
parent 816046663e
commit 21cf3c47f9
11 changed files with 71 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/manager/persistence/db.go
View File

@ -29,7 +29,7 @@ type DB struct {
// Soft deletion is not used by Flamenco. If it ever becomes necessary to // Soft deletion is not used by Flamenco. If it ever becomes necessary to
// support soft-deletion, see https://gorm.io/docs/delete.html#Soft-Delete // support soft-deletion, see https://gorm.io/docs/delete.html#Soft-Delete
type Model struct { type Model struct {
ID uint `gorm:"primarykey"` ID uint
CreatedAt time.Time CreatedAt time.Time
UpdatedAt time.Time UpdatedAt time.Time
} }
@ -167,8 +167,6 @@ type queriesTX struct {
} }
// queries returns the SQLC Queries struct, connected to this database. // queries returns the SQLC Queries struct, connected to this database.
// It is intended that all GORM queries will be migrated to use this interface
// instead.
// //
// After calling this function, all queries should use this transaction until it // After calling this function, all queries should use this transaction until it
// is closed (either committed or rolled back). Otherwise SQLite will deadlock, // is closed (either committed or rolled back). Otherwise SQLite will deadlock,

23
internal/manager/persistence/errors.go
View File

@ -5,16 +5,13 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"gorm.io/gorm"
) )
var ( var (
// TODO: let these errors wrap database/sql.ErrNoRows. ErrJobNotFound = PersistenceError{Message: "job not found", Err: sql.ErrNoRows}
ErrJobNotFound = PersistenceError{Message: "job not found", Err: gorm.ErrRecordNotFound} ErrTaskNotFound = PersistenceError{Message: "task not found", Err: sql.ErrNoRows}
ErrTaskNotFound = PersistenceError{Message: "task not found", Err: gorm.ErrRecordNotFound} ErrWorkerNotFound = PersistenceError{Message: "worker not found", Err: sql.ErrNoRows}
ErrWorkerNotFound = PersistenceError{Message: "worker not found", Err: gorm.ErrRecordNotFound} ErrWorkerTagNotFound = PersistenceError{Message: "worker tag not found", Err: sql.ErrNoRows}
ErrWorkerTagNotFound = PersistenceError{Message: "worker tag not found", Err: gorm.ErrRecordNotFound}
ErrDeletingWithoutFK = errors.New("refusing to delete a job when foreign keys are not enabled on the database") ErrDeletingWithoutFK = errors.New("refusing to delete a job when foreign keys are not enabled on the database")
@ -80,9 +77,6 @@ func wrapError(errorToWrap error, message string, format ...interface{}) error {
// This helps to keep Gorm as "implementation detail" of the persistence layer. // This helps to keep Gorm as "implementation detail" of the persistence layer.
func translateGormJobError(err error) error { func translateGormJobError(err error) error {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return ErrTaskNotFound
}
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrJobNotFound return ErrJobNotFound
} }
return err return err
@ -94,9 +88,6 @@ func translateGormTaskError(err error) error {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return ErrTaskNotFound return ErrTaskNotFound
} }
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrTaskNotFound
}
return err return err
} }
@ -106,9 +97,6 @@ func translateGormWorkerError(err error) error {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return ErrWorkerNotFound return ErrWorkerNotFound
} }
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrWorkerNotFound
}
return err return err
} }
@ -118,8 +106,5 @@ func translateGormWorkerTagError(err error) error {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return ErrWorkerTagNotFound return ErrWorkerTagNotFound
} }
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrWorkerTagNotFound
}
return err return err
} }

10
internal/manager/persistence/errors_test.go
View File

@ -2,16 +2,16 @@
package persistence package persistence
import ( import (
"database/sql"
"errors" "errors"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"gorm.io/gorm"
) )
func TestNotFoundErrors(t *testing.T) { func TestNotFoundErrors(t *testing.T) {
assert.ErrorIs(t, ErrJobNotFound, gorm.ErrRecordNotFound) assert.ErrorIs(t, ErrJobNotFound, sql.ErrNoRows)
assert.ErrorIs(t, ErrTaskNotFound, gorm.ErrRecordNotFound) assert.ErrorIs(t, ErrTaskNotFound, sql.ErrNoRows)
assert.Contains(t, ErrJobNotFound.Error(), "job") assert.Contains(t, ErrJobNotFound.Error(), "job")
assert.Contains(t, ErrTaskNotFound.Error(), "task") assert.Contains(t, ErrTaskNotFound.Error(), "task")
@ -19,7 +19,7 @@ func TestNotFoundErrors(t *testing.T) {
func TestTranslateGormJobError(t *testing.T) { func TestTranslateGormJobError(t *testing.T) {
assert.Nil(t, translateGormJobError(nil)) assert.Nil(t, translateGormJobError(nil))
assert.Equal(t, ErrJobNotFound, translateGormJobError(gorm.ErrRecordNotFound)) assert.Equal(t, ErrJobNotFound, translateGormJobError(sql.ErrNoRows))
otherError := errors.New("this error is not special for this function") otherError := errors.New("this error is not special for this function")
assert.Equal(t, otherError, translateGormJobError(otherError)) assert.Equal(t, otherError, translateGormJobError(otherError))
@ -27,7 +27,7 @@ func TestTranslateGormJobError(t *testing.T) {
func TestTranslateGormTaskError(t *testing.T) { func TestTranslateGormTaskError(t *testing.T) {
assert.Nil(t, translateGormTaskError(nil)) assert.Nil(t, translateGormTaskError(nil))
assert.Equal(t, ErrTaskNotFound, translateGormTaskError(gorm.ErrRecordNotFound)) assert.Equal(t, ErrTaskNotFound, translateGormTaskError(sql.ErrNoRows))
otherError := errors.New("this error is not special for this function") otherError := errors.New("this error is not special for this function")
assert.Equal(t, otherError, translateGormTaskError(otherError)) assert.Equal(t, otherError, translateGormTaskError(otherError))

5
internal/manager/persistence/integrity.go
View File

@ -148,11 +148,6 @@ func (db *DB) pragmaForeignKeyCheck(ctx context.Context) (ok bool) {
// ensureForeignKeysEnabled checks whether foreign keys are enabled, and if not, // ensureForeignKeysEnabled checks whether foreign keys are enabled, and if not,
// tries to enable them. // tries to enable them.
//
// This is likely caused by either GORM or its embedded SQLite creating a new
// connection to the low-level SQLite driver. Unfortunately the GORM-embedded
// SQLite doesn't have an 'on-connect' callback function to always enable
// foreign keys.
func (db *DB) ensureForeignKeysEnabled(ctx context.Context) { func (db *DB) ensureForeignKeysEnabled(ctx context.Context) {
fkEnabled, err := db.areForeignKeysEnabled(ctx) fkEnabled, err := db.areForeignKeysEnabled(ctx)

70
internal/manager/persistence/jobs.go
View File

@ -13,7 +13,6 @@ import (
"time" "time"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"gorm.io/gorm"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers" "projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/persistence/sqlc" "projects.blender.org/studio/flamenco/internal/manager/persistence/sqlc"
@ -22,23 +21,23 @@ import (
type Job struct { type Job struct {
Model Model
UUID string `gorm:"type:char(36);default:'';unique;index"` UUID string
Name string `gorm:"type:varchar(64);default:''"` Name string
JobType string `gorm:"type:varchar(32);default:''"` JobType string
Priority int `gorm:"type:smallint;default:0"` Priority int
Status api.JobStatus `gorm:"type:varchar(32);default:''"` Status api.JobStatus
Activity string `gorm:"type:varchar(255);default:''"` Activity string
Settings StringInterfaceMap `gorm:"type:jsonb"` Settings StringInterfaceMap
Metadata StringStringMap `gorm:"type:jsonb"` Metadata StringStringMap
DeleteRequestedAt sql.NullTime DeleteRequestedAt sql.NullTime
Storage JobStorageInfo `gorm:"embedded;embeddedPrefix:storage_"` Storage JobStorageInfo
WorkerTagID *uint WorkerTagID *uint
WorkerTag *WorkerTag `gorm:"foreignkey:WorkerTagID;references:ID;constraint:OnDelete:SET NULL"` WorkerTag *WorkerTag
} }
type StringInterfaceMap map[string]interface{} type StringInterfaceMap map[string]interface{}
@ -54,43 +53,32 @@ func (j *Job) DeleteRequested() bool {
// files. // files.
type JobStorageInfo struct { type JobStorageInfo struct {
// ShamanCheckoutID is only set when the job was actually using Shaman storage. // ShamanCheckoutID is only set when the job was actually using Shaman storage.
ShamanCheckoutID string `gorm:"type:varchar(255);default:''"` ShamanCheckoutID string
} }
type Task struct { type Task struct {
Model Model
UUID string `gorm:"type:char(36);default:'';unique;index"` UUID string
Name string `gorm:"type:varchar(64);default:''"` Name string
Type string `gorm:"type:varchar(32);default:''"` Type string
JobID uint `gorm:"default:0"` JobID uint
Job *Job `gorm:"foreignkey:JobID;references:ID;constraint:OnDelete:CASCADE"` Job *Job
JobUUID string `gorm:"-"` // Fetched by SQLC, handled by GORM in Task.AfterFind() JobUUID string // Fetched by SQLC, handled by GORM in Task.AfterFind()
Priority int `gorm:"type:smallint;default:50"` Priority int
Status api.TaskStatus `gorm:"type:varchar(16);default:''"` Status api.TaskStatus
// Which worker is/was working on this. // Which worker is/was working on this.
WorkerID *uint WorkerID *uint
Worker *Worker `gorm:"foreignkey:WorkerID;references:ID;constraint:OnDelete:SET NULL"` Worker *Worker
WorkerUUID string `gorm:"-"` // Fetched by SQLC, handled by GORM in Task.AfterFind() WorkerUUID string // Fetched by SQLC, handled by GORM in Task.AfterFind()
LastTouchedAt time.Time `gorm:"index"` // Should contain UTC timestamps. LastTouchedAt time.Time // Should contain UTC timestamps.
// Dependencies are tasks that need to be completed before this one can run. // Dependencies are tasks that need to be completed before this one can run.
Dependencies []*Task `gorm:"many2many:task_dependencies;constraint:OnDelete:CASCADE"` Dependencies []*Task
Commands Commands `gorm:"type:jsonb"` Commands Commands
Activity string `gorm:"type:varchar(255);default:''"` Activity string
}
// AfterFind updates the task JobUUID and WorkerUUID fields from its job/worker, if known.
func (t *Task) AfterFind(tx *gorm.DB) error {
if t.JobUUID == "" && t.Job != nil {
t.JobUUID = t.Job.UUID
}
if t.WorkerUUID == "" && t.Worker != nil {
t.WorkerUUID = t.Worker.UUID
}
return nil
} }
type Commands []Command type Commands []Command
@ -138,10 +126,10 @@ type TaskFailure struct {
// Don't include the standard Gorm ID, UpdatedAt, or DeletedAt fields, as they're useless here. // Don't include the standard Gorm ID, UpdatedAt, or DeletedAt fields, as they're useless here.
// Entries will never be updated, and should never be soft-deleted but just purged from existence. // Entries will never be updated, and should never be soft-deleted but just purged from existence.
CreatedAt time.Time CreatedAt time.Time
TaskID uint `gorm:"primaryKey;autoIncrement:false"` TaskID uint
Task *Task `gorm:"foreignkey:TaskID;references:ID;constraint:OnDelete:CASCADE"` Task *Task
WorkerID uint `gorm:"primaryKey;autoIncrement:false"` WorkerID uint
Worker *Worker `gorm:"foreignkey:WorkerID;references:ID;constraint:OnDelete:CASCADE"` Worker *Worker
} }
// StoreJob stores an AuthoredJob and its tasks, and saves it to the database. // StoreJob stores an AuthoredJob and its tasks, and saves it to the database.

10
internal/manager/persistence/jobs_blocklist.go
View File

@ -17,13 +17,13 @@ type JobBlock struct {
ID uint ID uint
CreatedAt time.Time CreatedAt time.Time
JobID uint `gorm:"default:0;uniqueIndex:job_worker_tasktype"` JobID uint
Job *Job `gorm:"foreignkey:JobID;references:ID;constraint:OnDelete:CASCADE"` Job *Job
WorkerID uint `gorm:"default:0;uniqueIndex:job_worker_tasktype"` WorkerID uint
Worker *Worker `gorm:"foreignkey:WorkerID;references:ID;constraint:OnDelete:CASCADE"` Worker *Worker
TaskType string `gorm:"uniqueIndex:job_worker_tasktype"` TaskType string
} }
// AddWorkerToJobBlocklist prevents this Worker of getting any task, of this type, on this job, from the task scheduler. // AddWorkerToJobBlocklist prevents this Worker of getting any task, of this type, on this job, from the task scheduler.

4
internal/manager/persistence/last_rendered.go
View File

@ -15,8 +15,8 @@ import (
// This is used to show the global last-rendered image in the web interface. // This is used to show the global last-rendered image in the web interface.
type LastRendered struct { type LastRendered struct {
Model Model
JobID uint `gorm:"default:0"` JobID uint
Job *Job `gorm:"foreignkey:JobID;references:ID;constraint:OnDelete:CASCADE"` Job *Job
} }
// SetLastRendered sets this job as the one with the most recent rendered image. // SetLastRendered sets this job as the one with the most recent rendered image.

8
internal/manager/persistence/sqlite_busy.go
View File

@ -26,14 +26,6 @@ func isDatabaseBusyError(err error) bool {
if err == nil { if err == nil {
return false return false
} }
// The exact error type is dependent on deep dependencies of GORM. The code
// below used to work, until an upgrade of one of those dependencies. This is
// why I feel it's more future-proof to just check for SQLITE_BUSY in the
// error text.
//
// sqlErr, ok := err.(*sqlite.Error)
// return ok && sqlErr.Code() == sqlite_lib.SQLITE_BUSY
return strings.Contains(err.Error(), "SQLITE_BUSY") return strings.Contains(err.Error(), "SQLITE_BUSY")
} }

12
internal/manager/persistence/worker_sleep_schedule.go
View File

@ -18,16 +18,16 @@ import (
type SleepSchedule struct { type SleepSchedule struct {
Model Model
WorkerID uint `gorm:"default:0;unique;index"` WorkerID uint
Worker *Worker `gorm:"foreignkey:WorkerID;references:ID;constraint:OnDelete:CASCADE"` Worker *Worker
IsActive bool `gorm:"default:false;index"` IsActive bool
// Space-separated two-letter strings indicating days of week the schedule is // Space-separated two-letter strings indicating days of week the schedule is
// active ("mo", "tu", etc.). Empty means "every day". // active ("mo", "tu", etc.). Empty means "every day".
DaysOfWeek string `gorm:"default:''"` DaysOfWeek string
StartTime TimeOfDay `gorm:"default:''"` StartTime TimeOfDay
EndTime TimeOfDay `gorm:"default:''"` EndTime TimeOfDay
NextCheck time.Time NextCheck time.Time
} }

8
internal/manager/persistence/worker_tag.go
View File

@ -12,11 +12,11 @@ import (
type WorkerTag struct { type WorkerTag struct {
Model Model
UUID string `gorm:"type:char(36);default:'';unique;index"` UUID string
Name string `gorm:"type:varchar(64);default:'';unique"` Name string
Description string `gorm:"type:varchar(255);default:''"` Description string
Workers []*Worker `gorm:"many2many:worker_tag_membership;constraint:OnDelete:CASCADE"` Workers []*Worker
} }
func (db *DB) CreateWorkerTag(ctx context.Context, wc *WorkerTag) error { func (db *DB) CreateWorkerTag(ctx context.Context, wc *WorkerTag) error {

31
internal/manager/persistence/workers.go
View File

@ -11,32 +11,31 @@ import (
"time" "time"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"gorm.io/gorm"
"projects.blender.org/studio/flamenco/internal/manager/persistence/sqlc" "projects.blender.org/studio/flamenco/internal/manager/persistence/sqlc"
"projects.blender.org/studio/flamenco/pkg/api" "projects.blender.org/studio/flamenco/pkg/api"
) )
type Worker struct { type Worker struct {
Model Model
DeletedAt gorm.DeletedAt `gorm:"index"` DeletedAt sql.NullTime
UUID string `gorm:"type:char(36);default:'';unique;index"` UUID string
Secret string `gorm:"type:varchar(255);default:''"` Secret string
Name string `gorm:"type:varchar(64);default:''"` Name string
Address string `gorm:"type:varchar(39);default:'';index"` // 39 = max length of IPv6 address. Address string // 39 = max length of IPv6 address.
Platform string `gorm:"type:varchar(16);default:''"` Platform string
Software string `gorm:"type:varchar(32);default:''"` Software string
Status api.WorkerStatus `gorm:"type:varchar(16);default:''"` Status api.WorkerStatus
LastSeenAt time.Time `gorm:"index"` // Should contain UTC timestamps. LastSeenAt time.Time // Should contain UTC timestamps.
CanRestart bool `gorm:"type:smallint;default:false"` CanRestart bool
StatusRequested api.WorkerStatus `gorm:"type:varchar(16);default:''"` StatusRequested api.WorkerStatus
LazyStatusRequest bool `gorm:"type:smallint;default:false"` LazyStatusRequest bool
SupportedTaskTypes string `gorm:"type:varchar(255);default:''"` // comma-separated list of task types. SupportedTaskTypes string // comma-separated list of task types.
Tags []*WorkerTag `gorm:"many2many:worker_tag_membership;constraint:OnDelete:CASCADE"` Tags []*WorkerTag
} }
func (w *Worker) Identifier() string { func (w *Worker) Identifier() string {
@ -315,7 +314,7 @@ func convertSqlcWorker(worker sqlc.Worker) *Worker {
CreatedAt: worker.CreatedAt, CreatedAt: worker.CreatedAt,
UpdatedAt: worker.UpdatedAt.Time, UpdatedAt: worker.UpdatedAt.Time,
}, },
DeletedAt: gorm.DeletedAt(worker.DeletedAt), DeletedAt: worker.DeletedAt,
UUID: worker.UUID, UUID: worker.UUID,
Secret: worker.Secret, Secret: worker.Secret,