From 931fd1a24c5bffbbc8d9d1d85cbb197c3cc01891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Fri, 28 Jan 2022 14:53:02 +0100 Subject: [PATCH] Move db/jobs tests to `persistence/jobs.go` --- internal/manager/persistence/db.go | 73 ----------- internal/manager/persistence/db_test.go | 85 ------------- .../persistence/{models.go => jobs.go} | 73 +++++++++++ internal/manager/persistence/jobs_test.go | 115 ++++++++++++++++++ 4 files changed, 188 insertions(+), 158 deletions(-) rename internal/manager/persistence/{models.go => jobs.go} (58%) create mode 100644 internal/manager/persistence/jobs_test.go diff --git a/internal/manager/persistence/db.go b/internal/manager/persistence/db.go index fa7e4260..add9eaca 100644 --- a/internal/manager/persistence/db.go +++ b/internal/manager/persistence/db.go @@ -23,14 +23,10 @@ package persistence import ( "context" - "fmt" "github.com/rs/zerolog/log" "gorm.io/driver/postgres" "gorm.io/gorm" - - "gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers" - "gitlab.com/blender/flamenco-ng-poc/pkg/api" ) // TODO : have this configurable from the CLI. @@ -68,72 +64,3 @@ func openDB(ctx context.Context, uri string) (*DB, error) { } return &db, nil } - -func (db *DB) StoreJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error { - return db.gormDB.Transaction(func(tx *gorm.DB) error { - // TODO: separate conversion of struct types from storing things in the database. - dbJob := Job{ - UUID: authoredJob.JobID, - Name: authoredJob.Name, - JobType: authoredJob.JobType, - Priority: int8(authoredJob.Priority), - Settings: StringInterfaceMap(authoredJob.Settings), - Metadata: StringStringMap(authoredJob.Metadata), - } - - if err := db.gormDB.Create(&dbJob).Error; err != nil { - return fmt.Errorf("error storing job: %v", err) - } - - for _, authoredTask := range authoredJob.Tasks { - var commands []Command - for _, authoredCommand := range authoredTask.Commands { - commands = append(commands, Command{ - Type: authoredCommand.Type, - Parameters: StringInterfaceMap(authoredCommand.Parameters), - }) - } - - dbTask := Task{ - Name: authoredTask.Name, - Type: authoredTask.Type, - Job: &dbJob, - Priority: authoredTask.Priority, - Status: string(api.TaskStatusProcessing), // TODO: is this the right place to set the default status? - // TODO: store dependencies - Commands: commands, - } - if err := db.gormDB.Create(&dbTask).Error; err != nil { - return fmt.Errorf("error storing task: %v", err) - } - } - - return nil - }) -} - -func (db *DB) FetchJob(ctx context.Context, jobID string) (*api.Job, error) { - dbJob := Job{} - findResult := db.gormDB.First(&dbJob, "uuid = ?", jobID) - if findResult.Error != nil { - return nil, findResult.Error - } - - apiJob := api.Job{ - SubmittedJob: api.SubmittedJob{ - Name: dbJob.Name, - Priority: int(dbJob.Priority), - Type: dbJob.JobType, - }, - - Id: dbJob.UUID, - Created: dbJob.CreatedAt, - Updated: dbJob.UpdatedAt, - Status: api.JobStatus(dbJob.Status), - } - - apiJob.Settings = &api.JobSettings{AdditionalProperties: dbJob.Settings} - apiJob.Metadata = &api.JobMetadata{AdditionalProperties: dbJob.Metadata} - - return &apiJob, nil -} diff --git a/internal/manager/persistence/db_test.go b/internal/manager/persistence/db_test.go index b2569d65..a9b4637f 100644 --- a/internal/manager/persistence/db_test.go +++ b/internal/manager/persistence/db_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/stretchr/testify/assert" - "gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers" "golang.org/x/net/context" "gorm.io/gorm" ) @@ -59,87 +58,3 @@ func createTestDB(t *testing.T) *DB { return db } - -func TestStoreAuthoredJob(t *testing.T) { - db := createTestDB(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - task1 := job_compilers.AuthoredTask{ - Name: "render-1-3", - Type: "blender", - Commands: []job_compilers.AuthoredCommand{ - { - Type: "blender-render", - Parameters: job_compilers.AuthoredCommandParameters{ - "exe": "{blender}", - "blendfile": "/path/to/file.blend", - "args": []interface{}{ - "--render-output", "/path/to/output/######.png", - "--render-format", "PNG", - "--render-frame", "1-3", - }, - }}, - }, - } - - task2 := task1 - task2.Name = "render-4-6" - task2.Commands[0].Parameters["frames"] = "4-6" - - task3 := job_compilers.AuthoredTask{ - Name: "preview-video", - Type: "ffmpeg", - Commands: []job_compilers.AuthoredCommand{ - { - Type: "merge-frames-to-video", - Parameters: job_compilers.AuthoredCommandParameters{ - "images": "/path/to/output/######.png", - "output": "/path/to/output/preview.mkv", - "ffmpegParams": "-c:v hevc -crf 31", - }}, - }, - Dependencies: []*job_compilers.AuthoredTask{&task1, &task2}, - } - - job := job_compilers.AuthoredJob{ - JobID: "263fd47e-b9f8-4637-b726-fd7e47ecfdae", - Name: "Test job", - Priority: 50, - Settings: job_compilers.JobSettings{ - "frames": "1-6", - "chunk_size": 3.0, // The roundtrip to JSON in PostgreSQL can make this a float. - }, - Metadata: job_compilers.JobMetadata{ - "author": "Sybren", - "project": "Sprite Fright", - }, - Tasks: []job_compilers.AuthoredTask{task1, task2, task3}, - } - - err := db.StoreJob(ctx, job) - assert.NoError(t, err) - - fetchedJob, err := db.FetchJob(ctx, job.JobID) - assert.NoError(t, err) - assert.NotNil(t, fetchedJob) - - // Test contents of fetched job - assert.Equal(t, job.JobID, fetchedJob.Id) - assert.Equal(t, job.Name, fetchedJob.Name) - assert.Equal(t, job.JobType, fetchedJob.Type) - assert.Equal(t, job.Priority, fetchedJob.Priority) - assert.EqualValues(t, map[string]interface{}(job.Settings), fetchedJob.Settings.AdditionalProperties) - assert.EqualValues(t, map[string]string(job.Metadata), fetchedJob.Metadata.AdditionalProperties) - - // Fetch tasks of job. - var dbJob Job - tx := db.gormDB.Where(&Job{UUID: job.JobID}).Find(&dbJob) - assert.NoError(t, tx.Error) - var tasks []Task - tx = db.gormDB.Where("job_id = ?", dbJob.ID).Find(&tasks) - assert.NoError(t, tx.Error) - - assert.Len(t, tasks, 3) -} diff --git a/internal/manager/persistence/models.go b/internal/manager/persistence/jobs.go similarity index 58% rename from internal/manager/persistence/models.go rename to internal/manager/persistence/jobs.go index c4e1012a..105e671e 100644 --- a/internal/manager/persistence/models.go +++ b/internal/manager/persistence/jobs.go @@ -21,10 +21,14 @@ package persistence * ***** END GPL LICENSE BLOCK ***** */ import ( + "context" "database/sql/driver" "encoding/json" "errors" + "fmt" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers" + "gitlab.com/blender/flamenco-ng-poc/pkg/api" "gorm.io/gorm" ) @@ -101,3 +105,72 @@ func (js *StringStringMap) Scan(value interface{}) error { } return json.Unmarshal(b, &js) } + +func (db *DB) StoreJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error { + return db.gormDB.Transaction(func(tx *gorm.DB) error { + // TODO: separate conversion of struct types from storing things in the database. + dbJob := Job{ + UUID: authoredJob.JobID, + Name: authoredJob.Name, + JobType: authoredJob.JobType, + Priority: int8(authoredJob.Priority), + Settings: StringInterfaceMap(authoredJob.Settings), + Metadata: StringStringMap(authoredJob.Metadata), + } + + if err := db.gormDB.Create(&dbJob).Error; err != nil { + return fmt.Errorf("error storing job: %v", err) + } + + for _, authoredTask := range authoredJob.Tasks { + var commands []Command + for _, authoredCommand := range authoredTask.Commands { + commands = append(commands, Command{ + Type: authoredCommand.Type, + Parameters: StringInterfaceMap(authoredCommand.Parameters), + }) + } + + dbTask := Task{ + Name: authoredTask.Name, + Type: authoredTask.Type, + Job: &dbJob, + Priority: authoredTask.Priority, + Status: string(api.TaskStatusProcessing), // TODO: is this the right place to set the default status? + // TODO: store dependencies + Commands: commands, + } + if err := db.gormDB.Create(&dbTask).Error; err != nil { + return fmt.Errorf("error storing task: %v", err) + } + } + + return nil + }) +} + +func (db *DB) FetchJob(ctx context.Context, jobID string) (*api.Job, error) { + dbJob := Job{} + findResult := db.gormDB.First(&dbJob, "uuid = ?", jobID) + if findResult.Error != nil { + return nil, findResult.Error + } + + apiJob := api.Job{ + SubmittedJob: api.SubmittedJob{ + Name: dbJob.Name, + Priority: int(dbJob.Priority), + Type: dbJob.JobType, + }, + + Id: dbJob.UUID, + Created: dbJob.CreatedAt, + Updated: dbJob.UpdatedAt, + Status: api.JobStatus(dbJob.Status), + } + + apiJob.Settings = &api.JobSettings{AdditionalProperties: dbJob.Settings} + apiJob.Metadata = &api.JobMetadata{AdditionalProperties: dbJob.Metadata} + + return &apiJob, nil +} diff --git a/internal/manager/persistence/jobs_test.go b/internal/manager/persistence/jobs_test.go new file mode 100644 index 00000000..2f6114b9 --- /dev/null +++ b/internal/manager/persistence/jobs_test.go @@ -0,0 +1,115 @@ +// Package persistence provides the database interface for Flamenco Manager. +package persistence + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers" + "golang.org/x/net/context" +) + +func TestStoreAuthoredJob(t *testing.T) { + db := createTestDB(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + task1 := job_compilers.AuthoredTask{ + Name: "render-1-3", + Type: "blender", + Commands: []job_compilers.AuthoredCommand{ + { + Type: "blender-render", + Parameters: job_compilers.AuthoredCommandParameters{ + "exe": "{blender}", + "blendfile": "/path/to/file.blend", + "args": []interface{}{ + "--render-output", "/path/to/output/######.png", + "--render-format", "PNG", + "--render-frame", "1-3", + }, + }}, + }, + } + + task2 := task1 + task2.Name = "render-4-6" + task2.Commands[0].Parameters["frames"] = "4-6" + + task3 := job_compilers.AuthoredTask{ + Name: "preview-video", + Type: "ffmpeg", + Commands: []job_compilers.AuthoredCommand{ + { + Type: "merge-frames-to-video", + Parameters: job_compilers.AuthoredCommandParameters{ + "images": "/path/to/output/######.png", + "output": "/path/to/output/preview.mkv", + "ffmpegParams": "-c:v hevc -crf 31", + }}, + }, + Dependencies: []*job_compilers.AuthoredTask{&task1, &task2}, + } + + job := job_compilers.AuthoredJob{ + JobID: "263fd47e-b9f8-4637-b726-fd7e47ecfdae", + Name: "Test job", + Priority: 50, + Settings: job_compilers.JobSettings{ + "frames": "1-6", + "chunk_size": 3.0, // The roundtrip to JSON in PostgreSQL can make this a float. + }, + Metadata: job_compilers.JobMetadata{ + "author": "Sybren", + "project": "Sprite Fright", + }, + Tasks: []job_compilers.AuthoredTask{task1, task2, task3}, + } + + err := db.StoreJob(ctx, job) + assert.NoError(t, err) + + fetchedJob, err := db.FetchJob(ctx, job.JobID) + assert.NoError(t, err) + assert.NotNil(t, fetchedJob) + + // Test contents of fetched job + assert.Equal(t, job.JobID, fetchedJob.Id) + assert.Equal(t, job.Name, fetchedJob.Name) + assert.Equal(t, job.JobType, fetchedJob.Type) + assert.Equal(t, job.Priority, fetchedJob.Priority) + assert.EqualValues(t, map[string]interface{}(job.Settings), fetchedJob.Settings.AdditionalProperties) + assert.EqualValues(t, map[string]string(job.Metadata), fetchedJob.Metadata.AdditionalProperties) + + // Fetch tasks of job. + var dbJob Job + tx := db.gormDB.Where(&Job{UUID: job.JobID}).Find(&dbJob) + assert.NoError(t, tx.Error) + var tasks []Task + tx = db.gormDB.Where("job_id = ?", dbJob.ID).Find(&tasks) + assert.NoError(t, tx.Error) + + assert.Len(t, tasks, 3) +}