blender/flamenco
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move db/jobs tests to persistence/jobs.go

Browse Source
This commit is contained in:
Sybren A. Stüvel 2022-01-28 14:53:02 +01:00
parent 3b1cb3c187
commit 931fd1a24c
4 changed files with 188 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
internal/manager/persistence/db.go
View File

@ -23,14 +23,10 @@ package persistence
import (
"context"
"fmt"
"github.com/rs/zerolog/log"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
"gitlab.com/blender/flamenco-ng-poc/pkg/api"
)
// TODO : have this configurable from the CLI.
@ -68,72 +64,3 @@ func openDB(ctx context.Context, uri string) (*DB, error) {
}
return &db, nil
}
func (db *DB) StoreJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error {
return db.gormDB.Transaction(func(tx *gorm.DB) error {
// TODO: separate conversion of struct types from storing things in the database.
dbJob := Job{
UUID: authoredJob.JobID,
Name: authoredJob.Name,
JobType: authoredJob.JobType,
Priority: int8(authoredJob.Priority),
Settings: StringInterfaceMap(authoredJob.Settings),
Metadata: StringStringMap(authoredJob.Metadata),
}
if err := db.gormDB.Create(&dbJob).Error; err != nil {
return fmt.Errorf("error storing job: %v", err)
}
for _, authoredTask := range authoredJob.Tasks {
var commands []Command
for _, authoredCommand := range authoredTask.Commands {
commands = append(commands, Command{
Type: authoredCommand.Type,
Parameters: StringInterfaceMap(authoredCommand.Parameters),
})
}
dbTask := Task{
Name: authoredTask.Name,
Type: authoredTask.Type,
Job: &dbJob,
Priority: authoredTask.Priority,
Status: string(api.TaskStatusProcessing), // TODO: is this the right place to set the default status?
// TODO: store dependencies
Commands: commands,
}
if err := db.gormDB.Create(&dbTask).Error; err != nil {
return fmt.Errorf("error storing task: %v", err)
}
}
return nil
})
}
func (db *DB) FetchJob(ctx context.Context, jobID string) (*api.Job, error) {
dbJob := Job{}
findResult := db.gormDB.First(&dbJob, "uuid = ?", jobID)
if findResult.Error != nil {
return nil, findResult.Error
}
apiJob := api.Job{
SubmittedJob: api.SubmittedJob{
Name: dbJob.Name,
Priority: int(dbJob.Priority),
Type: dbJob.JobType,
},
Id: dbJob.UUID,
Created: dbJob.CreatedAt,
Updated: dbJob.UpdatedAt,
Status: api.JobStatus(dbJob.Status),
}
apiJob.Settings = &api.JobSettings{AdditionalProperties: dbJob.Settings}
apiJob.Metadata = &api.JobMetadata{AdditionalProperties: dbJob.Metadata}
return &apiJob, nil
}

85
internal/manager/persistence/db_test.go
View File

@ -26,7 +26,6 @@ import (
"time"
"github.com/stretchr/testify/assert"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
"golang.org/x/net/context"
"gorm.io/gorm"
)
@ -59,87 +58,3 @@ func createTestDB(t *testing.T) *DB {
return db
}
func TestStoreAuthoredJob(t *testing.T) {
db := createTestDB(t)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
task1 := job_compilers.AuthoredTask{
Name: "render-1-3",
Type: "blender",
Commands: []job_compilers.AuthoredCommand{
{
Type: "blender-render",
Parameters: job_compilers.AuthoredCommandParameters{
"exe": "{blender}",
"blendfile": "/path/to/file.blend",
"args": []interface{}{
"--render-output", "/path/to/output/######.png",
"--render-format", "PNG",
"--render-frame", "1-3",
},
}},
},
}
task2 := task1
task2.Name = "render-4-6"
task2.Commands[0].Parameters["frames"] = "4-6"
task3 := job_compilers.AuthoredTask{
Name: "preview-video",
Type: "ffmpeg",
Commands: []job_compilers.AuthoredCommand{
{
Type: "merge-frames-to-video",
Parameters: job_compilers.AuthoredCommandParameters{
"images": "/path/to/output/######.png",
"output": "/path/to/output/preview.mkv",
"ffmpegParams": "-c:v hevc -crf 31",
}},
},
Dependencies: []*job_compilers.AuthoredTask{&task1, &task2},
}
job := job_compilers.AuthoredJob{
JobID: "263fd47e-b9f8-4637-b726-fd7e47ecfdae",
Name: "Test job",
Priority: 50,
Settings: job_compilers.JobSettings{
"frames": "1-6",
"chunk_size": 3.0, // The roundtrip to JSON in PostgreSQL can make this a float.
},
Metadata: job_compilers.JobMetadata{
"author": "Sybren",
"project": "Sprite Fright",
},
Tasks: []job_compilers.AuthoredTask{task1, task2, task3},
}
err := db.StoreJob(ctx, job)
assert.NoError(t, err)
fetchedJob, err := db.FetchJob(ctx, job.JobID)
assert.NoError(t, err)
assert.NotNil(t, fetchedJob)
// Test contents of fetched job
assert.Equal(t, job.JobID, fetchedJob.Id)
assert.Equal(t, job.Name, fetchedJob.Name)
assert.Equal(t, job.JobType, fetchedJob.Type)
assert.Equal(t, job.Priority, fetchedJob.Priority)
assert.EqualValues(t, map[string]interface{}(job.Settings), fetchedJob.Settings.AdditionalProperties)
assert.EqualValues(t, map[string]string(job.Metadata), fetchedJob.Metadata.AdditionalProperties)
// Fetch tasks of job.
var dbJob Job
tx := db.gormDB.Where(&Job{UUID: job.JobID}).Find(&dbJob)
assert.NoError(t, tx.Error)
var tasks []Task
tx = db.gormDB.Where("job_id = ?", dbJob.ID).Find(&tasks)
assert.NoError(t, tx.Error)
assert.Len(t, tasks, 3)
}

73
internal/manager/persistence/models.go → internal/manager/persistence/jobs.go
View File

@ -21,10 +21,14 @@ package persistence
* ***** END GPL LICENSE BLOCK ***** */
import (
"context"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
"gitlab.com/blender/flamenco-ng-poc/pkg/api"
"gorm.io/gorm"
)
@ -101,3 +105,72 @@ func (js *StringStringMap) Scan(value interface{}) error {
}
return json.Unmarshal(b, &js)
}
func (db *DB) StoreJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error {
return db.gormDB.Transaction(func(tx *gorm.DB) error {
// TODO: separate conversion of struct types from storing things in the database.
dbJob := Job{
UUID: authoredJob.JobID,
Name: authoredJob.Name,
JobType: authoredJob.JobType,
Priority: int8(authoredJob.Priority),
Settings: StringInterfaceMap(authoredJob.Settings),
Metadata: StringStringMap(authoredJob.Metadata),
}
if err := db.gormDB.Create(&dbJob).Error; err != nil {
return fmt.Errorf("error storing job: %v", err)
}
for _, authoredTask := range authoredJob.Tasks {
var commands []Command
for _, authoredCommand := range authoredTask.Commands {
commands = append(commands, Command{
Type: authoredCommand.Type,
Parameters: StringInterfaceMap(authoredCommand.Parameters),
})
}
dbTask := Task{
Name: authoredTask.Name,
Type: authoredTask.Type,
Job: &dbJob,
Priority: authoredTask.Priority,
Status: string(api.TaskStatusProcessing), // TODO: is this the right place to set the default status?
// TODO: store dependencies
Commands: commands,
}
if err := db.gormDB.Create(&dbTask).Error; err != nil {
return fmt.Errorf("error storing task: %v", err)
}
}
return nil
})
}
func (db *DB) FetchJob(ctx context.Context, jobID string) (*api.Job, error) {
dbJob := Job{}
findResult := db.gormDB.First(&dbJob, "uuid = ?", jobID)
if findResult.Error != nil {
return nil, findResult.Error
}
apiJob := api.Job{
SubmittedJob: api.SubmittedJob{
Name: dbJob.Name,
Priority: int(dbJob.Priority),
Type: dbJob.JobType,
},
Id: dbJob.UUID,
Created: dbJob.CreatedAt,
Updated: dbJob.UpdatedAt,
Status: api.JobStatus(dbJob.Status),
}
apiJob.Settings = &api.JobSettings{AdditionalProperties: dbJob.Settings}
apiJob.Metadata = &api.JobMetadata{AdditionalProperties: dbJob.Metadata}
return &apiJob, nil
}

115
internal/manager/persistence/jobs_test.go Normal file
View File

@ -0,0 +1,115 @@
// Package persistence provides the database interface for Flamenco Manager.
package persistence
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
*
* ***** END GPL LICENSE BLOCK ***** */
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
"golang.org/x/net/context"
)
func TestStoreAuthoredJob(t *testing.T) {
db := createTestDB(t)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
task1 := job_compilers.AuthoredTask{
Name: "render-1-3",
Type: "blender",
Commands: []job_compilers.AuthoredCommand{
{
Type: "blender-render",
Parameters: job_compilers.AuthoredCommandParameters{
"exe": "{blender}",
"blendfile": "/path/to/file.blend",
"args": []interface{}{
"--render-output", "/path/to/output/######.png",
"--render-format", "PNG",
"--render-frame", "1-3",
},
}},
},
}
task2 := task1
task2.Name = "render-4-6"
task2.Commands[0].Parameters["frames"] = "4-6"
task3 := job_compilers.AuthoredTask{
Name: "preview-video",
Type: "ffmpeg",
Commands: []job_compilers.AuthoredCommand{
{
Type: "merge-frames-to-video",
Parameters: job_compilers.AuthoredCommandParameters{
"images": "/path/to/output/######.png",
"output": "/path/to/output/preview.mkv",
"ffmpegParams": "-c:v hevc -crf 31",
}},
},
Dependencies: []*job_compilers.AuthoredTask{&task1, &task2},
}
job := job_compilers.AuthoredJob{
JobID: "263fd47e-b9f8-4637-b726-fd7e47ecfdae",
Name: "Test job",
Priority: 50,
Settings: job_compilers.JobSettings{
"frames": "1-6",
"chunk_size": 3.0, // The roundtrip to JSON in PostgreSQL can make this a float.
},
Metadata: job_compilers.JobMetadata{
"author": "Sybren",
"project": "Sprite Fright",
},
Tasks: []job_compilers.AuthoredTask{task1, task2, task3},
}
err := db.StoreJob(ctx, job)
assert.NoError(t, err)
fetchedJob, err := db.FetchJob(ctx, job.JobID)
assert.NoError(t, err)
assert.NotNil(t, fetchedJob)
// Test contents of fetched job
assert.Equal(t, job.JobID, fetchedJob.Id)
assert.Equal(t, job.Name, fetchedJob.Name)
assert.Equal(t, job.JobType, fetchedJob.Type)
assert.Equal(t, job.Priority, fetchedJob.Priority)
assert.EqualValues(t, map[string]interface{}(job.Settings), fetchedJob.Settings.AdditionalProperties)
assert.EqualValues(t, map[string]string(job.Metadata), fetchedJob.Metadata.AdditionalProperties)
// Fetch tasks of job.
var dbJob Job
tx := db.gormDB.Where(&Job{UUID: job.JobID}).Find(&dbJob)
assert.NoError(t, tx.Error)
var tasks []Task
tx = db.gormDB.Where("job_id = ?", dbJob.ID).Find(&tasks)
assert.NoError(t, tx.Error)
assert.Len(t, tasks, 3)
}