// Package job_compilers contains functionality to convert a Flamenco job
// definition into concrete tasks and commands to execute by Workers.
package job_compilers
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see .
*
* ***** END GPL LICENSE BLOCK ***** */
import (
"context"
"errors"
"time"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/require"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"gitlab.com/blender/flamenco-goja-test/pkg/api"
)
var ErrJobTypeUnknown = errors.New("job type unknown")
var ErrScriptIncomplete = errors.New("job compiler script incomplete")
// Service contains job compilers defined in JavaScript.
type Service struct {
compilers map[string]Compiler // Mapping from job type name to the job compiler of that type.
registry *require.Registry // Goja module registry.
}
type Compiler struct {
jobType string
program *goja.Program // Compiled JavaScript file.
filename string // The filename of that JS file.
}
type VM struct {
runtime *goja.Runtime // Goja VM containing the job compiler script.
compiler Compiler // Program loaded into this VM.
}
// jobCompileFunc is a function that fills job.Tasks.
type jobCompileFunc func(job *AuthoredJob) error
func Load() (*Service, error) {
compiler := Service{
compilers: map[string]Compiler{},
}
if err := compiler.loadScripts(); err != nil {
return nil, err
}
staticFileLoader := func(path string) ([]byte, error) {
content, err := compiler.loadScriptBytes(path)
if err != nil {
// The 'require' module uses this to try different variations of the path
// in order to find it (without .js, with .js, etc.), so don't log any of
// such errors.
return nil, require.ModuleFileDoesNotExistError
}
return content, nil
}
compiler.registry = require.NewRegistry(require.WithLoader(staticFileLoader))
compiler.registry.RegisterNativeModule("author", AuthorModule)
compiler.registry.RegisterNativeModule("path", PathModule)
compiler.registry.RegisterNativeModule("process", ProcessModule)
return &compiler, nil
}
func (s *Service) Compile(ctx context.Context, sj api.SubmittedJob) (*AuthoredJob, error) {
vm, err := s.compilerForJobType(sj.Type)
if err != nil {
return nil, err
}
// Create an AuthoredJob from this SubmittedJob.
aj := AuthoredJob{
JobID: uuid.New().String(), // Ignore the submitted ID.
Name: sj.Name,
JobType: sj.Type,
Priority: sj.Priority,
Status: api.JobStatusUnderConstruction,
Settings: make(JobSettings),
Metadata: make(JobMetadata),
}
if sj.Settings != nil {
for key, value := range *sj.Settings {
aj.Settings[key] = value
}
}
if sj.Metadata != nil {
for key, value := range *sj.Metadata {
// TODO: make sure OpenAPI understands these keys can only be strings.
aj.Metadata[key] = value.(string)
}
}
compiler, err := vm.getCompileJob()
if err != nil {
return nil, err
}
if err := compiler(&aj); err != nil {
return nil, err
}
log.Info().
Int("num_tasks", len(aj.Tasks)).
Str("name", aj.Name).
Str("jobtype", aj.JobType).
Msg("job compiled")
return &aj, nil
}
func (s *Service) Run(jobTypeName string) error {
vm, err := s.compilerForJobType(jobTypeName)
if err != nil {
return err
}
compileJob, err := vm.getCompileJob()
if err != nil {
return err
}
created, err := time.Parse(time.RFC3339, "2022-01-03T18:53:00+01:00")
if err != nil {
panic("hard-coded timestamp is wrong")
}
job := AuthoredJob{
JobID: uuid.New().String(),
JobType: "blender-render",
Priority: 50,
Name: "190_0030_A.lighting",
Created: created,
Settings: JobSettings{
"blender_cmd": "{blender}",
"chunk_size": 5,
"frames": "101-172",
"render_output": "{render}/sprites/farm_output/shots/190_credits/190_0030_A/190_0030_A.lighting/######",
"fps": 24.0,
"extract_audio": false,
"images_or_video": "images",
"format": "JPG",
"output_file_extension": ".jpg",
"filepath": "{shaman}/65/61672427b63a96392cd72d65/pro/shots/190_credits/190_0030_A/190_0030_A.lighting.flamenco.blend",
},
Metadata: JobMetadata{
"user": "Sybren A. Stüvel ",
"project": "Sprøte Frøte",
},
}
if err := compileJob(&job); err != nil {
return err
}
log.Info().
Int("tasks", len(job.Tasks)).
Str("name", job.Name).
Str("jobtype", job.JobType).
Msg("job created")
return nil
}
// ListJobTypes returns the list of available job types.
func (s *Service) ListJobTypes() api.AvailableJobTypes {
jobTypes := make([]api.AvailableJobType, 0)
for typeName := range s.compilers {
compiler, err := s.compilerForJobType(typeName)
if err != nil {
log.Warn().Err(err).Str("jobType", typeName).Msg("unable to determine job type settings")
continue
}
description, err := compiler.getJobTypeInfo()
if err != nil {
log.Warn().Err(err).Str("jobType", typeName).Msg("unable to determine job type settings")
continue
}
jobTypes = append(jobTypes, *description)
}
return api.AvailableJobTypes{JobTypes: jobTypes}
}
func (vm *VM) getCompileJob() (jobCompileFunc, error) {
compileJob, isCallable := goja.AssertFunction(vm.runtime.Get("compileJob"))
if !isCallable {
// TODO: construct a more elaborate Error type that contains this info, instead of logging here.
log.Error().
Str("jobType", vm.compiler.jobType).
Str("script", vm.compiler.filename).
Msg("script does not define a compileJob(job) function")
return nil, ErrScriptIncomplete
}
// TODO: wrap this in a nicer way.
return func(job *AuthoredJob) error {
_, err := compileJob(nil, vm.runtime.ToValue(job))
return err
}, nil
}
func (vm *VM) getJobTypeInfo() (*api.AvailableJobType, error) {
jtValue := vm.runtime.Get("JOB_TYPE")
var ajt api.AvailableJobType
if err := vm.runtime.ExportTo(jtValue, &ajt); err != nil {
// TODO: construct a more elaborate Error type that contains this info, instead of logging here.
log.Error().
Err(err).
Str("jobType", vm.compiler.jobType).
Str("script", vm.compiler.filename).
Msg("script does not define a proper JOB_TYPE object")
return nil, ErrScriptIncomplete
}
ajt.Name = vm.compiler.jobType
return &ajt, nil
}