From 289bcf6414bcb35516d51679a02479facb28cf3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 10 Jan 2022 15:24:40 +0100 Subject: [PATCH] Move job compiler JS code into its own function Each job compiler script now must define a `compileJob(job)` function, which will be called by Flamenco when necessary. This makes it possible to run the script without a job, and get other exported symbols from it, such as metadata about which settings its job type needs/exposes. --- .../manager/job_compilers/job_compilers.go | 59 +++++++++++------ internal/manager/job_compilers/scripts.go | 9 ++- .../scripts/simple_blender_render.js | 64 ++++++++++--------- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/internal/manager/job_compilers/job_compilers.go b/internal/manager/job_compilers/job_compilers.go index b94738ab..2aaac930 100644 --- a/internal/manager/job_compilers/job_compilers.go +++ b/internal/manager/job_compilers/job_compilers.go @@ -31,17 +31,21 @@ import ( ) var ErrJobTypeUnknown = errors.New("job type unknown") +var ErrScriptIncomplete = errors.New("job compiler script incomplete") type GojaJobCompiler struct { - vm *goja.Runtime + jobtypes map[string]JobType // Mapping from job type name to jobType struct. + registry *require.Registry // Goja module registry. +} - jobtypes map[string]*goja.Program +type JobType struct { + program *goja.Program // Compiled JavaScript file. + filename string // The filename of that JS file. } func Load() (*GojaJobCompiler, error) { compiler := GojaJobCompiler{ - vm: newGojaVM(), - jobtypes: map[string]*goja.Program{}, + jobtypes: map[string]JobType{}, } if err := compiler.loadScripts(); err != nil { @@ -59,23 +63,19 @@ func Load() (*GojaJobCompiler, error) { return content, nil } - registry := require.NewRegistry(require.WithLoader(staticFileLoader)) - registry.Enable(compiler.vm) - - registry.RegisterNativeModule("author", AuthorModule) - registry.RegisterNativeModule("path", PathModule) - registry.RegisterNativeModule("process", ProcessModule) - compiler.vm.Set("author", require.Require(compiler.vm, "author")) - compiler.vm.Set("path", require.Require(compiler.vm, "path")) - compiler.vm.Set("process", require.Require(compiler.vm, "process")) + compiler.registry = require.NewRegistry(require.WithLoader(staticFileLoader)) + compiler.registry.RegisterNativeModule("author", AuthorModule) + compiler.registry.RegisterNativeModule("path", PathModule) + compiler.registry.RegisterNativeModule("process", ProcessModule) return &compiler, nil } -func newGojaVM() *goja.Runtime { +func (c *GojaJobCompiler) newGojaVM() *goja.Runtime { vm := goja.New() vm.SetFieldNameMapper(goja.UncapFieldNameMapper()) + // Set some global functions for script debugging purposes. vm.Set("print", func(call goja.FunctionCall) goja.Value { log.Info().Interface("args", call.Arguments).Msg("print") return goja.Undefined() @@ -84,11 +84,18 @@ func newGojaVM() *goja.Runtime { log.Warn().Interface("args", call.Arguments).Msg("alert") return goja.Undefined() }) + + // Pre-import some useful modules. + c.registry.Enable(vm) + vm.Set("author", require.Require(vm, "author")) + vm.Set("path", require.Require(vm, "path")) + vm.Set("process", require.Require(vm, "process")) + return vm } -func (c *GojaJobCompiler) Run(jobType string) error { - program, ok := c.jobtypes[jobType] +func (c *GojaJobCompiler) Run(jobTypeName string) error { + jobType, ok := c.jobtypes[jobTypeName] if !ok { return ErrJobTypeUnknown } @@ -121,9 +128,25 @@ func (c *GojaJobCompiler) Run(jobType string) error { "project": "Sprøte Frøte", }, } - c.vm.Set("job", &job) - if _, err := c.vm.RunProgram(program); err != nil { + vm := c.newGojaVM() + + // This should register the `compileJob()` function called below: + if _, err := vm.RunProgram(jobType.program); err != nil { + return err + } + + compileJob, isCallable := goja.AssertFunction(vm.Get("compileJob")) + if !isCallable { + log.Error(). + Str("jobType", jobTypeName). + Str("script", jobType.filename). + Msg("script does not define a compileJob(job) function") + return ErrScriptIncomplete + + } + + if _, err := compileJob(nil, vm.ToValue(&job)); err != nil { return err } diff --git a/internal/manager/job_compilers/scripts.go b/internal/manager/job_compilers/scripts.go index 2297375b..3a880fcd 100644 --- a/internal/manager/job_compilers/scripts.go +++ b/internal/manager/job_compilers/scripts.go @@ -58,10 +58,13 @@ func (c *GojaJobCompiler) loadScripts() error { continue } - jobType := filenameToJobType(script.Name()) - c.jobtypes[jobType] = program + jobTypeName := filenameToJobType(script.Name()) + c.jobtypes[jobTypeName] = JobType{ + program: program, + filename: script.Name(), + } - log.Debug().Str("script", script.Name()).Str("jobType", jobType).Msg("loaded script") + log.Debug().Str("script", script.Name()).Str("jobType", jobTypeName).Msg("loaded script") } return nil diff --git a/internal/manager/job_compilers/scripts/simple_blender_render.js b/internal/manager/job_compilers/scripts/simple_blender_render.js index 12936216..da3cfed8 100644 --- a/internal/manager/job_compilers/scripts/simple_blender_render.js +++ b/internal/manager/job_compilers/scripts/simple_blender_render.js @@ -18,11 +18,6 @@ * * ***** END GPL LICENSE BLOCK ***** */ -print("Blender Render job submitted"); -print("job: ", job); - -const { created, settings } = job; - // Set of scene.render.image_settings.file_format values that produce // files which FFmpeg is known not to handle as input. const ffmpegIncompatibleImageFormats = new Set([ @@ -32,18 +27,39 @@ const ffmpegIncompatibleImageFormats = new Set([ "OPEN_EXR_MULTILAYER", // DNA values for these formats. ]); -// The render path contains a filename pattern, most likely '######' or -// something similar. This has to be removed, so that we end up with -// the directory that will contain the frames. -const renderOutput = path.dirname(settings.render_output); -const finalDir = path.dirname(renderOutput); -const renderDir = intermediatePath(finalDir); +function compileJob(job) { + print("Blender Render job submitted"); + print("job: ", job); + + const settings = job.settings; + + // The render path contains a filename pattern, most likely '######' or + // something similar. This has to be removed, so that we end up with + // the directory that will contain the frames. + const renderOutput = path.dirname(settings.render_output); + const finalDir = path.dirname(renderOutput); + const renderDir = intermediatePath(job, finalDir); + + const renderTasks = authorRenderTasks(settings, renderDir, renderOutput); + const videoTask = authorCreateVideoTask(renderTasks, renderDir); + + if (videoTask) { + // If there is a video task, all other tasks have to be done first. + for (const rt of renderTasks) { + videoTask.addDependency(rt); + } + job.addTask(videoTask); + } + for (const rt of renderTasks) { + job.addTask(rt); + } +} // Determine the intermediate render output path. -function intermediatePath(render_path) { - const basename = path.basename(render_path); - const name = `${basename}__intermediate-${created}`; - return path.join(path.dirname(render_path), name); +function intermediatePath(job, finalDir) { + const basename = path.basename(finalDir); + const name = `${basename}__intermediate-${job.created}`; + return path.join(path.dirname(finalDir), name); } function frameChunker(frames, callback) { @@ -53,7 +69,7 @@ function frameChunker(frames, callback) { callback("21-30"); } -function authorRenderTasks() { +function authorRenderTasks(settings, renderDir, renderOutput) { let renderTasks = []; frameChunker(settings.frames, function(chunk) { const task = author.Task(`render-${chunk}`); @@ -70,7 +86,7 @@ function authorRenderTasks() { return renderTasks; } -function authorCreateVideoTask() { +function authorCreateVideoTask(settings, renderDir) { if (ffmpegIncompatibleImageFormats.has(settings.format)) { return; } @@ -91,18 +107,4 @@ function authorCreateVideoTask() { print(`Creating output video for ${settings.format}`); return task; -} - -const renderTasks = authorRenderTasks(); -const videoTask = authorCreateVideoTask(renderTasks); - -if (videoTask) { - // If there is a video task, all other tasks have to be done first. - for (const rt of renderTasks) { - videoTask.addDependency(rt); - } - job.addTask(videoTask); -} -for (const rt of renderTasks) { - job.addTask(rt); } \ No newline at end of file