diff --git a/go.mod b/go.mod index a90bd335..78f2acbe 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,5 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7 github.com/mattn/go-colorable v0.1.12 github.com/rs/zerolog v1.26.1 + github.com/stretchr/testify v1.7.0 ) diff --git a/go.sum b/go.sum index cb9c20d4..e04e52a2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dop251/goja v0.0.0-20211217115348-3f9136fa235d h1:XT7Qdmcuwgsgz4GXejX7R5Morysk2GOpeguYJ9JoF5c= @@ -9,18 +11,25 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -52,6 +61,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/job_compilers/author.go b/job_compilers/author.go index 195fa526..5d727e48 100644 --- a/job_compilers/author.go +++ b/job_compilers/author.go @@ -1,7 +1,10 @@ package job_compilers import ( + "time" + "github.com/dop251/goja" + "github.com/rs/zerolog/log" ) // Author allows scripts to author tasks and commands. @@ -9,9 +12,29 @@ type Author struct { runtime *goja.Runtime } +type AuthoredJob struct { + JobID int64 + Name string + JobType string + Priority int8 + + Created time.Time + + Settings JobSettings + Metadata JobMetadata + + Tasks []AuthoredTask +} + +type JobSettings map[string]interface{} +type JobMetadata map[string]string + type AuthoredTask struct { Name string Commands []AuthoredCommand + + // Dependencies are tasks that need to be completed before this one can run. + Dependencies []*AuthoredTask } type AuthoredCommand struct { @@ -23,6 +46,7 @@ func (a *Author) Task(name string) (*AuthoredTask, error) { at := AuthoredTask{ name, make([]AuthoredCommand, 0), + make([]*AuthoredTask, 0), } return &at, nil } @@ -41,6 +65,17 @@ func AuthorModule(r *goja.Runtime, module *goja.Object) { obj.Set("Command", a.Command) } +func (aj *AuthoredJob) AddTask(at *AuthoredTask) { + log.Debug().Str("job", at.Name).Interface("task", at).Msg("add task") + aj.Tasks = append(aj.Tasks, *at) +} + func (at *AuthoredTask) AddCommand(ac *AuthoredCommand) { at.Commands = append(at.Commands, *ac) } + +func (at *AuthoredTask) AddDependency(dep *AuthoredTask) error { + // TODO: check for dependency cycles, return error if there. + at.Dependencies = append(at.Dependencies, dep) + return nil +} diff --git a/job_compilers/job_compilers.go b/job_compilers/job_compilers.go index d464a9fb..bee275cc 100644 --- a/job_compilers/job_compilers.go +++ b/job_compilers/job_compilers.go @@ -66,21 +66,6 @@ func newGojaVM() *goja.Runtime { return vm } -type Job struct { - JobID int64 - Name string - JobType string - Priority int8 - - Created time.Time - - Settings JobSettings - Metadata JobMetadata -} - -type JobSettings map[string]interface{} -type JobMetadata map[string]string - func (c *GojaJobCompiler) Run(jobType string) error { program, ok := c.jobtypes[jobType] if !ok { @@ -92,7 +77,7 @@ func (c *GojaJobCompiler) Run(jobType string) error { panic("hard-coded timestamp is wrong") } - job := Job{ + job := AuthoredJob{ JobID: 327, JobType: "blender-render", Priority: 50, @@ -106,8 +91,8 @@ func (c *GojaJobCompiler) Run(jobType string) error { "fps": 24.0, "extract_audio": false, "images_or_video": "images", - "format": "OPEN_EXR_MULTILAYER", - "output_file_extension": ".exr", + "format": "JPG", + "output_file_extension": ".jpg", "filepath": "{shaman}/65/61672427b63a96392cd72d65/pro/shots/190_credits/190_0030_A/190_0030_A.lighting.flamenco.blend", }, Metadata: JobMetadata{ @@ -117,11 +102,15 @@ func (c *GojaJobCompiler) Run(jobType string) error { } c.vm.Set("job", &job) - _, err = c.vm.RunProgram(program) - return err -} + if _, err := c.vm.RunProgram(program); err != nil { + return err + } -func (j *Job) NewTask(call goja.ConstructorCall) goja.Value { - log.Debug().Interface("args", call.Arguments).Msg("job.NewTask") - return goja.Undefined() + log.Info(). + Int("tasks", len(job.Tasks)). + Str("name", job.Name). + Str("jobtype", job.JobType). + Msg("job created") + + return nil } diff --git a/job_compilers/path.go b/job_compilers/path.go index fe7523b3..3f7577ad 100644 --- a/job_compilers/path.go +++ b/job_compilers/path.go @@ -1,7 +1,7 @@ package job_compilers import ( - "path" + "path/filepath" "github.com/dop251/goja" ) @@ -9,7 +9,14 @@ import ( // PathModule provides file path manipulation functions by wrapping Go's `path`. func PathModule(r *goja.Runtime, module *goja.Object) { obj := module.Get("exports").(*goja.Object) - obj.Set("basename", path.Base) - obj.Set("dirname", path.Dir) - obj.Set("join", path.Join) + obj.Set("basename", filepath.Base) + obj.Set("dirname", filepath.Dir) + obj.Set("join", filepath.Join) + obj.Set("stem", Stem) +} + +func Stem(fpath string) string { + base := filepath.Base(fpath) + ext := filepath.Ext(base) + return base[:len(base)-len(ext)] } diff --git a/job_compilers/path_test.go b/job_compilers/path_test.go new file mode 100644 index 00000000..e9ad706e --- /dev/null +++ b/job_compilers/path_test.go @@ -0,0 +1,15 @@ +package job_compilers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStem(t *testing.T) { + assert.Equal(t, "", Stem("")) + assert.Equal(t, "stem", Stem("stem.txt")) + assert.Equal(t, "stem.a", Stem("stem.a.b")) + assert.Equal(t, "file", Stem("/path/to/file.txt")) + // assert.Equal(t, "file", Stem("C:\\path\\to\\file.txt")) +} diff --git a/job_compilers/scripts/simple_blender_render.js b/job_compilers/scripts/simple_blender_render.js index f54b2395..5a814885 100644 --- a/job_compilers/scripts/simple_blender_render.js +++ b/job_compilers/scripts/simple_blender_render.js @@ -1,8 +1,24 @@ -print('Blender Render job submitted'); -print('job: ', job) +print("Blender Render job submitted"); +print("job: ", job); const { created, settings } = job; +// Set of scene.render.image_settings.file_format values that produce +// files which FFmpeg is known not to handle as input. +const ffmpegIncompatibleImageFormats = new Set([ + "EXR", + "MULTILAYER", // Old CLI-style format indicators + "OPEN_EXR", + "OPEN_EXR_MULTILAYER", // DNA values for these formats. +]); + +// The render path contains a filename pattern, most likely '######' or +// something similar. This has to be removed, so that we end up with +// the directory that will contain the frames. +const renderOutput = path.dirname(settings.render_output); +const finalDir = path.dirname(renderOutput); +const renderDir = intermediatePath(finalDir); + // Determine the intermediate render output path. function intermediatePath(render_path) { const basename = path.basename(render_path); @@ -11,33 +27,62 @@ function intermediatePath(render_path) { } function frameChunker(frames, callback) { - callback('1-10'); - callback('11-20'); - callback('21-30'); + // TODO: actually implement. + callback("1-10"); + callback("11-20"); + callback("21-30"); } -// The render path contains a filename pattern, most likely '######' or -// something similar. This has to be removed, so that we end up with -// the directory that will contain the frames. -const renderOutput = path.dirname(settings.render_output); -const finalDir = path.dirname(renderOutput); -const renderDir = intermediatePath(finalDir); +function authorRenderTasks() { + let renderTasks = []; + frameChunker(settings.frames, function(chunk) { + const task = author.Task(`render-${chunk}`); + const command = author.Command("blender-render", { + cmd: settings.blender_cmd, + filepath: settings.filepath, + format: settings.format, + render_output: path.join(renderDir, path.basename(renderOutput)), + frames: chunk, + }); + task.addCommand(command); + renderTasks.push(task); + }); + return renderTasks; +} -let renderTasks = []; -frameChunker(settings.frames, function(chunk) { - const task = author.Task(`render-${chunk}`); - const command = author.Command('blender-render', { - cmd: settings.blender_cmd, - filepath: settings.filepath, - format: settings.format, - render_output: path.join(renderDir, path.basename(renderOutput)), - frames: chunk, +function authorCreateVideoTask() { + if (ffmpegIncompatibleImageFormats.has(settings.format)) { + return; + } + if (!settings.fps || !settings.output_file_extension) { + return; + } + + const stem = path.stem(settings.filepath).replace('.flamenco', ''); + const outfile = path.join(renderDir, `${stem}-${settings.frames}.mp4`); + + const task = author.Task('create-video'); + const command = author.Command("create-video", { + input_files: path.join(renderDir, `*${settings.output_file_extension}`), + output_file: outfile, + fps: settings.fps, }); task.addCommand(command); - renderTasks.push(task); -}); -print(`done creating ${renderTasks.length} tasks`); -for (const task of renderTasks) { - print(task); + print(`Creating output video for ${settings.format}`); + return task; +} + +const renderTasks = authorRenderTasks(); +const videoTask = authorCreateVideoTask(renderTasks); + +if (videoTask) { + // If there is a video task, all other tasks have to be done first. + for (const rt of renderTasks) { + videoTask.addDependency(rt); + } + job.addTask(videoTask); +} +for (const rt of renderTasks) { + job.addTask(rt); } \ No newline at end of file