Add creation of 'create-video' task + setting dependencies

go.mod

@@ -7,4 +7,5 @@ require (
 	github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7
 	github.com/mattn/go-colorable v0.1.12
 	github.com/rs/zerolog v1.26.1
+	github.com/stretchr/testify v1.7.0
 )

go.sum

@@ -1,5 +1,7 @@
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dop251/goja v0.0.0-20211217115348-3f9136fa235d h1:XT7Qdmcuwgsgz4GXejX7R5Morysk2GOpeguYJ9JoF5c=
@@ -9,18 +11,25 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
 github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -52,6 +61,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

job_compilers/author.go

@@ -1,7 +1,10 @@
 package job_compilers
 
 import (
+	"time"
+
 	"github.com/dop251/goja"
+	"github.com/rs/zerolog/log"
 )
 
 // Author allows scripts to author tasks and commands.
@@ -9,9 +12,29 @@ type Author struct {
 	runtime *goja.Runtime
 }
 
+type AuthoredJob struct {
+	JobID    int64
+	Name     string
+	JobType  string
+	Priority int8
+
+	Created time.Time
+
+	Settings JobSettings
+	Metadata JobMetadata
+
+	Tasks []AuthoredTask
+}
+
+type JobSettings map[string]interface{}
+type JobMetadata map[string]string
+
 type AuthoredTask struct {
 	Name     string
 	Commands []AuthoredCommand
+
+	// Dependencies are tasks that need to be completed before this one can run.
+	Dependencies []*AuthoredTask
 }
 
 type AuthoredCommand struct {
@@ -23,6 +46,7 @@ func (a *Author) Task(name string) (*AuthoredTask, error) {
 	at := AuthoredTask{
 		name,
 		make([]AuthoredCommand, 0),
+		make([]*AuthoredTask, 0),
 	}
 	return &at, nil
 }
@@ -41,6 +65,17 @@ func AuthorModule(r *goja.Runtime, module *goja.Object) {
 	obj.Set("Command", a.Command)
 }
 
+func (aj *AuthoredJob) AddTask(at *AuthoredTask) {
+	log.Debug().Str("job", at.Name).Interface("task", at).Msg("add task")
+	aj.Tasks = append(aj.Tasks, *at)
+}
+
 func (at *AuthoredTask) AddCommand(ac *AuthoredCommand) {
 	at.Commands = append(at.Commands, *ac)
 }
+
+func (at *AuthoredTask) AddDependency(dep *AuthoredTask) error {
+	// TODO: check for dependency cycles, return error if there.
+	at.Dependencies = append(at.Dependencies, dep)
+	return nil
+}

job_compilers/job_compilers.go

@@ -66,21 +66,6 @@ func newGojaVM() *goja.Runtime {
 	return vm
 }
 
-type Job struct {
-	JobID    int64
-	Name     string
-	JobType  string
-	Priority int8
-
-	Created time.Time
-
-	Settings JobSettings
-	Metadata JobMetadata
-}
-
-type JobSettings map[string]interface{}
-type JobMetadata map[string]string
-
 func (c *GojaJobCompiler) Run(jobType string) error {
 	program, ok := c.jobtypes[jobType]
 	if !ok {
@@ -92,7 +77,7 @@ func (c *GojaJobCompiler) Run(jobType string) error {
 		panic("hard-coded timestamp is wrong")
 	}
 
-	job := Job{
+	job := AuthoredJob{
 		JobID:    327,
 		JobType:  "blender-render",
 		Priority: 50,
@@ -106,8 +91,8 @@ func (c *GojaJobCompiler) Run(jobType string) error {
 			"fps":                   24.0,
 			"extract_audio":         false,
 			"images_or_video":       "images",
-			"format":                "OPEN_EXR_MULTILAYER",
+			"format":                "JPG",
-			"output_file_extension": ".exr",
+			"output_file_extension": ".jpg",
 			"filepath":              "{shaman}/65/61672427b63a96392cd72d65/pro/shots/190_credits/190_0030_A/190_0030_A.lighting.flamenco.blend",
 		},
 		Metadata: JobMetadata{
@@ -117,11 +102,15 @@ func (c *GojaJobCompiler) Run(jobType string) error {
 	}
 	c.vm.Set("job", &job)
 
-	_, err = c.vm.RunProgram(program)
+	if _, err := c.vm.RunProgram(program); err != nil {
 		return err
-}
+	}
 
-func (j *Job) NewTask(call goja.ConstructorCall) goja.Value {
+	log.Info().
-	log.Debug().Interface("args", call.Arguments).Msg("job.NewTask")
+		Int("tasks", len(job.Tasks)).
-	return goja.Undefined()
+		Str("name", job.Name).
+		Str("jobtype", job.JobType).
+		Msg("job created")
+
+	return nil
 }

job_compilers/path.go

@@ -1,7 +1,7 @@
 package job_compilers
 
 import (
-	"path"
+	"path/filepath"
 
 	"github.com/dop251/goja"
 )
@@ -9,7 +9,14 @@ import (
 // PathModule provides file path manipulation functions by wrapping Go's `path`.
 func PathModule(r *goja.Runtime, module *goja.Object) {
 	obj := module.Get("exports").(*goja.Object)
-	obj.Set("basename", path.Base)
+	obj.Set("basename", filepath.Base)
-	obj.Set("dirname", path.Dir)
+	obj.Set("dirname", filepath.Dir)
-	obj.Set("join", path.Join)
+	obj.Set("join", filepath.Join)
+	obj.Set("stem", Stem)
+}
+
+func Stem(fpath string) string {
+	base := filepath.Base(fpath)
+	ext := filepath.Ext(base)
+	return base[:len(base)-len(ext)]
 }

job_compilers/path_test.go

@@ -0,0 +1,15 @@
+package job_compilers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStem(t *testing.T) {
+	assert.Equal(t, "", Stem(""))
+	assert.Equal(t, "stem", Stem("stem.txt"))
+	assert.Equal(t, "stem.a", Stem("stem.a.b"))
+	assert.Equal(t, "file", Stem("/path/to/file.txt"))
+	// assert.Equal(t, "file", Stem("C:\\path\\to\\file.txt"))
+}

job_compilers/scripts/simple_blender_render.js

@@ -1,8 +1,24 @@
-print('Blender Render job submitted');
+print("Blender Render job submitted");
-print('job: ', job)
+print("job: ", job);
 
 const { created, settings } = job;
 
+// Set of scene.render.image_settings.file_format values that produce
+// files which FFmpeg is known not to handle as input.
+const ffmpegIncompatibleImageFormats = new Set([
+    "EXR",
+    "MULTILAYER", // Old CLI-style format indicators
+    "OPEN_EXR",
+    "OPEN_EXR_MULTILAYER", // DNA values for these formats.
+]);
+
+// The render path contains a filename pattern, most likely '######' or
+// something similar. This has to be removed, so that we end up with
+// the directory that will contain the frames.
+const renderOutput = path.dirname(settings.render_output);
+const finalDir = path.dirname(renderOutput);
+const renderDir = intermediatePath(finalDir);
+
 // Determine the intermediate render output path.
 function intermediatePath(render_path) {
     const basename = path.basename(render_path);
@@ -11,22 +27,17 @@ function intermediatePath(render_path) {
 }
 
 function frameChunker(frames, callback) {
-    callback('1-10');
+    // TODO: actually implement.
-    callback('11-20');
+    callback("1-10");
-    callback('21-30');
+    callback("11-20");
+    callback("21-30");
 }
 
-// The render path contains a filename pattern, most likely '######' or
+function authorRenderTasks() {
-// something similar. This has to be removed, so that we end up with
+    let renderTasks = [];
-// the directory that will contain the frames.
+    frameChunker(settings.frames, function(chunk) {
-const renderOutput = path.dirname(settings.render_output);
-const finalDir = path.dirname(renderOutput);
-const renderDir = intermediatePath(finalDir);
-
-let renderTasks = [];
-frameChunker(settings.frames, function(chunk) {
         const task = author.Task(`render-${chunk}`);
-    const command = author.Command('blender-render', {
+        const command = author.Command("blender-render", {
             cmd: settings.blender_cmd,
             filepath: settings.filepath,
             format: settings.format,
@@ -35,9 +46,43 @@ frameChunker(settings.frames, function(chunk) {
         });
         task.addCommand(command);
         renderTasks.push(task);
-});
+    });
-
+    return renderTasks;
-print(`done creating ${renderTasks.length} tasks`);
+}
-for (const task of renderTasks) {
+
-    print(task);
+function authorCreateVideoTask() {
+    if (ffmpegIncompatibleImageFormats.has(settings.format)) {
+        return;
+    }
+    if (!settings.fps || !settings.output_file_extension) {
+        return;
+    }
+
+    const stem = path.stem(settings.filepath).replace('.flamenco', '');
+    const outfile = path.join(renderDir, `${stem}-${settings.frames}.mp4`);
+
+    const task = author.Task('create-video');
+    const command = author.Command("create-video", {
+        input_files: path.join(renderDir, `*${settings.output_file_extension}`),
+        output_file: outfile,
+        fps: settings.fps,
+    });
+    task.addCommand(command);
+
+    print(`Creating output video for ${settings.format}`);
+    return task;
+}
+
+const renderTasks = authorRenderTasks();
+const videoTask = authorCreateVideoTask(renderTasks);
+
+if (videoTask) {
+    // If there is a video task, all other tasks have to be done first.
+    for (const rt of renderTasks) {
+        videoTask.addDependency(rt);
+    }
+    job.addTask(videoTask);
+}
+for (const rt of renderTasks) {
+    job.addTask(rt);
 }