Manager: load job compiler scripts from disk as well

If there is a `scripts` directory next to the current executable, load
scripts from that directory as well.

It is still required to restart the Manager in order to pick up changes
to those scripts (including new/removed files), PLUS a refresh in the
add-on.

FEATURES.md

@@ -61,7 +61,7 @@ Note that list is **not** in any specific order.
 
 - [x] Let Manager write to task log when it's assigned to a worker.
 - [ ] Worker sleep schedule
-- [ ] Loading of job compiler scripts from disk
+- [x] Loading of job compiler scripts from disk
 - [ ] CLI option to write built-in job compiler scripts to disk
 - [ ] Per-job last rendered image
 - [ ] Support pausing jobs.

internal/manager/job_compilers/file_loader.go

@@ -0,0 +1,153 @@
+package job_compilers
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+import (
+	"embed"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	// embeddedScriptsFS gives access to the embedded scripts.
+	embeddedScriptsFS fs.FS
+
+	// onDiskScriptsFS gives access to the on-disk scripts, located in a `scripts`
+	// directory next to the `flamenco-manager` executable.
+	onDiskScriptsFS fs.FS = nil
+
+	fileLoaderInitialised = false
+)
+
+const scriptsDirName = "scripts"
+
+// Scripts from the `./scripts` subdirectory are embedded into the executable
+// here. Note that accessing these files still requires explicit use of the
+// `scripts/` subdirectory, which is abstracted away by `embeddedScriptFS`.
+//
+//go:embed scripts
+var _embeddedScriptsFS embed.FS
+
+func initFileLoader() {
+	if fileLoaderInitialised {
+		return
+	}
+
+	initEmbeddedFS()
+	initOnDiskFS()
+
+	fileLoaderInitialised = true
+}
+
+// getAvailableFilesystems returns the filesystems to load scripts from, where
+// earlier ones have priority over later ones.
+func getAvailableFilesystems() []fs.FS {
+	filesystems := []fs.FS{}
+
+	if onDiskScriptsFS != nil {
+		filesystems = append(filesystems, onDiskScriptsFS)
+	}
+
+	filesystems = append(filesystems, embeddedScriptsFS)
+	return filesystems
+}
+
+// loadFileFromAnyFS iterates over the available filesystems to find the
+// identified file, and returns its contents when found.
+//
+// Returns `os.ErrNotExist` if there is no filesystem that has this file.
+func loadFileFromAnyFS(path string) ([]byte, error) {
+	filesystems := getAvailableFilesystems()
+
+	for _, fs := range filesystems {
+		file, err := fs.Open(path)
+		if os.IsNotExist(err) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, fs, err)
+		}
+		return io.ReadAll(file)
+	}
+
+	return nil, os.ErrNotExist
+}
+
+func initEmbeddedFS() {
+	// Find embedded filesystem. Unless there were issues with the build of
+	// Flamenco Manager, this should always be here.
+	var err error
+	embeddedScriptsFS, err = fs.Sub(_embeddedScriptsFS, "scripts")
+	if err != nil {
+		panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err))
+	}
+}
+
+func initOnDiskFS() {
+	exename, err := os.Executable()
+	if err != nil {
+		log.Error().Err(err).Msg("job compiler: unable to determine the path of the currently running executable")
+		return
+	}
+	logger := log.With().Str("executable", exename).Logger()
+	logger.Debug().Msg("job compiler: searching for scripts directory next to executable")
+
+	// Try to find the scripts next to the executable.
+	scriptsDir, found := findOnDiskScriptsNextTo(exename)
+	if found {
+		log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable")
+		onDiskScriptsFS = os.DirFS(scriptsDir)
+		return
+	}
+
+	// Evaluate any symlinks and see if that produces a different path to the
+	// executable.
+	evalLinkExe, err := filepath.EvalSymlinks(exename)
+	if err != nil {
+		logger.Error().Err(err).Msg("job compiler: unable to evaluate any symlinks to the running executable")
+		return
+	}
+	if evalLinkExe == exename {
+		// Evaluating any symlinks didn't produce a different path; no need to do the same search twice.
+		return
+	}
+
+	scriptsDir, found = findOnDiskScriptsNextTo(evalLinkExe)
+	if !found {
+		logger.Debug().Msg("job compiler: did not find scripts directory next to executable")
+		return
+	}
+
+	log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable")
+	onDiskScriptsFS = os.DirFS(scriptsDir)
+}
+
+// Find the `scripts` directory sitting next to the currently-running executable.
+// Return the directory path, and a 'found' boolean indicating whether that path
+// is actually a directory.
+func findOnDiskScriptsNextTo(exename string) (string, bool) {
+	scriptsDir := filepath.Join(filepath.Dir(exename), scriptsDirName)
+
+	logger := log.With().Str("scriptsDir", scriptsDir).Logger()
+	logger.Trace().Msg("job compiler: finding on-disk scripts")
+
+	stat, err := os.Stat(scriptsDir)
+	if os.IsNotExist(err) {
+		return scriptsDir, false
+	}
+	if err != nil {
+		logger.Warn().Err(err).Msg("job compiler: error accessing scripts directory")
+		return scriptsDir, false
+	}
+	if !stat.IsDir() {
+		logger.Debug().Msg("job compiler: ignoring 'scripts' next to executable; it is not a directory")
+		return scriptsDir, false
+	}
+
+	return scriptsDir, true
+}

internal/manager/job_compilers/job_compilers.go

@@ -7,6 +7,7 @@ package job_compilers
 import (
 	"context"
 	"errors"
+	"os"
 	"sort"
 	"sync"
 	"time"
@@ -53,6 +54,8 @@ type TimeService interface {
 
 // Load returns a job compiler service with all JS files loaded.
 func Load(ts TimeService) (*Service, error) {
+	initFileLoader()
+
 	service := Service{
 		compilers:   map[string]Compiler{},
 		timeService: ts,
@@ -64,16 +67,14 @@ func Load(ts TimeService) (*Service, error) {
 	}
 
 	staticFileLoader := func(path string) ([]byte, error) {
-		// TODO: this should try different filesystems, once we allow loading from
+		content, err := loadFileFromAnyFS(path)
-		// disk as well.
+		if err == os.ErrNotExist {
-		content, err := loadScriptBytes(getEmbeddedScriptFS(), path)
-		if err != nil {
 			// The 'require' module uses this to try different variations of the path
 			// in order to find it (without .js, with .js, etc.), so don't log any of
 			// such errors.
 			return nil, require.ModuleFileDoesNotExistError
 		}
-		return content, nil
+		return content, err
 	}
 
 	service.registry = require.NewRegistry(require.WithLoader(staticFileLoader))

internal/manager/job_compilers/scripts.go

@@ -17,9 +17,33 @@ import (
 // loadScripts iterates over all JavaScript files, compiles them, and stores the
 // result into `s.compilers`.
 func (s *Service) loadScripts() error {
-	compilers, err := loadScriptsFrom(getEmbeddedScriptFS())
+	compilers := map[string]Compiler{}
-	if err != nil {
+
-		return err
+	// Collect all job compilers.
+	for _, fs := range getAvailableFilesystems() {
+		compilersfromFS, err := loadScriptsFrom(fs)
+		if err != nil {
+			log.Error().Err(err).Interface("fs", fs).Msg("job compiler: error loading scripts")
+			continue
+		}
+		if len(compilersfromFS) == 0 {
+			continue
+		}
+
+		log.Debug().Interface("fs", fs).
+			Int("numScripts", len(compilersfromFS)).
+			Msg("job compiler: found job compiler scripts")
+
+		// Merge the returned compilers into the big map, skipping ones that were
+		// already there.
+		for name := range compilersfromFS {
+			_, found := compilers[name]
+			if found {
+				continue
+			}
+
+			compilers[name] = compilersfromFS[name]
+		}
 	}
 
 	// Assign the new set of compilers in a thread-safe way.
@@ -30,8 +54,8 @@ func (s *Service) loadScripts() error {
 	return nil
 }
 
-// loadScriptsFrom iterates over all given directory entries, compiles the
+// loadScriptsFrom iterates over files in the root of the given filesystem,
-// files, and stores the result into `s.compilers`.
+// compiles the files, and returns the "name -> compiler" mapping.
 func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) {
 	dirEntries, err := fs.ReadDir(filesystem, ".")
 	if err != nil {
@@ -41,12 +65,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) {
 	compilers := map[string]Compiler{}
 
 	for _, dirEntry := range dirEntries {
+		if !dirEntry.Type().IsRegular() {
+			continue
+		}
+
 		filename := dirEntry.Name()
 		if !strings.HasSuffix(filename, ".js") {
 			continue
 		}
 
-		script_bytes, err := loadScriptBytes(filesystem, filename)
+		script_bytes, err := loadFileFromFS(filesystem, filename)
 		if err != nil {
 			log.Error().Err(err).Str("filename", filename).Msg("failed to read script")
 			continue
@@ -76,16 +104,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) {
 		log.Debug().
 			Str("script", filename).
 			Str("jobType", jobTypeName).
-			Msg("loaded script")
+			Msg("job compiler: loaded script")
 	}
 
 	return compilers, nil
 }
 
-func loadScriptBytes(filesystem fs.FS, path string) ([]byte, error) {
+func loadFileFromFS(filesystem fs.FS, path string) ([]byte, error) {
 	file, err := filesystem.Open(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open embedded script: %w", err)
+		return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, filesystem, err)
 	}
 	return io.ReadAll(file)
 }

internal/manager/job_compilers/scripts_embedded.go

@@ -1,28 +0,0 @@
-package job_compilers
-
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-import (
-	"embed"
-	"fmt"
-	"io/fs"
-)
-
-// Scripts from the `./scripts` subdirectory are embedded into the executable
-// here. Note that accessing these files still requires explicit use of the
-// `scripts/` subdirectory, which is abstracted away by `getEmbeddedScriptFS()`.
-// It is recommended to use that function to get the embedded scripts
-// filesystem.
-
-//go:embed scripts
-var _embeddedScriptsFS embed.FS
-
-// getEmbeddedScriptFS returns the `fs.FS` interface that allows access to the
-// embedded job compiler scripts.
-func getEmbeddedScriptFS() fs.FS {
-	scriptsSubFS, err := fs.Sub(_embeddedScriptsFS, "scripts")
-	if err != nil {
-		panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err))
-	}
-	return scriptsSubFS
-}

internal/manager/job_compilers/scripts_test.go

@@ -31,7 +31,8 @@ func TestLoadScriptsFrom_on_disk_js(t *testing.T) {
 }
 
 func TestLoadScriptsFrom_embedded(t *testing.T) {
-	compilers, err := loadScriptsFrom(getEmbeddedScriptFS())
+	initEmbeddedFS()
+	compilers, err := loadScriptsFrom(embeddedScriptsFS)
 
 	assert.NoError(t, err)
 	expectKeys := map[string]bool{
@@ -43,10 +44,10 @@ func TestLoadScriptsFrom_embedded(t *testing.T) {
 
 func BenchmarkLoadScripts_fromEmbedded(b *testing.B) {
 	zerolog.SetGlobalLevel(zerolog.Disabled)
+	initEmbeddedFS()
 
-	embeddedFS := getEmbeddedScriptFS()
 	for i := 0; i < b.N; i++ {
-		compilers, err := loadScriptsFrom(embeddedFS)
+		compilers, err := loadScriptsFrom(embeddedScriptsFS)
 		assert.NoError(b, err)
 		assert.NotEmpty(b, compilers)
 	}