diff --git a/FEATURES.md b/FEATURES.md index 413fdac3..4ee8f463 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -61,7 +61,7 @@ Note that list is **not** in any specific order. - [x] Let Manager write to task log when it's assigned to a worker. - [ ] Worker sleep schedule -- [ ] Loading of job compiler scripts from disk +- [x] Loading of job compiler scripts from disk - [ ] CLI option to write built-in job compiler scripts to disk - [ ] Per-job last rendered image - [ ] Support pausing jobs. diff --git a/internal/manager/job_compilers/file_loader.go b/internal/manager/job_compilers/file_loader.go new file mode 100644 index 00000000..0fd6ad60 --- /dev/null +++ b/internal/manager/job_compilers/file_loader.go @@ -0,0 +1,153 @@ +package job_compilers + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "embed" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +var ( + // embeddedScriptsFS gives access to the embedded scripts. + embeddedScriptsFS fs.FS + + // onDiskScriptsFS gives access to the on-disk scripts, located in a `scripts` + // directory next to the `flamenco-manager` executable. + onDiskScriptsFS fs.FS = nil + + fileLoaderInitialised = false +) + +const scriptsDirName = "scripts" + +// Scripts from the `./scripts` subdirectory are embedded into the executable +// here. Note that accessing these files still requires explicit use of the +// `scripts/` subdirectory, which is abstracted away by `embeddedScriptFS`. +// +//go:embed scripts +var _embeddedScriptsFS embed.FS + +func initFileLoader() { + if fileLoaderInitialised { + return + } + + initEmbeddedFS() + initOnDiskFS() + + fileLoaderInitialised = true +} + +// getAvailableFilesystems returns the filesystems to load scripts from, where +// earlier ones have priority over later ones. +func getAvailableFilesystems() []fs.FS { + filesystems := []fs.FS{} + + if onDiskScriptsFS != nil { + filesystems = append(filesystems, onDiskScriptsFS) + } + + filesystems = append(filesystems, embeddedScriptsFS) + return filesystems +} + +// loadFileFromAnyFS iterates over the available filesystems to find the +// identified file, and returns its contents when found. +// +// Returns `os.ErrNotExist` if there is no filesystem that has this file. +func loadFileFromAnyFS(path string) ([]byte, error) { + filesystems := getAvailableFilesystems() + + for _, fs := range filesystems { + file, err := fs.Open(path) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, fs, err) + } + return io.ReadAll(file) + } + + return nil, os.ErrNotExist +} + +func initEmbeddedFS() { + // Find embedded filesystem. Unless there were issues with the build of + // Flamenco Manager, this should always be here. + var err error + embeddedScriptsFS, err = fs.Sub(_embeddedScriptsFS, "scripts") + if err != nil { + panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err)) + } +} + +func initOnDiskFS() { + exename, err := os.Executable() + if err != nil { + log.Error().Err(err).Msg("job compiler: unable to determine the path of the currently running executable") + return + } + logger := log.With().Str("executable", exename).Logger() + logger.Debug().Msg("job compiler: searching for scripts directory next to executable") + + // Try to find the scripts next to the executable. + scriptsDir, found := findOnDiskScriptsNextTo(exename) + if found { + log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable") + onDiskScriptsFS = os.DirFS(scriptsDir) + return + } + + // Evaluate any symlinks and see if that produces a different path to the + // executable. + evalLinkExe, err := filepath.EvalSymlinks(exename) + if err != nil { + logger.Error().Err(err).Msg("job compiler: unable to evaluate any symlinks to the running executable") + return + } + if evalLinkExe == exename { + // Evaluating any symlinks didn't produce a different path; no need to do the same search twice. + return + } + + scriptsDir, found = findOnDiskScriptsNextTo(evalLinkExe) + if !found { + logger.Debug().Msg("job compiler: did not find scripts directory next to executable") + return + } + + log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable") + onDiskScriptsFS = os.DirFS(scriptsDir) +} + +// Find the `scripts` directory sitting next to the currently-running executable. +// Return the directory path, and a 'found' boolean indicating whether that path +// is actually a directory. +func findOnDiskScriptsNextTo(exename string) (string, bool) { + scriptsDir := filepath.Join(filepath.Dir(exename), scriptsDirName) + + logger := log.With().Str("scriptsDir", scriptsDir).Logger() + logger.Trace().Msg("job compiler: finding on-disk scripts") + + stat, err := os.Stat(scriptsDir) + if os.IsNotExist(err) { + return scriptsDir, false + } + if err != nil { + logger.Warn().Err(err).Msg("job compiler: error accessing scripts directory") + return scriptsDir, false + } + if !stat.IsDir() { + logger.Debug().Msg("job compiler: ignoring 'scripts' next to executable; it is not a directory") + return scriptsDir, false + } + + return scriptsDir, true +} diff --git a/internal/manager/job_compilers/job_compilers.go b/internal/manager/job_compilers/job_compilers.go index 510f51b3..5ba76b5e 100644 --- a/internal/manager/job_compilers/job_compilers.go +++ b/internal/manager/job_compilers/job_compilers.go @@ -7,6 +7,7 @@ package job_compilers import ( "context" "errors" + "os" "sort" "sync" "time" @@ -53,6 +54,8 @@ type TimeService interface { // Load returns a job compiler service with all JS files loaded. func Load(ts TimeService) (*Service, error) { + initFileLoader() + service := Service{ compilers: map[string]Compiler{}, timeService: ts, @@ -64,16 +67,14 @@ func Load(ts TimeService) (*Service, error) { } staticFileLoader := func(path string) ([]byte, error) { - // TODO: this should try different filesystems, once we allow loading from - // disk as well. - content, err := loadScriptBytes(getEmbeddedScriptFS(), path) - if err != nil { + content, err := loadFileFromAnyFS(path) + if err == os.ErrNotExist { // The 'require' module uses this to try different variations of the path // in order to find it (without .js, with .js, etc.), so don't log any of // such errors. return nil, require.ModuleFileDoesNotExistError } - return content, nil + return content, err } service.registry = require.NewRegistry(require.WithLoader(staticFileLoader)) diff --git a/internal/manager/job_compilers/scripts.go b/internal/manager/job_compilers/scripts.go index eba75129..20982d7a 100644 --- a/internal/manager/job_compilers/scripts.go +++ b/internal/manager/job_compilers/scripts.go @@ -17,9 +17,33 @@ import ( // loadScripts iterates over all JavaScript files, compiles them, and stores the // result into `s.compilers`. func (s *Service) loadScripts() error { - compilers, err := loadScriptsFrom(getEmbeddedScriptFS()) - if err != nil { - return err + compilers := map[string]Compiler{} + + // Collect all job compilers. + for _, fs := range getAvailableFilesystems() { + compilersfromFS, err := loadScriptsFrom(fs) + if err != nil { + log.Error().Err(err).Interface("fs", fs).Msg("job compiler: error loading scripts") + continue + } + if len(compilersfromFS) == 0 { + continue + } + + log.Debug().Interface("fs", fs). + Int("numScripts", len(compilersfromFS)). + Msg("job compiler: found job compiler scripts") + + // Merge the returned compilers into the big map, skipping ones that were + // already there. + for name := range compilersfromFS { + _, found := compilers[name] + if found { + continue + } + + compilers[name] = compilersfromFS[name] + } } // Assign the new set of compilers in a thread-safe way. @@ -30,8 +54,8 @@ func (s *Service) loadScripts() error { return nil } -// loadScriptsFrom iterates over all given directory entries, compiles the -// files, and stores the result into `s.compilers`. +// loadScriptsFrom iterates over files in the root of the given filesystem, +// compiles the files, and returns the "name -> compiler" mapping. func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { dirEntries, err := fs.ReadDir(filesystem, ".") if err != nil { @@ -41,12 +65,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { compilers := map[string]Compiler{} for _, dirEntry := range dirEntries { + if !dirEntry.Type().IsRegular() { + continue + } + filename := dirEntry.Name() if !strings.HasSuffix(filename, ".js") { continue } - script_bytes, err := loadScriptBytes(filesystem, filename) + script_bytes, err := loadFileFromFS(filesystem, filename) if err != nil { log.Error().Err(err).Str("filename", filename).Msg("failed to read script") continue @@ -76,16 +104,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { log.Debug(). Str("script", filename). Str("jobType", jobTypeName). - Msg("loaded script") + Msg("job compiler: loaded script") } return compilers, nil } -func loadScriptBytes(filesystem fs.FS, path string) ([]byte, error) { +func loadFileFromFS(filesystem fs.FS, path string) ([]byte, error) { file, err := filesystem.Open(path) if err != nil { - return nil, fmt.Errorf("failed to open embedded script: %w", err) + return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, filesystem, err) } return io.ReadAll(file) } diff --git a/internal/manager/job_compilers/scripts_embedded.go b/internal/manager/job_compilers/scripts_embedded.go deleted file mode 100644 index d8d1d8cc..00000000 --- a/internal/manager/job_compilers/scripts_embedded.go +++ /dev/null @@ -1,28 +0,0 @@ -package job_compilers - -// SPDX-License-Identifier: GPL-3.0-or-later - -import ( - "embed" - "fmt" - "io/fs" -) - -// Scripts from the `./scripts` subdirectory are embedded into the executable -// here. Note that accessing these files still requires explicit use of the -// `scripts/` subdirectory, which is abstracted away by `getEmbeddedScriptFS()`. -// It is recommended to use that function to get the embedded scripts -// filesystem. - -//go:embed scripts -var _embeddedScriptsFS embed.FS - -// getEmbeddedScriptFS returns the `fs.FS` interface that allows access to the -// embedded job compiler scripts. -func getEmbeddedScriptFS() fs.FS { - scriptsSubFS, err := fs.Sub(_embeddedScriptsFS, "scripts") - if err != nil { - panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err)) - } - return scriptsSubFS -} diff --git a/internal/manager/job_compilers/scripts_test.go b/internal/manager/job_compilers/scripts_test.go index a51c50d2..aee41e76 100644 --- a/internal/manager/job_compilers/scripts_test.go +++ b/internal/manager/job_compilers/scripts_test.go @@ -31,7 +31,8 @@ func TestLoadScriptsFrom_on_disk_js(t *testing.T) { } func TestLoadScriptsFrom_embedded(t *testing.T) { - compilers, err := loadScriptsFrom(getEmbeddedScriptFS()) + initEmbeddedFS() + compilers, err := loadScriptsFrom(embeddedScriptsFS) assert.NoError(t, err) expectKeys := map[string]bool{ @@ -43,10 +44,10 @@ func TestLoadScriptsFrom_embedded(t *testing.T) { func BenchmarkLoadScripts_fromEmbedded(b *testing.B) { zerolog.SetGlobalLevel(zerolog.Disabled) + initEmbeddedFS() - embeddedFS := getEmbeddedScriptFS() for i := 0; i < b.N; i++ { - compilers, err := loadScriptsFrom(embeddedFS) + compilers, err := loadScriptsFrom(embeddedScriptsFS) assert.NoError(b, err) assert.NotEmpty(b, compilers) }