diff --git a/FEATURES.md b/FEATURES.md index 4ee8f463..f6b581d9 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -77,6 +77,7 @@ Note that list is **not** in any specific order. - [ ] User/Job Submission API authentication - [ ] Auto-removal of old Workers - [ ] Ensure "task state machine" can run in a single database transaction. +- [ ] Refactor `internal/manager/task_logs` so that it uses `internal/manager/local_storage`. ## Worker diff --git a/internal/manager/local_storage/local_storage.go b/internal/manager/local_storage/local_storage.go new file mode 100644 index 00000000..5fd0389f --- /dev/null +++ b/internal/manager/local_storage/local_storage.go @@ -0,0 +1,88 @@ +package local_storage + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "fmt" + "os" + "path/filepath" + + "git.blender.org/flamenco/pkg/crosspath" + "github.com/rs/zerolog/log" +) + +type StorageInfo struct { + rootPath string +} + +// NewNextToExe returns a storage representation that sits next to the +// currently-running executable. If that directory cannot be determined, falls +// back to the current working directory. +func NewNextToExe(subdir string) StorageInfo { + exeDir := getSuitableStorageRoot() + storagePath := filepath.Join(exeDir, subdir) + + return StorageInfo{ + rootPath: storagePath, + } +} + +// ForJob returns the directory path for storing job-related files. +func (si StorageInfo) ForJob(jobUUID string) string { + return filepath.Join(si.rootPath, pathForJob(jobUUID)) +} + +// Erase removes the entire storage directory from disk. +func (si StorageInfo) Erase() error { + // A few safety measures before erasing the planet. + if si.rootPath == "" { + return fmt.Errorf("%+v.Erase(): refusing to erase empty directory", si) + } + if crosspath.IsRoot(si.rootPath) { + return fmt.Errorf("%+v.Erase(): refusing to erase root directory", si) + } + if home, found := os.LookupEnv("HOME"); found && home == si.rootPath { + return fmt.Errorf("%+v.Erase(): refusing to erase home directory %s", si, home) + } + + log.Debug().Str("path", si.rootPath).Msg("erasing storage directory") + return os.RemoveAll(si.rootPath) +} + +// MustErase removes the entire storage directory from disk, and panics if it +// cannot do that. This is primarily aimed at cleaning up at the end of unit +// tests. +func (si StorageInfo) MustErase() { + err := si.Erase() + if err != nil { + panic(err) + } +} + +// Returns a sub-directory suitable for files of this job. +// Note that this is intentionally in sync with the `filepath()` function in +// `internal/manager/task_logs/task_logs.go`. +func pathForJob(jobUUID string) string { + if jobUUID == "" { + return "jobless" + } + return filepath.Join("job-"+jobUUID[:4], jobUUID) +} + +func getSuitableStorageRoot() string { + exename, err := os.Executable() + if err == nil { + return filepath.Dir(exename) + } + log.Error().Err(err).Msg("unable to determine the path of the currently running executable") + + // Fall back to current working directory. + cwd, err := os.Getwd() + if err == nil { + return cwd + } + log.Error().Err(err).Msg("unable to determine the current working directory") + + // Fall back to "." if all else fails. + return "." +} diff --git a/internal/manager/local_storage/local_storage_test.go b/internal/manager/local_storage/local_storage_test.go new file mode 100644 index 00000000..3149fb98 --- /dev/null +++ b/internal/manager/local_storage/local_storage_test.go @@ -0,0 +1,58 @@ +package local_storage + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewNextToExe(t *testing.T) { + si := NewNextToExe("nø ASCïÏ") + + // Unit test executables typically are in `/tmp/go-build{random number}`. + assert.Contains(t, si.rootPath, "go-build") + assert.Equal(t, filepath.Base(si.rootPath), "nø ASCïÏ", + "the real path should end in the given directory name") +} + +func TestNewNextToExe_noSubdir(t *testing.T) { + exePath, err := os.Executable() + if !assert.NoError(t, err) { + t.FailNow() + } + exeName := filepath.Base(exePath) + + // The filesystem in an empty "subdirectory" next to the executable should + // contain the executable. + si := NewNextToExe("") + _, err = os.Stat(filepath.Join(si.rootPath, exeName)) + assert.NoErrorf(t, err, "should be able to stat this executable %s", exeName) +} + +func TestForJob(t *testing.T) { + si := NewNextToExe("task-logs") + jobPath := si.ForJob("08e126ef-d773-468b-8bab-19a8213cf2ff") + + expectedSuffix := filepath.Join("task-logs", "job-08e1", "08e126ef-d773-468b-8bab-19a8213cf2ff") + hasSuffix := strings.HasSuffix(jobPath, expectedSuffix) + assert.Truef(t, hasSuffix, "expected %s to have suffix %s", jobPath, expectedSuffix) +} + +func TestErase(t *testing.T) { + si := NewNextToExe("task-logs") + assert.NoDirExists(t, si.rootPath, "creating a StorageInfo should not create the directory") + + jobPath := si.ForJob("08e126ef-d773-468b-8bab-19a8213cf2ff") + assert.NoDirExists(t, jobPath, "getting a path should not create it") + + assert.NoError(t, os.MkdirAll(jobPath, os.ModePerm)) + assert.DirExists(t, jobPath, "os.MkdirAll is borked") + + assert.NoError(t, si.Erase()) + assert.NoDirExists(t, si.rootPath, "Erase() should erase the root path, and everything in it") +} diff --git a/internal/manager/task_logs/task_logs.go b/internal/manager/task_logs/task_logs.go index de828a49..2422acfd 100644 --- a/internal/manager/task_logs/task_logs.go +++ b/internal/manager/task_logs/task_logs.go @@ -158,6 +158,10 @@ func (s *Storage) RotateFile(logger zerolog.Logger, jobID, taskID string) { } // filepath returns the file path suitable to write a log file. +// Note that this intentionally shares the behaviour of `pathForJob()` in +// `internal/manager/local_storage/local_storage.go`; it is intended that the +// file handling code in this source file is migrated to use the `local_storage` +// package at some point. func (s *Storage) filepath(jobID, taskID string) string { var dirpath string if jobID == "" {