Manager: add local_storage package for managing storage locations

Add a `local_storage` package that finds a suitable place to put files.
Currently it just looks at the location of the currently running
executable; it can later do other things. It can be queried for directory
to put job-specific files.

It is intended to be used by the under-development "last rendered output"
processing system, to store an image file per job. Later we should also
refactor the task log handling system to use this.

FEATURES.md

@@ -77,6 +77,7 @@ Note that list is **not** in any specific order.
 - [ ] User/Job Submission API authentication
 - [ ] Auto-removal of old Workers
 - [ ] Ensure "task state machine" can run in a single database transaction.
+- [ ] Refactor `internal/manager/task_logs` so that it uses `internal/manager/local_storage`.
 
 ## Worker
 

internal/manager/local_storage/local_storage.go

@@ -0,0 +1,88 @@
+package local_storage
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"git.blender.org/flamenco/pkg/crosspath"
+	"github.com/rs/zerolog/log"
+)
+
+type StorageInfo struct {
+	rootPath string
+}
+
+// NewNextToExe returns a storage representation that sits next to the
+// currently-running executable. If that directory cannot be determined, falls
+// back to the current working directory.
+func NewNextToExe(subdir string) StorageInfo {
+	exeDir := getSuitableStorageRoot()
+	storagePath := filepath.Join(exeDir, subdir)
+
+	return StorageInfo{
+		rootPath: storagePath,
+	}
+}
+
+// ForJob returns the directory path for storing job-related files.
+func (si StorageInfo) ForJob(jobUUID string) string {
+	return filepath.Join(si.rootPath, pathForJob(jobUUID))
+}
+
+// Erase removes the entire storage directory from disk.
+func (si StorageInfo) Erase() error {
+	// A few safety measures before erasing the planet.
+	if si.rootPath == "" {
+		return fmt.Errorf("%+v.Erase(): refusing to erase empty directory", si)
+	}
+	if crosspath.IsRoot(si.rootPath) {
+		return fmt.Errorf("%+v.Erase(): refusing to erase root directory", si)
+	}
+	if home, found := os.LookupEnv("HOME"); found && home == si.rootPath {
+		return fmt.Errorf("%+v.Erase(): refusing to erase home directory %s", si, home)
+	}
+
+	log.Debug().Str("path", si.rootPath).Msg("erasing storage directory")
+	return os.RemoveAll(si.rootPath)
+}
+
+// MustErase removes the entire storage directory from disk, and panics if it
+// cannot do that. This is primarily aimed at cleaning up at the end of unit
+// tests.
+func (si StorageInfo) MustErase() {
+	err := si.Erase()
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Returns a sub-directory suitable for files of this job.
+// Note that this is intentionally in sync with the `filepath()` function in
+// `internal/manager/task_logs/task_logs.go`.
+func pathForJob(jobUUID string) string {
+	if jobUUID == "" {
+		return "jobless"
+	}
+	return filepath.Join("job-"+jobUUID[:4], jobUUID)
+}
+
+func getSuitableStorageRoot() string {
+	exename, err := os.Executable()
+	if err == nil {
+		return filepath.Dir(exename)
+	}
+	log.Error().Err(err).Msg("unable to determine the path of the currently running executable")
+
+	// Fall back to current working directory.
+	cwd, err := os.Getwd()
+	if err == nil {
+		return cwd
+	}
+	log.Error().Err(err).Msg("unable to determine the current working directory")
+
+	// Fall back to "." if all else fails.
+	return "."
+}

internal/manager/local_storage/local_storage_test.go

@@ -0,0 +1,58 @@
+package local_storage
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewNextToExe(t *testing.T) {
+	si := NewNextToExe("nø ASCïÏ")
+
+	// Unit test executables typically are in `/tmp/go-build{random number}`.
+	assert.Contains(t, si.rootPath, "go-build")
+	assert.Equal(t, filepath.Base(si.rootPath), "nø ASCïÏ",
+		"the real path should end in the given directory name")
+}
+
+func TestNewNextToExe_noSubdir(t *testing.T) {
+	exePath, err := os.Executable()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	exeName := filepath.Base(exePath)
+
+	// The filesystem in an empty "subdirectory" next to the executable should
+	// contain the executable.
+	si := NewNextToExe("")
+	_, err = os.Stat(filepath.Join(si.rootPath, exeName))
+	assert.NoErrorf(t, err, "should be able to stat this executable %s", exeName)
+}
+
+func TestForJob(t *testing.T) {
+	si := NewNextToExe("task-logs")
+	jobPath := si.ForJob("08e126ef-d773-468b-8bab-19a8213cf2ff")
+
+	expectedSuffix := filepath.Join("task-logs", "job-08e1", "08e126ef-d773-468b-8bab-19a8213cf2ff")
+	hasSuffix := strings.HasSuffix(jobPath, expectedSuffix)
+	assert.Truef(t, hasSuffix, "expected %s to have suffix %s", jobPath, expectedSuffix)
+}
+
+func TestErase(t *testing.T) {
+	si := NewNextToExe("task-logs")
+	assert.NoDirExists(t, si.rootPath, "creating a StorageInfo should not create the directory")
+
+	jobPath := si.ForJob("08e126ef-d773-468b-8bab-19a8213cf2ff")
+	assert.NoDirExists(t, jobPath, "getting a path should not create it")
+
+	assert.NoError(t, os.MkdirAll(jobPath, os.ModePerm))
+	assert.DirExists(t, jobPath, "os.MkdirAll is borked")
+
+	assert.NoError(t, si.Erase())
+	assert.NoDirExists(t, si.rootPath, "Erase() should erase the root path, and everything in it")
+}

internal/manager/task_logs/task_logs.go

@@ -158,6 +158,10 @@ func (s *Storage) RotateFile(logger zerolog.Logger, jobID, taskID string) {
 }
 
 // filepath returns the file path suitable to write a log file.
+// Note that this intentionally shares the behaviour of `pathForJob()` in
+// `internal/manager/local_storage/local_storage.go`; it is intended that the
+// file handling code in this source file is migrated to use the `local_storage`
+// package at some point.
 func (s *Storage) filepath(jobID, taskID string) string {
 	var dirpath string
 	if jobID == "" {