Shaman: only configure the root directory of the Shaman files

Flamenco v2 allowed separate configuration of the Shaman file store and
checkout paths. This is now just one setting for "the storage". The file
store will be in `{storage}/file-store` and the checkout will happen in
`{storage}/jobs`.

internal/manager/config/defaults.go

@@ -21,9 +21,8 @@ var defaultConfig = Conf{
 		SSDPDiscovery: true,
 
 		Shaman: shaman_config.Config{
-			Enabled:       false,
+			Enabled:     false,
-			FileStorePath: "./shaman-file-storage/file-store",
+			StoragePath: "./shaman-file-storage",
-			CheckoutPath:  "./shaman-file-storage/checkout",
 			GarbageCollect: shaman_config.GarbageCollect{
 				Period:            24 * time.Hour,
 				MaxAge:            31 * 24 * time.Hour,

pkg/shaman/checkout/manager.go

@@ -73,15 +73,16 @@ var (
 
 // NewManager creates and returns a new Checkout Manager.
 func NewManager(conf config.Config, fileStore *filestore.Store) *Manager {
-	logger := log.With().Str("checkoutDir", conf.CheckoutPath).Logger()
+	checkoutDir := conf.CheckoutPath()
+	logger := log.With().Str("checkoutDir", checkoutDir).Logger()
 	logger.Info().Msg("opening checkout directory")
 
-	err := os.MkdirAll(conf.CheckoutPath, 0777)
+	err := os.MkdirAll(checkoutDir, 0777)
 	if err != nil {
 		logger.Error().Err(err).Msg("unable to create checkout directory")
 	}
 
-	return &Manager{conf.CheckoutPath, fileStore, new(sync.WaitGroup), new(sync.Mutex)}
+	return &Manager{checkoutDir, fileStore, new(sync.WaitGroup), new(sync.Mutex)}
 }
 
 // Close waits for still-running touch() calls to finish, then returns.

pkg/shaman/cleanup.go

@@ -72,7 +72,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) {
 	ageThreshold := s.gcAgeThreshold()
 
 	logger := log.With().
-		Str("checkoutPath", s.config.CheckoutPath).
+		Str("checkoutPath", s.config.CheckoutPath()).
 		Str("fileStorePath", s.fileStore.StoragePath()).
 		Time("ageThreshold", ageThreshold).
 		Logger()
@@ -99,7 +99,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) {
 		Msg("found old files, going to check for links")
 
 	// Scan the checkout area and extra checkout paths, and discard any old file that is linked.
-	dirsToCheck := []string{s.config.CheckoutPath}
+	dirsToCheck := []string{s.config.CheckoutPath()}
 	dirsToCheck = append(dirsToCheck, s.config.GarbageCollect.ExtraCheckoutDirs...)
 	for _, checkDir := range dirsToCheck {
 		if err := s.gcFilterLinkedFiles(checkDir, oldFiles, logger, &stats); err != nil {

pkg/shaman/cleanup_test.go

@@ -43,7 +43,7 @@ func createTestShaman() (*Server, func()) {
 
 func makeOld(shaman *Server, expectOld mtimeMap, relPath string) {
 	oldTime := time.Now().Add(-2 * shaman.config.GarbageCollect.MaxAge)
-	absPath := path.Join(shaman.config.FileStorePath, relPath)
+	absPath := path.Join(shaman.config.FileStorePath(), relPath)
 
 	err := os.Chtimes(absPath, oldTime, oldTime)
 	if err != nil {
@@ -71,7 +71,7 @@ func TestGCFindOldFiles(t *testing.T) {
 	server, cleanup := createTestShaman()
 	defer cleanup()
 
-	filestore.LinkTestFileStore(server.config.FileStorePath)
+	filestore.LinkTestFileStore(server.config.FileStorePath())
 
 	// Since all the links have just been created, nothing should be considered old.
 	ageThreshold := server.gcAgeThreshold()
@@ -98,7 +98,7 @@ func TestGCComponents(t *testing.T) {
 	extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout")
 	server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir}
 
-	filestore.LinkTestFileStore(server.config.FileStorePath)
+	filestore.LinkTestFileStore(server.config.FileStorePath())
 
 	copymap := func(somemap mtimeMap) mtimeMap {
 		theCopy := mtimeMap{}
@@ -123,14 +123,14 @@ func TestGCComponents(t *testing.T) {
 
 	// No symlinks created yet, so this should report all the files in oldFiles.
 	oldFiles := copymap(expectOld)
-	err := server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), nil)
+	err := server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), nil)
 	assert.Nil(t, err)
 	assert.EqualValues(t, expectOld, oldFiles)
 
 	// Create some symlinks
 	checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID")
 	assert.Nil(t, err)
-	err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath,
+	err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(),
 		path.Join(checkoutInfo.RelativePath, "use-of-3367.blob"))
 	assert.Nil(t, err)
 	err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir,
@@ -144,7 +144,7 @@ func TestGCComponents(t *testing.T) {
 	}
 	oldFiles = copymap(expectOld)
 	stats := GCStats{}
-	err = server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats)
+	err = server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats)
 	assert.Equal(t, 1, stats.numSymlinksChecked) // 1 is in checkoutPath, the other in extraCheckoutDir
 	assert.Nil(t, err)
 	assert.Equal(t, len(expectRemovable)+1, len(oldFiles)) // one file is linked from the extra checkout dir
@@ -182,7 +182,7 @@ func TestGarbageCollect(t *testing.T) {
 	extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout")
 	server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir}
 
-	filestore.LinkTestFileStore(server.config.FileStorePath)
+	filestore.LinkTestFileStore(server.config.FileStorePath())
 
 	// Make some files old.
 	expectOld := mtimeMap{}
@@ -200,7 +200,7 @@ func TestGarbageCollect(t *testing.T) {
 	// Create some symlinks
 	checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID")
 	assert.Nil(t, err)
-	err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath,
+	err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(),
 		path.Join(checkoutInfo.RelativePath, "use-of-3367.blob"))
 	assert.Nil(t, err)
 	err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir,

pkg/shaman/config/config.go

@@ -23,20 +23,29 @@
 package config
 
 import (
+	"path/filepath"
 	"time"
 )
 
+const (
+	// fileStoreSubdir is the sub-directory of the configured storage path, used
+	// for the file store (i.e. the place where binary blobs are uploaded to).
+	fileStoreSubdir = "file-store"
+
+	// checkoutSubDir is the sub-directory of the configured storage path, used
+	// for the checkouts of job files (f.e. the blend files used for render jobs,
+	// symlinked from the file store dir).
+	checkoutSubDir = "jobs"
+)
+
 // Config contains all the Shaman configuration
 type Config struct {
 	// Used only for unit tests, so that they know where the temporary
 	// directory created for this test is located.
 	TestTempDir string `yaml:"-"`
 
-	Enabled bool `yaml:"enabled"`
+	Enabled        bool           `yaml:"enabled"`
-
+	StoragePath    string         `yaml:"storagePath"`
-	FileStorePath string `yaml:"fileStorePath"`
-	CheckoutPath  string `yaml:"checkoutPath"`
-
 	GarbageCollect GarbageCollect `yaml:"garbageCollect"`
 }
 
@@ -53,3 +62,16 @@ type GarbageCollect struct {
 	// while we're performing a manual sweep.
 	SilentlyDisable bool `yaml:"-"`
 }
+
+// FileStorePath returns the sub-directory of the configured storage path,
+// used for the file store (i.e. the place where binary blobs are uploaded to).
+func (c Config) FileStorePath() string {
+	return filepath.Join(c.StoragePath, fileStoreSubdir)
+}
+
+// CheckoutPath returns the sub-directory of the configured storage path, used
+// for the checkouts of job files (f.e. the blend files used for render jobs,
+// symlinked from the file store dir).
+func (c Config) CheckoutPath() string {
+	return filepath.Join(c.StoragePath, checkoutSubDir)
+}

pkg/shaman/config/testing.go

@@ -25,7 +25,6 @@ package config
 import (
 	"io/ioutil"
 	"os"
-	"path"
 	"time"
 )
 
@@ -37,10 +36,9 @@ func CreateTestConfig() (conf Config, cleanup func()) {
 	}
 
 	conf = Config{
-		TestTempDir:   tempDir,
+		TestTempDir: tempDir,
-		Enabled:       true,
+		Enabled:     true,
-		FileStorePath: path.Join(tempDir, "file-store"),
+		StoragePath: tempDir,
-		CheckoutPath:  path.Join(tempDir, "checkout"),
 
 		GarbageCollect: GarbageCollect{
 			Period:            8 * time.Hour,

pkg/shaman/filestore/filestore.go

@@ -41,11 +41,12 @@ type Store struct {
 
 // New returns a new file store.
 func New(conf config.Config) *Store {
-	log.Info().Str("storageDir", conf.FileStorePath).Msg("shaman: opening file store")
+	storageDir := conf.FileStorePath()
+	log.Info().Str("storageDir", storageDir).Msg("shaman: opening file store")
 	store := &Store{
-		conf.FileStorePath,
+		storageDir,
-		storageBin{conf.FileStorePath, "uploading", true, ".tmp"},
+		storageBin{storageDir, "uploading", true, ".tmp"},
-		storageBin{conf.FileStorePath, "stored", false, ".blob"},
+		storageBin{storageDir, "stored", false, ".blob"},
 	}
 	store.createDirectoryStructure()
 	return store

pkg/shaman/filestore/testing.go

@@ -41,7 +41,7 @@ func CreateTestStore() *Store {
 	}
 
 	conf := config.Config{
-		FileStorePath: tempDir,
+		StoragePath: tempDir,
 	}
 	return New(conf)
 }

pkg/shaman/server.go

@@ -56,7 +56,7 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server {
 		return nil
 	}
 
-	if conf.CheckoutPath == "" {
+	if conf.StoragePath == "" {
 		log.Error().Interface("config", conf).Msg("shaman: no checkout path configured, unable to start")
 		return nil
 	}