From 6c827ffc52f71c940832fcbb37cf2a588498b783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Fri, 25 Mar 2022 14:32:42 +0100 Subject: [PATCH] Shaman: only configure the root directory of the Shaman files Flamenco v2 allowed separate configuration of the Shaman file store and checkout paths. This is now just one setting for "the storage". The file store will be in `{storage}/file-store` and the checkout will happen in `{storage}/jobs`. --- internal/manager/config/defaults.go | 5 ++--- pkg/shaman/checkout/manager.go | 7 ++++--- pkg/shaman/cleanup.go | 4 ++-- pkg/shaman/cleanup_test.go | 16 +++++++-------- pkg/shaman/config/config.go | 32 ++++++++++++++++++++++++----- pkg/shaman/config/testing.go | 8 +++----- pkg/shaman/filestore/filestore.go | 9 ++++---- pkg/shaman/filestore/testing.go | 2 +- pkg/shaman/server.go | 2 +- 9 files changed, 53 insertions(+), 32 deletions(-) diff --git a/internal/manager/config/defaults.go b/internal/manager/config/defaults.go index 31afd1eb..3a9ee971 100644 --- a/internal/manager/config/defaults.go +++ b/internal/manager/config/defaults.go @@ -21,9 +21,8 @@ var defaultConfig = Conf{ SSDPDiscovery: true, Shaman: shaman_config.Config{ - Enabled: false, - FileStorePath: "./shaman-file-storage/file-store", - CheckoutPath: "./shaman-file-storage/checkout", + Enabled: false, + StoragePath: "./shaman-file-storage", GarbageCollect: shaman_config.GarbageCollect{ Period: 24 * time.Hour, MaxAge: 31 * 24 * time.Hour, diff --git a/pkg/shaman/checkout/manager.go b/pkg/shaman/checkout/manager.go index 86cfdd77..92c335a1 100644 --- a/pkg/shaman/checkout/manager.go +++ b/pkg/shaman/checkout/manager.go @@ -73,15 +73,16 @@ var ( // NewManager creates and returns a new Checkout Manager. func NewManager(conf config.Config, fileStore *filestore.Store) *Manager { - logger := log.With().Str("checkoutDir", conf.CheckoutPath).Logger() + checkoutDir := conf.CheckoutPath() + logger := log.With().Str("checkoutDir", checkoutDir).Logger() logger.Info().Msg("opening checkout directory") - err := os.MkdirAll(conf.CheckoutPath, 0777) + err := os.MkdirAll(checkoutDir, 0777) if err != nil { logger.Error().Err(err).Msg("unable to create checkout directory") } - return &Manager{conf.CheckoutPath, fileStore, new(sync.WaitGroup), new(sync.Mutex)} + return &Manager{checkoutDir, fileStore, new(sync.WaitGroup), new(sync.Mutex)} } // Close waits for still-running touch() calls to finish, then returns. diff --git a/pkg/shaman/cleanup.go b/pkg/shaman/cleanup.go index 1baa78f8..e475970e 100644 --- a/pkg/shaman/cleanup.go +++ b/pkg/shaman/cleanup.go @@ -72,7 +72,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) { ageThreshold := s.gcAgeThreshold() logger := log.With(). - Str("checkoutPath", s.config.CheckoutPath). + Str("checkoutPath", s.config.CheckoutPath()). Str("fileStorePath", s.fileStore.StoragePath()). Time("ageThreshold", ageThreshold). Logger() @@ -99,7 +99,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) { Msg("found old files, going to check for links") // Scan the checkout area and extra checkout paths, and discard any old file that is linked. - dirsToCheck := []string{s.config.CheckoutPath} + dirsToCheck := []string{s.config.CheckoutPath()} dirsToCheck = append(dirsToCheck, s.config.GarbageCollect.ExtraCheckoutDirs...) for _, checkDir := range dirsToCheck { if err := s.gcFilterLinkedFiles(checkDir, oldFiles, logger, &stats); err != nil { diff --git a/pkg/shaman/cleanup_test.go b/pkg/shaman/cleanup_test.go index c1b1efa3..6458b40b 100644 --- a/pkg/shaman/cleanup_test.go +++ b/pkg/shaman/cleanup_test.go @@ -43,7 +43,7 @@ func createTestShaman() (*Server, func()) { func makeOld(shaman *Server, expectOld mtimeMap, relPath string) { oldTime := time.Now().Add(-2 * shaman.config.GarbageCollect.MaxAge) - absPath := path.Join(shaman.config.FileStorePath, relPath) + absPath := path.Join(shaman.config.FileStorePath(), relPath) err := os.Chtimes(absPath, oldTime, oldTime) if err != nil { @@ -71,7 +71,7 @@ func TestGCFindOldFiles(t *testing.T) { server, cleanup := createTestShaman() defer cleanup() - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) // Since all the links have just been created, nothing should be considered old. ageThreshold := server.gcAgeThreshold() @@ -98,7 +98,7 @@ func TestGCComponents(t *testing.T) { extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout") server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir} - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) copymap := func(somemap mtimeMap) mtimeMap { theCopy := mtimeMap{} @@ -123,14 +123,14 @@ func TestGCComponents(t *testing.T) { // No symlinks created yet, so this should report all the files in oldFiles. oldFiles := copymap(expectOld) - err := server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), nil) + err := server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), nil) assert.Nil(t, err) assert.EqualValues(t, expectOld, oldFiles) // Create some symlinks checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID") assert.Nil(t, err) - err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath, + err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(), path.Join(checkoutInfo.RelativePath, "use-of-3367.blob")) assert.Nil(t, err) err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir, @@ -144,7 +144,7 @@ func TestGCComponents(t *testing.T) { } oldFiles = copymap(expectOld) stats := GCStats{} - err = server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats) + err = server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats) assert.Equal(t, 1, stats.numSymlinksChecked) // 1 is in checkoutPath, the other in extraCheckoutDir assert.Nil(t, err) assert.Equal(t, len(expectRemovable)+1, len(oldFiles)) // one file is linked from the extra checkout dir @@ -182,7 +182,7 @@ func TestGarbageCollect(t *testing.T) { extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout") server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir} - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) // Make some files old. expectOld := mtimeMap{} @@ -200,7 +200,7 @@ func TestGarbageCollect(t *testing.T) { // Create some symlinks checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID") assert.Nil(t, err) - err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath, + err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(), path.Join(checkoutInfo.RelativePath, "use-of-3367.blob")) assert.Nil(t, err) err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir, diff --git a/pkg/shaman/config/config.go b/pkg/shaman/config/config.go index c8787669..334cb96a 100644 --- a/pkg/shaman/config/config.go +++ b/pkg/shaman/config/config.go @@ -23,20 +23,29 @@ package config import ( + "path/filepath" "time" ) +const ( + // fileStoreSubdir is the sub-directory of the configured storage path, used + // for the file store (i.e. the place where binary blobs are uploaded to). + fileStoreSubdir = "file-store" + + // checkoutSubDir is the sub-directory of the configured storage path, used + // for the checkouts of job files (f.e. the blend files used for render jobs, + // symlinked from the file store dir). + checkoutSubDir = "jobs" +) + // Config contains all the Shaman configuration type Config struct { // Used only for unit tests, so that they know where the temporary // directory created for this test is located. TestTempDir string `yaml:"-"` - Enabled bool `yaml:"enabled"` - - FileStorePath string `yaml:"fileStorePath"` - CheckoutPath string `yaml:"checkoutPath"` - + Enabled bool `yaml:"enabled"` + StoragePath string `yaml:"storagePath"` GarbageCollect GarbageCollect `yaml:"garbageCollect"` } @@ -53,3 +62,16 @@ type GarbageCollect struct { // while we're performing a manual sweep. SilentlyDisable bool `yaml:"-"` } + +// FileStorePath returns the sub-directory of the configured storage path, +// used for the file store (i.e. the place where binary blobs are uploaded to). +func (c Config) FileStorePath() string { + return filepath.Join(c.StoragePath, fileStoreSubdir) +} + +// CheckoutPath returns the sub-directory of the configured storage path, used +// for the checkouts of job files (f.e. the blend files used for render jobs, +// symlinked from the file store dir). +func (c Config) CheckoutPath() string { + return filepath.Join(c.StoragePath, checkoutSubDir) +} diff --git a/pkg/shaman/config/testing.go b/pkg/shaman/config/testing.go index 7b362090..4da32890 100644 --- a/pkg/shaman/config/testing.go +++ b/pkg/shaman/config/testing.go @@ -25,7 +25,6 @@ package config import ( "io/ioutil" "os" - "path" "time" ) @@ -37,10 +36,9 @@ func CreateTestConfig() (conf Config, cleanup func()) { } conf = Config{ - TestTempDir: tempDir, - Enabled: true, - FileStorePath: path.Join(tempDir, "file-store"), - CheckoutPath: path.Join(tempDir, "checkout"), + TestTempDir: tempDir, + Enabled: true, + StoragePath: tempDir, GarbageCollect: GarbageCollect{ Period: 8 * time.Hour, diff --git a/pkg/shaman/filestore/filestore.go b/pkg/shaman/filestore/filestore.go index affc59f2..2a8f519d 100644 --- a/pkg/shaman/filestore/filestore.go +++ b/pkg/shaman/filestore/filestore.go @@ -41,11 +41,12 @@ type Store struct { // New returns a new file store. func New(conf config.Config) *Store { - log.Info().Str("storageDir", conf.FileStorePath).Msg("shaman: opening file store") + storageDir := conf.FileStorePath() + log.Info().Str("storageDir", storageDir).Msg("shaman: opening file store") store := &Store{ - conf.FileStorePath, - storageBin{conf.FileStorePath, "uploading", true, ".tmp"}, - storageBin{conf.FileStorePath, "stored", false, ".blob"}, + storageDir, + storageBin{storageDir, "uploading", true, ".tmp"}, + storageBin{storageDir, "stored", false, ".blob"}, } store.createDirectoryStructure() return store diff --git a/pkg/shaman/filestore/testing.go b/pkg/shaman/filestore/testing.go index 09e6420b..d663f9c2 100644 --- a/pkg/shaman/filestore/testing.go +++ b/pkg/shaman/filestore/testing.go @@ -41,7 +41,7 @@ func CreateTestStore() *Store { } conf := config.Config{ - FileStorePath: tempDir, + StoragePath: tempDir, } return New(conf) } diff --git a/pkg/shaman/server.go b/pkg/shaman/server.go index d7edac36..a6122f04 100644 --- a/pkg/shaman/server.go +++ b/pkg/shaman/server.go @@ -56,7 +56,7 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { return nil } - if conf.CheckoutPath == "" { + if conf.StoragePath == "" { log.Error().Interface("config", conf).Msg("shaman: no checkout path configured, unable to start") return nil }