diff --git a/internal/manager/config/defaults.go b/internal/manager/config/defaults.go index 31afd1eb..3a9ee971 100644 --- a/internal/manager/config/defaults.go +++ b/internal/manager/config/defaults.go @@ -21,9 +21,8 @@ var defaultConfig = Conf{ SSDPDiscovery: true, Shaman: shaman_config.Config{ - Enabled: false, - FileStorePath: "./shaman-file-storage/file-store", - CheckoutPath: "./shaman-file-storage/checkout", + Enabled: false, + StoragePath: "./shaman-file-storage", GarbageCollect: shaman_config.GarbageCollect{ Period: 24 * time.Hour, MaxAge: 31 * 24 * time.Hour, diff --git a/pkg/shaman/checkout/manager.go b/pkg/shaman/checkout/manager.go index 86cfdd77..92c335a1 100644 --- a/pkg/shaman/checkout/manager.go +++ b/pkg/shaman/checkout/manager.go @@ -73,15 +73,16 @@ var ( // NewManager creates and returns a new Checkout Manager. func NewManager(conf config.Config, fileStore *filestore.Store) *Manager { - logger := log.With().Str("checkoutDir", conf.CheckoutPath).Logger() + checkoutDir := conf.CheckoutPath() + logger := log.With().Str("checkoutDir", checkoutDir).Logger() logger.Info().Msg("opening checkout directory") - err := os.MkdirAll(conf.CheckoutPath, 0777) + err := os.MkdirAll(checkoutDir, 0777) if err != nil { logger.Error().Err(err).Msg("unable to create checkout directory") } - return &Manager{conf.CheckoutPath, fileStore, new(sync.WaitGroup), new(sync.Mutex)} + return &Manager{checkoutDir, fileStore, new(sync.WaitGroup), new(sync.Mutex)} } // Close waits for still-running touch() calls to finish, then returns. diff --git a/pkg/shaman/cleanup.go b/pkg/shaman/cleanup.go index 1baa78f8..e475970e 100644 --- a/pkg/shaman/cleanup.go +++ b/pkg/shaman/cleanup.go @@ -72,7 +72,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) { ageThreshold := s.gcAgeThreshold() logger := log.With(). - Str("checkoutPath", s.config.CheckoutPath). + Str("checkoutPath", s.config.CheckoutPath()). Str("fileStorePath", s.fileStore.StoragePath()). Time("ageThreshold", ageThreshold). Logger() @@ -99,7 +99,7 @@ func (s *Server) GCStorage(doDryRun bool) (stats GCStats) { Msg("found old files, going to check for links") // Scan the checkout area and extra checkout paths, and discard any old file that is linked. - dirsToCheck := []string{s.config.CheckoutPath} + dirsToCheck := []string{s.config.CheckoutPath()} dirsToCheck = append(dirsToCheck, s.config.GarbageCollect.ExtraCheckoutDirs...) for _, checkDir := range dirsToCheck { if err := s.gcFilterLinkedFiles(checkDir, oldFiles, logger, &stats); err != nil { diff --git a/pkg/shaman/cleanup_test.go b/pkg/shaman/cleanup_test.go index c1b1efa3..6458b40b 100644 --- a/pkg/shaman/cleanup_test.go +++ b/pkg/shaman/cleanup_test.go @@ -43,7 +43,7 @@ func createTestShaman() (*Server, func()) { func makeOld(shaman *Server, expectOld mtimeMap, relPath string) { oldTime := time.Now().Add(-2 * shaman.config.GarbageCollect.MaxAge) - absPath := path.Join(shaman.config.FileStorePath, relPath) + absPath := path.Join(shaman.config.FileStorePath(), relPath) err := os.Chtimes(absPath, oldTime, oldTime) if err != nil { @@ -71,7 +71,7 @@ func TestGCFindOldFiles(t *testing.T) { server, cleanup := createTestShaman() defer cleanup() - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) // Since all the links have just been created, nothing should be considered old. ageThreshold := server.gcAgeThreshold() @@ -98,7 +98,7 @@ func TestGCComponents(t *testing.T) { extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout") server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir} - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) copymap := func(somemap mtimeMap) mtimeMap { theCopy := mtimeMap{} @@ -123,14 +123,14 @@ func TestGCComponents(t *testing.T) { // No symlinks created yet, so this should report all the files in oldFiles. oldFiles := copymap(expectOld) - err := server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), nil) + err := server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), nil) assert.Nil(t, err) assert.EqualValues(t, expectOld, oldFiles) // Create some symlinks checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID") assert.Nil(t, err) - err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath, + err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(), path.Join(checkoutInfo.RelativePath, "use-of-3367.blob")) assert.Nil(t, err) err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir, @@ -144,7 +144,7 @@ func TestGCComponents(t *testing.T) { } oldFiles = copymap(expectOld) stats := GCStats{} - err = server.gcFilterLinkedFiles(server.config.CheckoutPath, oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats) + err = server.gcFilterLinkedFiles(server.config.CheckoutPath(), oldFiles, log.With().Str("package", "shaman/test").Logger(), &stats) assert.Equal(t, 1, stats.numSymlinksChecked) // 1 is in checkoutPath, the other in extraCheckoutDir assert.Nil(t, err) assert.Equal(t, len(expectRemovable)+1, len(oldFiles)) // one file is linked from the extra checkout dir @@ -182,7 +182,7 @@ func TestGarbageCollect(t *testing.T) { extraCheckoutDir := path.Join(server.config.TestTempDir, "extra-checkout") server.config.GarbageCollect.ExtraCheckoutDirs = []string{extraCheckoutDir} - filestore.LinkTestFileStore(server.config.FileStorePath) + filestore.LinkTestFileStore(server.config.FileStorePath()) // Make some files old. expectOld := mtimeMap{} @@ -200,7 +200,7 @@ func TestGarbageCollect(t *testing.T) { // Create some symlinks checkoutInfo, err := server.checkoutMan.PrepareCheckout("checkoutID") assert.Nil(t, err) - err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath, + err = server.checkoutMan.SymlinkToCheckout(absPaths["3367.blob"], server.config.CheckoutPath(), path.Join(checkoutInfo.RelativePath, "use-of-3367.blob")) assert.Nil(t, err) err = server.checkoutMan.SymlinkToCheckout(absPaths["781.blob"], extraCheckoutDir, diff --git a/pkg/shaman/config/config.go b/pkg/shaman/config/config.go index c8787669..334cb96a 100644 --- a/pkg/shaman/config/config.go +++ b/pkg/shaman/config/config.go @@ -23,20 +23,29 @@ package config import ( + "path/filepath" "time" ) +const ( + // fileStoreSubdir is the sub-directory of the configured storage path, used + // for the file store (i.e. the place where binary blobs are uploaded to). + fileStoreSubdir = "file-store" + + // checkoutSubDir is the sub-directory of the configured storage path, used + // for the checkouts of job files (f.e. the blend files used for render jobs, + // symlinked from the file store dir). + checkoutSubDir = "jobs" +) + // Config contains all the Shaman configuration type Config struct { // Used only for unit tests, so that they know where the temporary // directory created for this test is located. TestTempDir string `yaml:"-"` - Enabled bool `yaml:"enabled"` - - FileStorePath string `yaml:"fileStorePath"` - CheckoutPath string `yaml:"checkoutPath"` - + Enabled bool `yaml:"enabled"` + StoragePath string `yaml:"storagePath"` GarbageCollect GarbageCollect `yaml:"garbageCollect"` } @@ -53,3 +62,16 @@ type GarbageCollect struct { // while we're performing a manual sweep. SilentlyDisable bool `yaml:"-"` } + +// FileStorePath returns the sub-directory of the configured storage path, +// used for the file store (i.e. the place where binary blobs are uploaded to). +func (c Config) FileStorePath() string { + return filepath.Join(c.StoragePath, fileStoreSubdir) +} + +// CheckoutPath returns the sub-directory of the configured storage path, used +// for the checkouts of job files (f.e. the blend files used for render jobs, +// symlinked from the file store dir). +func (c Config) CheckoutPath() string { + return filepath.Join(c.StoragePath, checkoutSubDir) +} diff --git a/pkg/shaman/config/testing.go b/pkg/shaman/config/testing.go index 7b362090..4da32890 100644 --- a/pkg/shaman/config/testing.go +++ b/pkg/shaman/config/testing.go @@ -25,7 +25,6 @@ package config import ( "io/ioutil" "os" - "path" "time" ) @@ -37,10 +36,9 @@ func CreateTestConfig() (conf Config, cleanup func()) { } conf = Config{ - TestTempDir: tempDir, - Enabled: true, - FileStorePath: path.Join(tempDir, "file-store"), - CheckoutPath: path.Join(tempDir, "checkout"), + TestTempDir: tempDir, + Enabled: true, + StoragePath: tempDir, GarbageCollect: GarbageCollect{ Period: 8 * time.Hour, diff --git a/pkg/shaman/filestore/filestore.go b/pkg/shaman/filestore/filestore.go index affc59f2..2a8f519d 100644 --- a/pkg/shaman/filestore/filestore.go +++ b/pkg/shaman/filestore/filestore.go @@ -41,11 +41,12 @@ type Store struct { // New returns a new file store. func New(conf config.Config) *Store { - log.Info().Str("storageDir", conf.FileStorePath).Msg("shaman: opening file store") + storageDir := conf.FileStorePath() + log.Info().Str("storageDir", storageDir).Msg("shaman: opening file store") store := &Store{ - conf.FileStorePath, - storageBin{conf.FileStorePath, "uploading", true, ".tmp"}, - storageBin{conf.FileStorePath, "stored", false, ".blob"}, + storageDir, + storageBin{storageDir, "uploading", true, ".tmp"}, + storageBin{storageDir, "stored", false, ".blob"}, } store.createDirectoryStructure() return store diff --git a/pkg/shaman/filestore/testing.go b/pkg/shaman/filestore/testing.go index 09e6420b..d663f9c2 100644 --- a/pkg/shaman/filestore/testing.go +++ b/pkg/shaman/filestore/testing.go @@ -41,7 +41,7 @@ func CreateTestStore() *Store { } conf := config.Config{ - FileStorePath: tempDir, + StoragePath: tempDir, } return New(conf) } diff --git a/pkg/shaman/server.go b/pkg/shaman/server.go index d7edac36..a6122f04 100644 --- a/pkg/shaman/server.go +++ b/pkg/shaman/server.go @@ -56,7 +56,7 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { return nil } - if conf.CheckoutPath == "" { + if conf.StoragePath == "" { log.Error().Interface("config", conf).Msg("shaman: no checkout path configured, unable to start") return nil }