/* (c) 2019, Blender Foundation - Sybren A. Stüvel * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package checkout import ( "errors" "os" "path" "path/filepath" "sync" "time" "github.com/rs/zerolog/log" "git.blender.org/flamenco/pkg/shaman/config" "git.blender.org/flamenco/pkg/shaman/filestore" "git.blender.org/flamenco/pkg/shaman/touch" ) // Manager creates checkouts and provides info about missing files. type Manager struct { checkoutBasePath string fileStore filestore.Storage wg sync.WaitGroup } // ResolvedCheckoutInfo contains the result of validating the Checkout ID and parsing it into a final path. type ResolvedCheckoutInfo struct { // The absolute path on our filesystem. absolutePath string // The path relative to the Manager.checkoutBasePath. This is what is // sent back to the client. RelativePath string } // Errors returned by the Checkout Manager. var ( ErrCheckoutAlreadyExists = errors.New("A checkout with this ID already exists") ErrInvalidCheckoutID = errors.New("The Checkout ID is invalid") ) // NewManager creates and returns a new Checkout Manager. func NewManager(conf config.Config, fileStore filestore.Storage) *Manager { logger := log.With().Str("checkoutDir", conf.CheckoutPath).Logger() logger.Info().Msg("opening checkout directory") err := os.MkdirAll(conf.CheckoutPath, 0777) if err != nil { logger.Error().Err(err).Msg("unable to create checkout directory") } return &Manager{conf.CheckoutPath, fileStore, sync.WaitGroup{}} } // Close waits for still-running touch() calls to finish, then returns. func (m *Manager) Close() { log.Info().Msg("shutting down Checkout manager") m.wg.Wait() } func (m *Manager) pathForCheckoutID(checkoutID string) (ResolvedCheckoutInfo, error) { if !isValidCheckoutID(checkoutID) { return ResolvedCheckoutInfo{}, ErrInvalidCheckoutID } // When changing the number of path components the checkout ID is turned into, // be sure to also update the EraseCheckout() function for this. // We're expecting ObjectIDs as checkoutIDs, which means most variation // is in the last characters. lastBitIndex := len(checkoutID) - 2 relativePath := path.Join(checkoutID[lastBitIndex:], checkoutID) return ResolvedCheckoutInfo{ absolutePath: path.Join(m.checkoutBasePath, relativePath), RelativePath: relativePath, }, nil } // PrepareCheckout creates the root directory for a specific checkout. // Returns the path relative to the checkout root directory. func (m *Manager) PrepareCheckout(checkoutID string) (ResolvedCheckoutInfo, error) { checkoutPaths, err := m.pathForCheckoutID(checkoutID) if err != nil { return ResolvedCheckoutInfo{}, err } logger := log.With(). Str("checkoutPath", checkoutPaths.absolutePath). Str("checkoutID", checkoutID). Logger() if stat, err := os.Stat(checkoutPaths.absolutePath); !os.IsNotExist(err) { if err == nil { if stat.IsDir() { logger.Debug().Msg("checkout path exists") } else { logger.Error().Msg("checkout path exists but is not a directory") } // No error stat'ing this path, indicating it's an existing checkout. return ResolvedCheckoutInfo{}, ErrCheckoutAlreadyExists } // If it's any other error, it's really a problem on our side. logger.Error().Err(err).Msg("unable to stat checkout directory") return ResolvedCheckoutInfo{}, err } if err := os.MkdirAll(checkoutPaths.absolutePath, 0777); err != nil { logger.Error().Err(err).Msg("unable to create checkout directory") } logger.Info().Str("relPath", checkoutPaths.RelativePath).Msg("created checkout directory") return checkoutPaths, nil } // EraseCheckout removes the checkout directory structure identified by the ID. func (m *Manager) EraseCheckout(checkoutID string) error { checkoutPaths, err := m.pathForCheckoutID(checkoutID) if err != nil { return err } logger := log.With(). Str("checkoutPath", checkoutPaths.absolutePath). Str("checkoutID", checkoutID). Logger() if err := os.RemoveAll(checkoutPaths.absolutePath); err != nil { logger.Error().Err(err).Msg("unable to remove checkout directory") return err } // Try to remove the parent path as well, to not keep the dangling two-letter dirs. // Failure is fine, though, because there is no guarantee it's empty anyway. os.Remove(path.Dir(checkoutPaths.absolutePath)) logger.Info().Msg("removed checkout directory") return nil } // SymlinkToCheckout creates a symlink at symlinkPath to blobPath. // It does *not* do any validation of the validity of the paths! func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath string) error { symlinkPath := path.Join(checkoutPath, symlinkRelativePath) logger := log.With(). Str("blobPath", blobPath). Str("symlinkPath", symlinkPath). Logger() blobPath, err := filepath.Abs(blobPath) if err != nil { logger.Error().Err(err).Msg("unable to make blobPath absolute") return err } logger.Debug().Msg("creating symlink") // This is expected to fail sometimes, because we don't create parent directories yet. // We only create those when we get a failure from symlinking. err = os.Symlink(blobPath, symlinkPath) if err == nil { return err } if !os.IsNotExist(err) { logger.Error().Err(err).Msg("unable to create symlink") return err } logger.Debug().Msg("creating parent directory") dir := path.Dir(symlinkPath) if err := os.MkdirAll(dir, 0777); err != nil { logger.Error().Err(err).Msg("unable to create parent directory") return err } if err := os.Symlink(blobPath, symlinkPath); err != nil { logger.Error().Err(err).Msg("unable to create symlink, after creating parent directory") return err } // Change the modification time of the blob to mark it as 'referenced' just now. m.wg.Add(1) go func() { touchFile(blobPath) m.wg.Done() }() return nil } // touchFile changes the modification time of the blob to mark it as 'referenced' just now. func touchFile(blobPath string) error { if blobPath == "" { return os.ErrInvalid } now := time.Now() err := touch.Touch(blobPath) if err != nil { return err } duration := time.Now().Sub(now) logger := log.With().Str("file", blobPath).Logger() if duration > 1*time.Second { logger.Warn().Str("duration", duration.String()).Msg("done touching but took a long time") } logger.Debug().Msg("done touching") return err }