From 71f2947c4bb20e6345e791e04b8d701390421912 Mon Sep 17 00:00:00 2001 From: Bastien Montagne Date: Thu, 8 Jun 2023 16:20:43 +0200 Subject: [PATCH] Add 'copy-file' command. (#104220) Initial implementation with some basic tests. The API command only accept one source and destination path, which must be absolute. If the destination already exists, it will not be ovewritten, and an error is generated. JS job definition code using this new command can easily handle list of paths and/or relative paths if needed. Reviewed-on: https://projects.blender.org/studio/flamenco/pulls/104220 --- internal/worker/command_exe.go | 1 + internal/worker/command_file_mgmt.go | 117 +++++++++++++++ internal/worker/command_file_mgmt_test.go | 167 ++++++++++++++++++++++ 3 files changed, 285 insertions(+) diff --git a/internal/worker/command_exe.go b/internal/worker/command_exe.go index 0224a64e..e906b453 100644 --- a/internal/worker/command_exe.go +++ b/internal/worker/command_exe.go @@ -90,6 +90,7 @@ func NewCommandExecutor(cli CommandLineRunner, listener CommandListener, timeSer // file-management "move-directory": ce.cmdMoveDirectory, + "copy-file": ce.cmdCopyFile, } return ce diff --git a/internal/worker/command_file_mgmt.go b/internal/worker/command_file_mgmt.go index 43fe1b44..e1e60c91 100644 --- a/internal/worker/command_file_mgmt.go +++ b/internal/worker/command_file_mgmt.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -80,6 +81,77 @@ func (ce *CommandExecutor) cmdMoveDirectory(ctx context.Context, logger zerolog. return ce.moveAndLog(ctx, taskID, cmd.Name, src, dest) } +// cmdCopyFiles executes the "copy-file" command. +// It takes an absolute source and destination file path, +// and copies the source file to its destination, if possible. +// Missing directories in destination path are created as needed. +// If the target path already exists, an error is returned. Destination will not be overwritten. +func (ce *CommandExecutor) cmdCopyFile(ctx context.Context, logger zerolog.Logger, taskID string, cmd api.Command) error { + var src, dest string + var ok bool + + logger = logger.With(). + Interface("command", cmd). + Str("src", src). + Str("dest", dest). + Logger() + + if src, ok = cmdParameter[string](cmd, "src"); !ok || src == "" { + msg := "missing or empty 'src' parameter" + err := NewParameterMissingError("src", cmd) + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + if !filepath.IsAbs(src) { + msg := fmt.Sprintf("source path %q is not absolute, not copying anything", src) + err := NewParameterInvalidError("src", cmd, "path is not absolute") + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + if !fileExists(src) { + msg := fmt.Sprintf("source path %q does not exist, not copying anything", src) + err := NewParameterInvalidError("src", cmd, "path does not exist") + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + + if dest, ok = cmdParameter[string](cmd, "dest"); !ok || dest == "" { + msg := "missing or empty 'dest' parameter" + err := NewParameterMissingError("dest", cmd) + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + if !filepath.IsAbs(dest) { + msg := fmt.Sprintf("destination path %q is not absolute, not copying anything", src) + err := NewParameterInvalidError("dest", cmd, "path is not absolute") + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + if fileExists(dest) { + msg := fmt.Sprintf("destination path %q already exists, not copying anything", dest) + err := NewParameterInvalidError("dest", cmd, "path already exists") + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg) + } + + msg := fmt.Sprintf("copying %q to %q", src, dest) + if err := ce.errorLogProcess(ctx, logger, cmd, taskID, nil, msg); err != nil { + return err + } + + err, logMsg := fileCopy(src, dest) + return ce.errorLogProcess(ctx, logger, cmd, taskID, err, logMsg) +} + +func (ce *CommandExecutor) errorLogProcess(ctx context.Context, logger zerolog.Logger, cmd api.Command, taskID string, err error, logMsg string) error { + msg := fmt.Sprintf("%s: %s", cmd.Name, logMsg) + + if err != nil { + logger.Warn().Msg(msg) + } else { + logger.Info().Msg(msg) + } + + if logErr := ce.listener.LogProduced(ctx, taskID, msg); logErr != nil { + return logErr + } + return err +} + // moveAndLog renames a file/directory from `src` to `dest`, and logs the moveAndLog. // The other parameters are just for logging. func (ce *CommandExecutor) moveAndLog(ctx context.Context, taskID, cmdName, src, dest string) error { @@ -99,6 +171,51 @@ func (ce *CommandExecutor) moveAndLog(ctx context.Context, taskID, cmdName, src, return nil } +func fileCopy(src, dest string) (error, string) { + src_file, err := os.Open(src) + if err != nil { + msg := fmt.Sprintf("failed to open source file %q: %v", src, err) + return err, msg + } + defer src_file.Close() + + src_file_stat, err := src_file.Stat() + if err != nil { + msg := fmt.Sprintf("failed to stat source file %q: %v", src, err) + return err, msg + } + + if !src_file_stat.Mode().IsRegular() { + err := &os.PathError{Op: "stat", Path: src, Err: errors.New("Not a regular file")} + msg := fmt.Sprintf("invalid source file %q: %v", src, err) + return err, msg + } + + dest_dirpath := filepath.Dir(dest) + if !fileExists(dest_dirpath) { + if err := os.MkdirAll(dest_dirpath, 0750); err != nil { + msg := fmt.Sprintf("failed to create directories %q: %v", dest_dirpath, err) + return err, msg + } + } + + dest_file, err := os.Create(dest) + if err != nil { + msg := fmt.Sprintf("failed to create destination file %q: %v", dest, err) + return err, msg + } + defer dest_file.Close() + + if _, err := io.Copy(dest_file, src_file); err != nil { + msg := fmt.Sprintf("failed to copy %q to %q: %v", src, dest, err) + return err, msg + } + + msg := fmt.Sprintf("copied %q to %q", src, dest) + return nil, msg +} + + func fileExists(filename string) bool { _, err := os.Stat(filename) return !errors.Is(err, fs.ErrNotExist) diff --git a/internal/worker/command_file_mgmt_test.go b/internal/worker/command_file_mgmt_test.go index 02ae0a96..4c68baa9 100644 --- a/internal/worker/command_file_mgmt_test.go +++ b/internal/worker/command_file_mgmt_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/assert" ) +// `move-directory` tests. + type cmdMoveDirFixture struct { mockCtrl *gomock.Controller ce *CommandExecutor @@ -264,6 +266,163 @@ func (f cmdMoveDirFixture) run() error { return f.ce.Run(f.ctx, taskID, cmd) } +// `copy-file` tests. + +type cmdCopyFileFixture struct { + mockCtrl *gomock.Controller + ce *CommandExecutor + mocks *CommandExecutorMocks + ctx context.Context + + temppath string + cwd string + + absolute_src_path string + absolute_dest_path string +} + +func TestCmdCopyFile(t *testing.T) { + f := newCmdCopyFileFixture(t) + defer f.finish(t) + + src_dirpath := filepath.Join(f.temppath, "src_path/to") + dest_dirpath := filepath.Join(f.temppath, "dest_path/to") + + f.absolute_src_path = filepath.Join(src_dirpath, "file1.txt") + f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt") + + directoryEnsureExist(src_dirpath) + assert.DirExists(t, src_dirpath) + + fileCreateEmpty(f.absolute_src_path) + assert.FileExists(t, f.absolute_src_path) + + assert.NoDirExists(t, dest_dirpath) + assert.NoFileExists(t, f.absolute_dest_path) + + f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID, + fmt.Sprintf("copy-file: copying %q to %q", f.absolute_src_path, f.absolute_dest_path)) + + f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID, + fmt.Sprintf("copy-file: copied %q to %q", f.absolute_src_path, f.absolute_dest_path)) + + assert.NoError(t, f.run()) + + assert.DirExists(t, src_dirpath) + assert.DirExists(t, dest_dirpath) + assert.FileExists(t, f.absolute_src_path) + assert.FileExists(t, f.absolute_dest_path) +} + +func TestCmdCopyFileDestinationExists(t *testing.T) { + f := newCmdCopyFileFixture(t) + defer f.finish(t) + + src_dirpath := filepath.Join(f.temppath, "src_path/to") + dest_dirpath := filepath.Join(f.temppath, "dest_path/to") + + f.absolute_src_path = filepath.Join(src_dirpath, "file1.txt") + f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt") + + directoryEnsureExist(src_dirpath) + assert.DirExists(t, src_dirpath) + + fileCreateEmpty(f.absolute_src_path) + assert.FileExists(t, f.absolute_src_path) + + assert.NoDirExists(t, dest_dirpath) + assert.NoFileExists(t, f.absolute_dest_path) + + directoryEnsureExist(dest_dirpath) + assert.DirExists(t, dest_dirpath) + + fileCreateEmpty(f.absolute_dest_path) + assert.FileExists(t, f.absolute_dest_path) + + f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID, + fmt.Sprintf("copy-file: destination path %q already exists, not copying anything", f.absolute_dest_path)) + + assert.Error(t, f.run()) +} + + +func TestCmdCopyFileSourceIsDir(t *testing.T) { + f := newCmdCopyFileFixture(t) + defer f.finish(t) + + src_dirpath := filepath.Join(f.temppath, "src_path/to") + dest_dirpath := filepath.Join(f.temppath, "dest_path/to") + + f.absolute_src_path = src_dirpath + f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt") + + directoryEnsureExist(src_dirpath) + assert.DirExists(t, src_dirpath) + + assert.NoDirExists(t, dest_dirpath) + assert.NoFileExists(t, f.absolute_dest_path) + + f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID, + fmt.Sprintf("copy-file: copying %q to %q", f.absolute_src_path, f.absolute_dest_path)) + + f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID, + fmt.Sprintf("copy-file: invalid source file %q: stat %s: Not a regular file", + f.absolute_src_path, f.absolute_src_path)) + + assert.Error(t, f.run()) +} + + +func newCmdCopyFileFixture(t *testing.T) cmdCopyFileFixture { + mockCtrl := gomock.NewController(t) + ce, mocks := testCommandExecutor(t, mockCtrl) + + temppath, err := os.MkdirTemp("", "test-copy-file") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("getcw: %v", err) + } + + if err := os.Chdir(temppath); err != nil { + t.Fatalf("chdir(%s): %v", temppath, err) + } + + return cmdCopyFileFixture{ + mockCtrl: mockCtrl, + ce: ce, + mocks: mocks, + ctx: context.Background(), + temppath: temppath, + cwd: cwd, + } +} + +func (f cmdCopyFileFixture) finish(t *testing.T) { + if err := os.Chdir(f.cwd); err != nil { + t.Fatalf("chdir(%s): %v", f.cwd, err) + } + + os.RemoveAll(f.temppath) + f.mockCtrl.Finish() +} + +func (f cmdCopyFileFixture) run() error { + cmd := api.Command{ + Name: "copy-file", + Parameters: map[string]interface{}{ + "src": f.absolute_src_path, + "dest": f.absolute_dest_path, + }, + } + return f.ce.Run(f.ctx, taskID, cmd) +} + +// Misc utils + func ensureDirExists(dirpath string) { if err := os.MkdirAll(dirpath, fs.ModePerm); err != nil { panic(fmt.Sprintf("unable to create dir %s: %v", dirpath, err)) @@ -278,3 +437,11 @@ func fileCreateEmpty(filename string) { } file.Close() } + +func directoryEnsureExist(dirpath string) { + err := os.MkdirAll(dirpath, 0750) + + if err != nil { + panic(err.Error()) + } +}