Add 'copy-file' command. (#104220)

Initial implementation with some basic tests.

The API command only accept one source and destination path, which must
be absolute. If the destination already exists, it will not be
ovewritten, and an error is generated.

JS job definition code using this new command can easily handle list of
paths and/or relative paths if needed.

Reviewed-on: https://projects.blender.org/studio/flamenco/pulls/104220

internal/worker/command_exe.go

@@ -90,6 +90,7 @@ func NewCommandExecutor(cli CommandLineRunner, listener CommandListener, timeSer
 
 		// file-management
 		"move-directory": ce.cmdMoveDirectory,
+		"copy-file": ce.cmdCopyFile,
 	}
 
 	return ce

internal/worker/command_file_mgmt.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -80,6 +81,77 @@ func (ce *CommandExecutor) cmdMoveDirectory(ctx context.Context, logger zerolog.
 	return ce.moveAndLog(ctx, taskID, cmd.Name, src, dest)
 }
 
+// cmdCopyFiles executes the "copy-file" command.
+// It takes an absolute source and destination file path,
+// and copies the source file to its destination, if possible.
+// Missing directories in destination path are created as needed.
+// If the target path already exists, an error is returned. Destination will not be overwritten.
+func (ce *CommandExecutor) cmdCopyFile(ctx context.Context, logger zerolog.Logger, taskID string, cmd api.Command) error {
+	var src, dest string
+	var ok bool
+
+	logger = logger.With().
+		Interface("command", cmd).
+		Str("src", src).
+		Str("dest", dest).
+		Logger()
+
+	if src, ok = cmdParameter[string](cmd, "src"); !ok || src == "" {
+		msg := "missing or empty 'src' parameter"
+		err := NewParameterMissingError("src", cmd)
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+	if !filepath.IsAbs(src) {
+		msg := fmt.Sprintf("source path %q is not absolute, not copying anything", src)
+		err := NewParameterInvalidError("src", cmd, "path is not absolute")
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+	if !fileExists(src) {
+		msg := fmt.Sprintf("source path %q does not exist, not copying anything", src)
+		err := NewParameterInvalidError("src", cmd, "path does not exist")
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+
+	if dest, ok = cmdParameter[string](cmd, "dest"); !ok || dest == "" {
+		msg := "missing or empty 'dest' parameter"
+		err := NewParameterMissingError("dest", cmd)
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+	if !filepath.IsAbs(dest) {
+		msg := fmt.Sprintf("destination path %q is not absolute, not copying anything", src)
+		err := NewParameterInvalidError("dest", cmd, "path is not absolute")
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+	if fileExists(dest) {
+		msg := fmt.Sprintf("destination path %q already exists, not copying anything", dest)
+		err := NewParameterInvalidError("dest", cmd, "path already exists")
+		return ce.errorLogProcess(ctx, logger, cmd, taskID, err, msg)
+	}
+
+	msg := fmt.Sprintf("copying %q to %q", src, dest)
+	if err := ce.errorLogProcess(ctx, logger, cmd, taskID, nil, msg); err != nil {
+		return err
+	}
+
+	err, logMsg := fileCopy(src, dest)
+	return ce.errorLogProcess(ctx, logger, cmd, taskID, err, logMsg)
+}
+
+func (ce *CommandExecutor) errorLogProcess(ctx context.Context, logger zerolog.Logger, cmd api.Command, taskID string, err error, logMsg string) error {
+	msg := fmt.Sprintf("%s: %s", cmd.Name, logMsg)
+
+	if err != nil {
+		logger.Warn().Msg(msg)
+	} else {
+		logger.Info().Msg(msg)
+	}
+
+	if logErr := ce.listener.LogProduced(ctx, taskID, msg); logErr != nil {
+		return logErr
+	}
+	return err
+}
+
 // moveAndLog renames a file/directory from `src` to `dest`, and logs the moveAndLog.
 // The other parameters are just for logging.
 func (ce *CommandExecutor) moveAndLog(ctx context.Context, taskID, cmdName, src, dest string) error {
@@ -99,6 +171,51 @@ func (ce *CommandExecutor) moveAndLog(ctx context.Context, taskID, cmdName, src,
 	return nil
 }
 
+func fileCopy(src, dest string) (error, string) {
+	src_file, err := os.Open(src)
+	if err != nil {
+		msg := fmt.Sprintf("failed to open source file %q: %v", src, err)
+		return err, msg
+	}
+	defer src_file.Close()
+
+	src_file_stat, err := src_file.Stat()
+	if err != nil {
+		msg := fmt.Sprintf("failed to stat source file %q: %v", src, err)
+		return err, msg
+	}
+
+	if !src_file_stat.Mode().IsRegular() {
+		err := &os.PathError{Op: "stat", Path: src, Err: errors.New("Not a regular file")}
+		msg := fmt.Sprintf("invalid source file %q: %v", src, err)
+		return err, msg
+	}
+
+	dest_dirpath := filepath.Dir(dest)
+	if !fileExists(dest_dirpath) {
+		if err := os.MkdirAll(dest_dirpath, 0750); err != nil {
+			msg := fmt.Sprintf("failed to create directories %q: %v", dest_dirpath, err)
+			return err, msg
+		}
+	}
+
+	dest_file, err := os.Create(dest)
+	if err != nil {
+		msg := fmt.Sprintf("failed to create destination file %q: %v", dest, err)
+		return err, msg
+	}
+	defer dest_file.Close()
+
+	if _, err := io.Copy(dest_file, src_file); err != nil {
+		msg := fmt.Sprintf("failed to copy %q to %q: %v", src, dest, err)
+		return err, msg
+	}
+
+	msg := fmt.Sprintf("copied %q to %q", src, dest)
+	return nil, msg
+}
+
+
 func fileExists(filename string) bool {
 	_, err := os.Stat(filename)
 	return !errors.Is(err, fs.ErrNotExist)

internal/worker/command_file_mgmt_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+// `move-directory` tests.
+
 type cmdMoveDirFixture struct {
 	mockCtrl *gomock.Controller
 	ce       *CommandExecutor
@@ -264,6 +266,163 @@ func (f cmdMoveDirFixture) run() error {
 	return f.ce.Run(f.ctx, taskID, cmd)
 }
 
+// `copy-file` tests.
+
+type cmdCopyFileFixture struct {
+	mockCtrl *gomock.Controller
+	ce       *CommandExecutor
+	mocks    *CommandExecutorMocks
+	ctx      context.Context
+
+	temppath string
+	cwd      string
+
+	absolute_src_path  string
+	absolute_dest_path string
+}
+
+func TestCmdCopyFile(t *testing.T) {
+	f := newCmdCopyFileFixture(t)
+	defer f.finish(t)
+
+	src_dirpath := filepath.Join(f.temppath, "src_path/to")
+	dest_dirpath := filepath.Join(f.temppath, "dest_path/to")
+
+	f.absolute_src_path = filepath.Join(src_dirpath, "file1.txt")
+	f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt")
+
+	directoryEnsureExist(src_dirpath)
+	assert.DirExists(t, src_dirpath)
+
+	fileCreateEmpty(f.absolute_src_path)
+	assert.FileExists(t, f.absolute_src_path)
+
+	assert.NoDirExists(t, dest_dirpath)
+	assert.NoFileExists(t, f.absolute_dest_path)
+
+	f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID,
+		fmt.Sprintf("copy-file: copying %q to %q", f.absolute_src_path, f.absolute_dest_path))
+
+	f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID,
+		fmt.Sprintf("copy-file: copied %q to %q", f.absolute_src_path, f.absolute_dest_path))
+
+	assert.NoError(t, f.run())
+
+	assert.DirExists(t, src_dirpath)
+	assert.DirExists(t, dest_dirpath)
+	assert.FileExists(t, f.absolute_src_path)
+	assert.FileExists(t, f.absolute_dest_path)
+}
+
+func TestCmdCopyFileDestinationExists(t *testing.T) {
+	f := newCmdCopyFileFixture(t)
+	defer f.finish(t)
+
+	src_dirpath := filepath.Join(f.temppath, "src_path/to")
+	dest_dirpath := filepath.Join(f.temppath, "dest_path/to")
+
+	f.absolute_src_path = filepath.Join(src_dirpath, "file1.txt")
+	f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt")
+
+	directoryEnsureExist(src_dirpath)
+	assert.DirExists(t, src_dirpath)
+
+	fileCreateEmpty(f.absolute_src_path)
+	assert.FileExists(t, f.absolute_src_path)
+
+	assert.NoDirExists(t, dest_dirpath)
+	assert.NoFileExists(t, f.absolute_dest_path)
+
+	directoryEnsureExist(dest_dirpath)
+	assert.DirExists(t, dest_dirpath)
+
+	fileCreateEmpty(f.absolute_dest_path)
+	assert.FileExists(t, f.absolute_dest_path)
+
+	f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID,
+		fmt.Sprintf("copy-file: destination path %q already exists, not copying anything", f.absolute_dest_path))
+
+	assert.Error(t, f.run())
+}
+
+
+func TestCmdCopyFileSourceIsDir(t *testing.T) {
+	f := newCmdCopyFileFixture(t)
+	defer f.finish(t)
+
+	src_dirpath := filepath.Join(f.temppath, "src_path/to")
+	dest_dirpath := filepath.Join(f.temppath, "dest_path/to")
+
+	f.absolute_src_path = src_dirpath
+	f.absolute_dest_path = filepath.Join(dest_dirpath, "file2.txt")
+
+	directoryEnsureExist(src_dirpath)
+	assert.DirExists(t, src_dirpath)
+
+	assert.NoDirExists(t, dest_dirpath)
+	assert.NoFileExists(t, f.absolute_dest_path)
+
+	f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID,
+		fmt.Sprintf("copy-file: copying %q to %q", f.absolute_src_path, f.absolute_dest_path))
+
+	f.mocks.listener.EXPECT().LogProduced(gomock.Any(), taskID,
+		fmt.Sprintf("copy-file: invalid source file %q: stat %s: Not a regular file",
+			f.absolute_src_path, f.absolute_src_path))
+
+	assert.Error(t, f.run())
+}
+
+
+func newCmdCopyFileFixture(t *testing.T) cmdCopyFileFixture {
+	mockCtrl := gomock.NewController(t)
+	ce, mocks := testCommandExecutor(t, mockCtrl)
+
+	temppath, err := os.MkdirTemp("", "test-copy-file")
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %v", err)
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("getcw: %v", err)
+	}
+
+	if err := os.Chdir(temppath); err != nil {
+		t.Fatalf("chdir(%s): %v", temppath, err)
+	}
+
+	return cmdCopyFileFixture{
+		mockCtrl: mockCtrl,
+		ce:       ce,
+		mocks:    mocks,
+		ctx:      context.Background(),
+		temppath: temppath,
+		cwd:      cwd,
+	}
+}
+
+func (f cmdCopyFileFixture) finish(t *testing.T) {
+	if err := os.Chdir(f.cwd); err != nil {
+		t.Fatalf("chdir(%s): %v", f.cwd, err)
+	}
+
+	os.RemoveAll(f.temppath)
+	f.mockCtrl.Finish()
+}
+
+func (f cmdCopyFileFixture) run() error {        
+	cmd := api.Command{
+		Name: "copy-file",
+		Parameters: map[string]interface{}{
+			"src": f.absolute_src_path,
+			"dest": f.absolute_dest_path,
+		},
+	}
+	return f.ce.Run(f.ctx, taskID, cmd)
+}
+
+// Misc utils
+
 func ensureDirExists(dirpath string) {
 	if err := os.MkdirAll(dirpath, fs.ModePerm); err != nil {
 		panic(fmt.Sprintf("unable to create dir %s: %v", dirpath, err))
@@ -278,3 +437,11 @@ func fileCreateEmpty(filename string) {
 	}
 	file.Close()
 }
+
+func directoryEnsureExist(dirpath string) {
+	err := os.MkdirAll(dirpath, 0750)
+
+	if err != nil {
+		panic(err.Error())
+	}
+}