Manager: more task state change tests

internal/manager/task_state_machine/task_state_machine_test.go

@@ -22,6 +22,7 @@ package task_state_machine
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -81,7 +82,7 @@ func taskWithStatus(jobStatus api.JobStatus, taskStatus api.TaskStatus) *persist
 	}
 	task := persistence.Task{
 		Model: gorm.Model{ID: 327},
-		UUID:  "testtask-f474-4e28-aeea-8cbaf2fc96a5",
+		UUID:  "testtask-0001-4e28-aeea-8cbaf2fc96a5",
 
 		JobID: job.ID,
 		Job:   &job,
@@ -92,15 +93,130 @@ func taskWithStatus(jobStatus api.JobStatus, taskStatus api.TaskStatus) *persist
 	return &task
 }
 
-func TestTaskStatusChange(t *testing.T) {
+/* taskOfSameJob() creates a task of a certain status, on the same job as the given task. */
+func taskOfSameJob(task *persistence.Task, taskStatus api.TaskStatus) *persistence.Task {
+	newTaskID := task.ID + 1
+	return &persistence.Task{
+		Model:  gorm.Model{ID: newTaskID},
+		UUID:   fmt.Sprintf("testtask-%04d-4e28-aeea-8cbaf2fc96a5", newTaskID),
+		JobID:  task.JobID,
+		Job:    task.Job,
+		Status: taskStatus,
+	}
+}
+
+func taskStateMachineTestFixtures(t *testing.T) (*gomock.Controller, context.Context, *StateMachine, *StateMachineMocks) {
 	mockCtrl := gomock.NewController(t)
-	defer mockCtrl.Finish()
 	ctx := context.Background()
-
 	sm, mocks := mockedTaskStateMachine(mockCtrl)
+	return mockCtrl, ctx, sm, mocks
+}
 
+// In the comments below, "T" indicates the performed task status change, and
+// "J" the expected resulting job status change.
+
+func TestTaskStatusChangeQueuedToActive(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// T: queued > active  --> J: queued > active
 	task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued)
 	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusActive)
 	mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive)
 	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusActive))
 }
+
+func TestTaskStatusChangeActiveToCompleted(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// Job has three tasks.
+	task := taskWithStatus(api.JobStatusActive, api.TaskStatusActive)
+	task2 := taskOfSameJob(task, api.TaskStatusActive)
+	task3 := taskOfSameJob(task, api.TaskStatusActive)
+
+	// First task completing: T: active > completed --> J: active > active
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusCompleted)
+	mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(1, 3, nil) // 1 of 3 complete.
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCompleted))
+
+	// Second task hickup: T: active > soft-failed --> J: active > active
+	mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusSoftFailed)
+	assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusSoftFailed))
+
+	// Second task completing: T: soft-failed > completed --> J: active > active
+	mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusCompleted)
+	mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(2, 3, nil) // 2 of 3 complete.
+	assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCompleted))
+
+	// Third task completing: T: active > completed --> J: active > completed
+	mocks.expectSaveTaskWithStatus(t, task3, api.TaskStatusCompleted)
+	mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(3, 3, nil) // 3 of 3 complete.
+	mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusCompleted)
+	assert.NoError(t, sm.TaskStatusChange(ctx, task3, api.TaskStatusCompleted))
+}
+
+func TestTaskStatusChangeQueuedToFailed(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// T: queued > failed (1% task failure) --> J: queued > active
+	task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued)
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusFailed)
+	mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive)
+	mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(1, 100, nil) // 1 out of 100 failed.
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed))
+}
+
+func TestTaskStatusChangeActiveToFailedFailJob(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// T: active > failed (10% task failure) --> J: active > failed
+	task := taskWithStatus(api.JobStatusActive, api.TaskStatusActive)
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusFailed)
+	mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusFailed)
+	mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(10, 100, nil) // 10 out of 100 failed.
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed))
+}
+
+func TestTaskStatusChangeRequeueOnCompletedJob(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// T: completed > queued --> J: completed > queued
+	task := taskWithStatus(api.JobStatusCompleted, api.TaskStatusCompleted)
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusQueued)
+	mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusQueued)
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusQueued))
+}
+
+func TestTaskStatusChangeCancelSingleTask(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	task := taskWithStatus(api.JobStatusCancelRequested, api.TaskStatusCancelRequested)
+	task2 := taskOfSameJob(task, api.TaskStatusCancelRequested)
+	job := task.Job
+
+	// T1: cancel-requested > cancelled --> J: cancel-requested > cancel-requested
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusCanceled)
+	mocks.persist.EXPECT().JobHasTasksInStatus(ctx, job, api.TaskStatusCancelRequested).Return(true, nil)
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCanceled))
+
+	// T2: cancel-requested > cancelled --> J: cancel-requested > canceled
+	mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusCanceled)
+	mocks.persist.EXPECT().JobHasTasksInStatus(ctx, job, api.TaskStatusCancelRequested).Return(false, nil)
+	mocks.expectSaveJobWithStatus(t, job, api.JobStatusCanceled)
+	assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCanceled))
+}
+
+func TestTaskStatusChangeUnknownStatus(t *testing.T) {
+	mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t)
+	defer mockCtrl.Finish()
+
+	// T: queued > borked --> saved to DB but otherwise ignored
+	task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued)
+	mocks.expectSaveTaskWithStatus(t, task, api.TaskStatus("borked"))
+	assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatus("borked")))
+}