diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index 79a8b32c..b2a5eb19 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -22,6 +22,7 @@ package task_state_machine import ( "context" + "fmt" "testing" "github.com/golang/mock/gomock" @@ -81,7 +82,7 @@ func taskWithStatus(jobStatus api.JobStatus, taskStatus api.TaskStatus) *persist } task := persistence.Task{ Model: gorm.Model{ID: 327}, - UUID: "testtask-f474-4e28-aeea-8cbaf2fc96a5", + UUID: "testtask-0001-4e28-aeea-8cbaf2fc96a5", JobID: job.ID, Job: &job, @@ -92,15 +93,130 @@ func taskWithStatus(jobStatus api.JobStatus, taskStatus api.TaskStatus) *persist return &task } -func TestTaskStatusChange(t *testing.T) { +/* taskOfSameJob() creates a task of a certain status, on the same job as the given task. */ +func taskOfSameJob(task *persistence.Task, taskStatus api.TaskStatus) *persistence.Task { + newTaskID := task.ID + 1 + return &persistence.Task{ + Model: gorm.Model{ID: newTaskID}, + UUID: fmt.Sprintf("testtask-%04d-4e28-aeea-8cbaf2fc96a5", newTaskID), + JobID: task.JobID, + Job: task.Job, + Status: taskStatus, + } +} + +func taskStateMachineTestFixtures(t *testing.T) (*gomock.Controller, context.Context, *StateMachine, *StateMachineMocks) { mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() ctx := context.Background() - sm, mocks := mockedTaskStateMachine(mockCtrl) + return mockCtrl, ctx, sm, mocks +} +// In the comments below, "T" indicates the performed task status change, and +// "J" the expected resulting job status change. + +func TestTaskStatusChangeQueuedToActive(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // T: queued > active --> J: queued > active task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued) mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusActive) mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive) assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusActive)) } + +func TestTaskStatusChangeActiveToCompleted(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // Job has three tasks. + task := taskWithStatus(api.JobStatusActive, api.TaskStatusActive) + task2 := taskOfSameJob(task, api.TaskStatusActive) + task3 := taskOfSameJob(task, api.TaskStatusActive) + + // First task completing: T: active > completed --> J: active > active + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusCompleted) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(1, 3, nil) // 1 of 3 complete. + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCompleted)) + + // Second task hickup: T: active > soft-failed --> J: active > active + mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusSoftFailed) + assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusSoftFailed)) + + // Second task completing: T: soft-failed > completed --> J: active > active + mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusCompleted) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(2, 3, nil) // 2 of 3 complete. + assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCompleted)) + + // Third task completing: T: active > completed --> J: active > completed + mocks.expectSaveTaskWithStatus(t, task3, api.TaskStatusCompleted) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(3, 3, nil) // 3 of 3 complete. + mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusCompleted) + assert.NoError(t, sm.TaskStatusChange(ctx, task3, api.TaskStatusCompleted)) +} + +func TestTaskStatusChangeQueuedToFailed(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // T: queued > failed (1% task failure) --> J: queued > active + task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued) + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusFailed) + mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(1, 100, nil) // 1 out of 100 failed. + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed)) +} + +func TestTaskStatusChangeActiveToFailedFailJob(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // T: active > failed (10% task failure) --> J: active > failed + task := taskWithStatus(api.JobStatusActive, api.TaskStatusActive) + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusFailed) + mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusFailed) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(10, 100, nil) // 10 out of 100 failed. + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed)) +} + +func TestTaskStatusChangeRequeueOnCompletedJob(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // T: completed > queued --> J: completed > queued + task := taskWithStatus(api.JobStatusCompleted, api.TaskStatusCompleted) + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusQueued) + mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusQueued) + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusQueued)) +} + +func TestTaskStatusChangeCancelSingleTask(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + task := taskWithStatus(api.JobStatusCancelRequested, api.TaskStatusCancelRequested) + task2 := taskOfSameJob(task, api.TaskStatusCancelRequested) + job := task.Job + + // T1: cancel-requested > cancelled --> J: cancel-requested > cancel-requested + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatusCanceled) + mocks.persist.EXPECT().JobHasTasksInStatus(ctx, job, api.TaskStatusCancelRequested).Return(true, nil) + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCanceled)) + + // T2: cancel-requested > cancelled --> J: cancel-requested > canceled + mocks.expectSaveTaskWithStatus(t, task2, api.TaskStatusCanceled) + mocks.persist.EXPECT().JobHasTasksInStatus(ctx, job, api.TaskStatusCancelRequested).Return(false, nil) + mocks.expectSaveJobWithStatus(t, job, api.JobStatusCanceled) + assert.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCanceled)) +} + +func TestTaskStatusChangeUnknownStatus(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + // T: queued > borked --> saved to DB but otherwise ignored + task := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued) + mocks.expectSaveTaskWithStatus(t, task, api.TaskStatus("borked")) + assert.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatus("borked"))) +}