diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go index 651b6a27..8196b6bf 100644 --- a/internal/manager/api_impl/api_impl.go +++ b/internal/manager/api_impl/api_impl.go @@ -103,6 +103,7 @@ type JobCompiler interface { type LogStorage interface { Write(logger zerolog.Logger, jobID, taskID string, logText string) error RotateFile(logger zerolog.Logger, jobID, taskID string) + Tail(jobID, taskID string) (string, error) } type ConfigService interface { diff --git a/internal/manager/api_impl/jobs.go b/internal/manager/api_impl/jobs.go index e74eb766..6271c6ef 100644 --- a/internal/manager/api_impl/jobs.go +++ b/internal/manager/api_impl/jobs.go @@ -12,6 +12,7 @@ import ( "git.blender.org/flamenco/internal/manager/job_compilers" "git.blender.org/flamenco/internal/manager/persistence" "git.blender.org/flamenco/internal/manager/webupdates" + "git.blender.org/flamenco/internal/uuid" "git.blender.org/flamenco/pkg/api" ) @@ -178,6 +179,35 @@ func (f *Flamenco) SetTaskStatus(e echo.Context, taskID string) error { return e.NoContent(http.StatusNoContent) } +func (f *Flamenco) FetchTaskLogTail(e echo.Context, taskID string) error { + logger := requestLogger(e) + ctx := e.Request().Context() + + logger = logger.With().Str("task", taskID).Logger() + if !uuid.IsValid(taskID) { + logger.Warn().Msg("fetchTaskLogTail: bad task ID ") + return sendAPIError(e, http.StatusBadRequest, "bad task ID") + } + + dbTask, err := f.persist.FetchTask(ctx, taskID) + if err != nil { + if errors.Is(err, persistence.ErrTaskNotFound) { + return sendAPIError(e, http.StatusNotFound, "no such task") + } + logger.Error().Err(err).Msg("error fetching task") + return sendAPIError(e, http.StatusInternalServerError, "error fetching task: %v", err) + } + + tail, err := f.logStorage.Tail(dbTask.Job.UUID, taskID) + if err != nil { + logger.Error().Err(err).Msg("unable to fetch task log tail") + return sendAPIError(e, http.StatusInternalServerError, "error fetching task log tail: %v", err) + } + + logger.Debug().Msg("fetched task tail") + return e.String(http.StatusOK, tail) +} + func jobDBtoAPI(dbJob *persistence.Job) api.Job { apiJob := api.Job{ SubmittedJob: api.SubmittedJob{ diff --git a/internal/manager/api_impl/mocks/api_impl_mock.gen.go b/internal/manager/api_impl/mocks/api_impl_mock.gen.go index 3e7bf926..b9792088 100644 --- a/internal/manager/api_impl/mocks/api_impl_mock.gen.go +++ b/internal/manager/api_impl/mocks/api_impl_mock.gen.go @@ -378,6 +378,21 @@ func (mr *MockLogStorageMockRecorder) RotateFile(arg0, arg1, arg2 interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RotateFile", reflect.TypeOf((*MockLogStorage)(nil).RotateFile), arg0, arg1, arg2) } +// Tail mocks base method. +func (m *MockLogStorage) Tail(arg0, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tail", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Tail indicates an expected call of Tail. +func (mr *MockLogStorageMockRecorder) Tail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tail", reflect.TypeOf((*MockLogStorage)(nil).Tail), arg0, arg1) +} + // Write mocks base method. func (m *MockLogStorage) Write(arg0 zerolog.Logger, arg1, arg2, arg3 string) error { m.ctrl.T.Helper() diff --git a/internal/manager/task_logs/task_logs.go b/internal/manager/task_logs/task_logs.go index fe672983..9a7966cb 100644 --- a/internal/manager/task_logs/task_logs.go +++ b/internal/manager/task_logs/task_logs.go @@ -3,7 +3,9 @@ package task_logs // SPDX-License-Identifier: GPL-3.0-or-later import ( + "bytes" "fmt" + "io" "os" "path" "path/filepath" @@ -12,6 +14,11 @@ import ( "github.com/rs/zerolog/log" ) +const ( + // tailSize is the maximum number of bytes read by the Tail() function. + tailSize int64 = 2048 +) + // Storage can write data to task logs, rotate logs, etc. type Storage struct { BasePath string // Directory where task logs are stored. @@ -105,3 +112,54 @@ func (s *Storage) filepath(jobID, taskID string) string { filename := fmt.Sprintf("task-%v.txt", taskID) return path.Join(dirpath, filename) } + +// Tail reads the final few lines of a task log. +func (s *Storage) Tail(jobID, taskID string) (string, error) { + filepath := s.filepath(jobID, taskID) + + file, err := os.Open(filepath) + if err != nil { + return "", fmt.Errorf("unable to open log file for reading: %w", err) + } + + fileSize, err := file.Seek(0, os.SEEK_END) + if err != nil { + return "", fmt.Errorf("unable to seek to end of log file: %w", err) + } + + // Number of bytes to read. + var ( + buffer []byte + numBytes int + ) + if fileSize <= tailSize { + // The file is small, just read all of it. + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return "", fmt.Errorf("unable to seek to start of log file: %w", err) + } + buffer, err = io.ReadAll(file) + } else { + // Read the last 'tailSize' number of bytes. + _, err = file.Seek(-tailSize, os.SEEK_END) + if err != nil { + return "", fmt.Errorf("unable to seek in log file: %w", err) + } + buffer = make([]byte, tailSize) + numBytes, err = file.Read(buffer) + + // Try to remove contents up to the first newline, as it's very likely we just + // seeked into the middle of a line. + firstNewline := bytes.IndexByte(buffer, byte('\n')) + if 0 <= firstNewline && firstNewline < numBytes-1 { + buffer = buffer[firstNewline+1:] + } else { + // The file consists of a single line of text; don't strip the first line. + } + } + if err != nil { + return "", fmt.Errorf("error reading log file: %w", err) + } + + return string(buffer), nil +} diff --git a/internal/manager/task_logs/task_logs_test.go b/internal/manager/task_logs/task_logs_test.go index 8df591f4..12d181d4 100644 --- a/internal/manager/task_logs/task_logs_test.go +++ b/internal/manager/task_logs/task_logs_test.go @@ -3,6 +3,7 @@ package task_logs // SPDX-License-Identifier: GPL-3.0-or-later import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -75,3 +76,60 @@ func TestLogRotation(t *testing.T) { _, err = os.Stat(filename) assert.True(t, os.IsNotExist(err)) } + +func TestLogTail(t *testing.T) { + s := tempStorage() + defer os.RemoveAll(s.BasePath) + + jobID := "25c5a51c-e0dd-44f7-9f87-74f3d1fbbd8c" + taskID := "20ff9d06-53ec-4019-9e2e-1774f05f170a" + + err := s.Write(zerolog.Nop(), jobID, taskID, "Just a single line") + assert.NoError(t, err) + contents, err := s.Tail(jobID, taskID) + assert.NoError(t, err) + assert.Equal(t, "Just a single line\n", string(contents)) + + // A short file shouldn't do any line stripping. + err = s.Write(zerolog.Nop(), jobID, taskID, "And another line!") + assert.NoError(t, err) + contents, err = s.Tail(jobID, taskID) + assert.NoError(t, err) + assert.Equal(t, "Just a single line\nAnd another line!\n", string(contents)) + + bigString := "" + for lineNum := 1; lineNum < 1000; lineNum++ { + bigString += fmt.Sprintf("This is line #%d\n", lineNum) + } + err = s.Write(zerolog.Nop(), jobID, taskID, bigString) + assert.NoError(t, err) + + contents, err = s.Tail(jobID, taskID) + assert.NoError(t, err) + assert.Equal(t, + "This is line #887\nThis is line #888\nThis is line #889\nThis is line #890\nThis is line #891\n"+ + "This is line #892\nThis is line #893\nThis is line #894\nThis is line #895\nThis is line #896\n"+ + "This is line #897\nThis is line #898\nThis is line #899\nThis is line #900\nThis is line #901\n"+ + "This is line #902\nThis is line #903\nThis is line #904\nThis is line #905\nThis is line #906\n"+ + "This is line #907\nThis is line #908\nThis is line #909\nThis is line #910\nThis is line #911\n"+ + "This is line #912\nThis is line #913\nThis is line #914\nThis is line #915\nThis is line #916\n"+ + "This is line #917\nThis is line #918\nThis is line #919\nThis is line #920\nThis is line #921\n"+ + "This is line #922\nThis is line #923\nThis is line #924\nThis is line #925\nThis is line #926\n"+ + "This is line #927\nThis is line #928\nThis is line #929\nThis is line #930\nThis is line #931\n"+ + "This is line #932\nThis is line #933\nThis is line #934\nThis is line #935\nThis is line #936\n"+ + "This is line #937\nThis is line #938\nThis is line #939\nThis is line #940\nThis is line #941\n"+ + "This is line #942\nThis is line #943\nThis is line #944\nThis is line #945\nThis is line #946\n"+ + "This is line #947\nThis is line #948\nThis is line #949\nThis is line #950\nThis is line #951\n"+ + "This is line #952\nThis is line #953\nThis is line #954\nThis is line #955\nThis is line #956\n"+ + "This is line #957\nThis is line #958\nThis is line #959\nThis is line #960\nThis is line #961\n"+ + "This is line #962\nThis is line #963\nThis is line #964\nThis is line #965\nThis is line #966\n"+ + "This is line #967\nThis is line #968\nThis is line #969\nThis is line #970\nThis is line #971\n"+ + "This is line #972\nThis is line #973\nThis is line #974\nThis is line #975\nThis is line #976\n"+ + "This is line #977\nThis is line #978\nThis is line #979\nThis is line #980\nThis is line #981\n"+ + "This is line #982\nThis is line #983\nThis is line #984\nThis is line #985\nThis is line #986\n"+ + "This is line #987\nThis is line #988\nThis is line #989\nThis is line #990\nThis is line #991\n"+ + "This is line #992\nThis is line #993\nThis is line #994\nThis is line #995\nThis is line #996\n"+ + "This is line #997\nThis is line #998\nThis is line #999\n", + string(contents), + ) +}