From 12481a47e7b85572aea63e02d20220ba73c87795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 21 Feb 2022 18:09:02 +0100 Subject: [PATCH] Start of configuration/settings framework, including variable replacement --- cmd/flamenco-manager-poc/main.go | 7 +- internal/manager/api_impl/api_impl.go | 10 +- .../api_impl/mocks/api_impl_mock.gen.go | 39 +- .../api_impl/test-flamenco-manager.yaml | 60 ++ internal/manager/api_impl/test_support.go | 5 +- internal/manager/api_impl/varrepl.go | 59 ++ internal/manager/api_impl/varrepl_test.go | 113 +++ internal/manager/config/service.go | 47 ++ internal/manager/config/settings.go | 650 ++++++++++++++++++ internal/manager/config/settings_test.go | 68 ++ 10 files changed, 1053 insertions(+), 5 deletions(-) create mode 100644 internal/manager/api_impl/test-flamenco-manager.yaml create mode 100644 internal/manager/api_impl/varrepl.go create mode 100644 internal/manager/api_impl/varrepl_test.go create mode 100644 internal/manager/config/service.go create mode 100644 internal/manager/config/settings.go create mode 100644 internal/manager/config/settings_test.go diff --git a/cmd/flamenco-manager-poc/main.go b/cmd/flamenco-manager-poc/main.go index 401e864a..5f0afcb1 100644 --- a/cmd/flamenco-manager-poc/main.go +++ b/cmd/flamenco-manager-poc/main.go @@ -37,6 +37,7 @@ import ( "gitlab.com/blender/flamenco-ng-poc/internal/appinfo" "gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/config" "gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers" "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence" "gitlab.com/blender/flamenco-ng-poc/internal/manager/swagger_ui" @@ -58,6 +59,10 @@ func main() { if cliArgs.version { return } + + // Load configuration. + configService := config.NewService() + if cliArgs.initDB { log.Info().Msg("creating databases") err := persistence.InitialSetup() @@ -88,7 +93,7 @@ func main() { log.Fatal().Err(err).Msg("error loading job compilers") } logStorage := task_logs.NewStorage("./task-logs") // TODO: load job storage path from configuration. - flamenco := api_impl.NewFlamenco(compiler, persist, logStorage) + flamenco := api_impl.NewFlamenco(compiler, persist, logStorage, configService) e := buildWebService(flamenco, persist) // Start the web server. diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go index 5ea37a53..c497f7a3 100644 --- a/internal/manager/api_impl/api_impl.go +++ b/internal/manager/api_impl/api_impl.go @@ -36,10 +36,11 @@ type Flamenco struct { jobCompiler JobCompiler persist PersistenceService logStorage LogStorage + config ConfigService } // Generate mock implementations of these interfaces. -//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage +//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage,ConfigService type PersistenceService interface { StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error @@ -68,14 +69,19 @@ type LogStorage interface { RotateFile(logger zerolog.Logger, jobID, taskID string) } +type ConfigService interface { + ExpandVariables(valueToExpand, audience, platform string) string +} + var _ api.ServerInterface = (*Flamenco)(nil) // NewFlamenco creates a new Flamenco service, using the given JobCompiler. -func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage) *Flamenco { +func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage, cs ConfigService) *Flamenco { return &Flamenco{ jobCompiler: jc, persist: jps, logStorage: ls, + config: cs, } } diff --git a/internal/manager/api_impl/mocks/api_impl_mock.gen.go b/internal/manager/api_impl/mocks/api_impl_mock.gen.go index d505e61d..c6ab90f1 100644 --- a/internal/manager/api_impl/mocks/api_impl_mock.gen.go +++ b/internal/manager/api_impl/mocks/api_impl_mock.gen.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage) +// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage,ConfigService) // Package mocks is a generated GoMock package. package mocks @@ -268,3 +268,40 @@ func (mr *MockLogStorageMockRecorder) Write(arg0, arg1, arg2, arg3 interface{}) mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLogStorage)(nil).Write), arg0, arg1, arg2, arg3) } + +// MockConfigService is a mock of ConfigService interface. +type MockConfigService struct { + ctrl *gomock.Controller + recorder *MockConfigServiceMockRecorder +} + +// MockConfigServiceMockRecorder is the mock recorder for MockConfigService. +type MockConfigServiceMockRecorder struct { + mock *MockConfigService +} + +// NewMockConfigService creates a new mock instance. +func NewMockConfigService(ctrl *gomock.Controller) *MockConfigService { + mock := &MockConfigService{ctrl: ctrl} + mock.recorder = &MockConfigServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConfigService) EXPECT() *MockConfigServiceMockRecorder { + return m.recorder +} + +// ExpandVariables mocks base method. +func (m *MockConfigService) ExpandVariables(arg0, arg1, arg2 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExpandVariables", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + return ret0 +} + +// ExpandVariables indicates an expected call of ExpandVariables. +func (mr *MockConfigServiceMockRecorder) ExpandVariables(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpandVariables", reflect.TypeOf((*MockConfigService)(nil).ExpandVariables), arg0, arg1, arg2) +} diff --git a/internal/manager/api_impl/test-flamenco-manager.yaml b/internal/manager/api_impl/test-flamenco-manager.yaml new file mode 100644 index 00000000..65a596d6 --- /dev/null +++ b/internal/manager/api_impl/test-flamenco-manager.yaml @@ -0,0 +1,60 @@ +# This file is loaded by unit tests. +_meta: + version: 3 +mode: develop +listen: '[::0]:8083' +own_url: http://192.168.3.108:8083/ +flamenco: http://localhost:51234/ +manager_id: 5852bc5198377351f95d103e +manager_secret: SRVwA7wAxPRfudvqTDOLXwPn1cDRIlADz5Ef9kHk7d52Us + +task_logs_path: /tmp/flamenco-unittests +blacklist_threshold: 3 + +shaman: + enabled: false + +variables: + blender: + direction: oneway + values: + - audience: users + platform: linux + value: /linux/path/to/blender + - audience: workers + platform: linux + value: /opt/myblenderbuild/blender + - platform: windows + value: 'c:/temp/blender.exe' + - platform: darwin + value: /opt/myblenderbuild/blender + ffmpeg: + direction: oneway + values: + - platform: linux + value: /usr/bin/ffmpeg + - platform: windows + value: xxx + - platform: darwin + value: xxx + render_long: + direction: twoway + values: + - platform: windows + value: s:/flamenco/render/long + - platform: linux + value: /shared/flamenco/render/long + - platform: darwin + value: /Volume/shared/flamenco/render/long + + job_storage: + direction: twoway + values: + - platform: windows + value: s:/flamenco/jobs + - platform: linux + value: /shared/flamenco/jobs + - platform: darwin + value: /Volume/shared/flamenco/jobs + - platform: autumn + value: hey diff --git a/internal/manager/api_impl/test_support.go b/internal/manager/api_impl/test_support.go index 9d2868b6..571c335d 100644 --- a/internal/manager/api_impl/test_support.go +++ b/internal/manager/api_impl/test_support.go @@ -39,18 +39,21 @@ type mockedFlamenco struct { flamenco *Flamenco jobCompiler *mocks.MockJobCompiler persistence *mocks.MockPersistenceService + config *mocks.MockConfigService } func newMockedFlamenco(mockCtrl *gomock.Controller) mockedFlamenco { jc := mocks.NewMockJobCompiler(mockCtrl) ps := mocks.NewMockPersistenceService(mockCtrl) ls := mocks.NewMockLogStorage(mockCtrl) - f := NewFlamenco(jc, ps, ls) + cs := mocks.NewMockConfigService(mockCtrl) + f := NewFlamenco(jc, ps, ls, cs) return mockedFlamenco{ flamenco: f, jobCompiler: jc, persistence: ps, + config: cs, } } diff --git a/internal/manager/api_impl/varrepl.go b/internal/manager/api_impl/varrepl.go new file mode 100644 index 00000000..bc5816fc --- /dev/null +++ b/internal/manager/api_impl/varrepl.go @@ -0,0 +1,59 @@ +package api_impl + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "reflect" + + "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence" +) + +var stringType = reflect.TypeOf("somestring") + +type VariableReplacer interface { + ExpandVariables(valueToExpand, audience, platform string) string +} + +// replaceTaskVariables performs variable replacement for worker tasks. +func replaceTaskVariables(replacer VariableReplacer, task persistence.Task, worker persistence.Worker) persistence.Task { + repl := func(value string) string { + return replacer.ExpandVariables(value, "workers", worker.Platform) + } + + for cmdIndex, cmd := range task.Commands { + for key, value := range cmd.Parameters { + switch v := value.(type) { + case string: + task.Commands[cmdIndex].Parameters[key] = repl(v) + case []string: + replaced := make([]string, len(v)) + for idx := range v { + replaced[idx] = repl(v[idx]) + } + task.Commands[cmdIndex].Parameters[key] = replaced + default: + continue + } + } + } + + return task +} diff --git a/internal/manager/api_impl/varrepl_test.go b/internal/manager/api_impl/varrepl_test.go new file mode 100644 index 00000000..7ec53513 --- /dev/null +++ b/internal/manager/api_impl/varrepl_test.go @@ -0,0 +1,113 @@ +package api_impl + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/config" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence" +) + +func varreplTestTask() persistence.Task { + return persistence.Task{ + Commands: []persistence.Command{ + {Name: "echo", Parameters: persistence.StringInterfaceMap{ + "message": "Running Blender from {blender} {blender}"}}, + {Name: "sleep", Parameters: persistence.StringInterfaceMap{ + "{blender}": 3}}, + { + Name: "blender_render", + Parameters: persistence.StringInterfaceMap{ + "filepath": "{job_storage}/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend", + "exe": "{blender}", + "otherpath": "{hey}/haha", + "frames": "47", + "cycles_chunk": 1.0, + "args": []string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"}, + }, + }, + }, + } +} + +func TestReplaceVariables(t *testing.T) { + worker := persistence.Worker{Platform: "linux"} + task := varreplTestTask() + conf := config.GetTestConfig() + replacedTask := replaceTaskVariables(&conf, task, worker) + + // Single string value. + assert.Equal(t, + "/opt/myblenderbuild/blender", + replacedTask.Commands[2].Parameters["exe"], + ) + + // Array value. + assert.Equal(t, + []string{"--render-out", "/shared/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"}, + replacedTask.Commands[2].Parameters["args"], + ) + + // Substitution should happen as often as needed. + assert.Equal(t, + "Running Blender from /opt/myblenderbuild/blender /opt/myblenderbuild/blender", + replacedTask.Commands[0].Parameters["message"], + ) + + // No substitution should happen on keys, just on values. + assert.Equal(t, 3, replacedTask.Commands[1].Parameters["{blender}"]) +} + +func TestReplacePathsWindows(t *testing.T) { + worker := persistence.Worker{Platform: "windows"} + task := varreplTestTask() + conf := config.GetTestConfig() + replacedTask := replaceTaskVariables(&conf, task, worker) + + assert.Equal(t, + "s:/flamenco/jobs/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend", + replacedTask.Commands[2].Parameters["filepath"], + ) + assert.Equal(t, + []string{"--render-out", "s:/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"}, + replacedTask.Commands[2].Parameters["args"], + ) + assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"]) +} + +func TestReplacePathsUnknownOS(t *testing.T) { + worker := persistence.Worker{Platform: "autumn"} + task := varreplTestTask() + conf := config.GetTestConfig() + replacedTask := replaceTaskVariables(&conf, task, worker) + + assert.Equal(t, + "hey/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend", + replacedTask.Commands[2].Parameters["filepath"], + ) + assert.Equal(t, + []string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"}, + replacedTask.Commands[2].Parameters["args"], + ) + assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"]) +} diff --git a/internal/manager/config/service.go b/internal/manager/config/service.go new file mode 100644 index 00000000..8a4f0d9b --- /dev/null +++ b/internal/manager/config/service.go @@ -0,0 +1,47 @@ +package config + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +// Service provides access to Flamenco Manager configuration. +type Service struct { + config Conf +} + +func NewService() *Service { + return &Service{} +} + +func (s *Service) Load() error { + config, err := getConf() + if err != nil { + return err + } + s.config = config + return nil +} + +func (s *Service) ExpandVariables(valueToExpand, audience, platform string) string { + return s.config.ExpandVariables(valueToExpand, audience, platform) +} + +func (s *Service) Get() *Conf { + return &s.config +} diff --git a/internal/manager/config/settings.go b/internal/manager/config/settings.go new file mode 100644 index 00000000..9ebd8fcf --- /dev/null +++ b/internal/manager/config/settings.go @@ -0,0 +1,650 @@ +package config + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "errors" + "fmt" + "io" + "os" + "path" + "runtime" + "strings" + "time" + + "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v2" + + "gitlab.com/blender/flamenco-ng-poc/pkg/api" +) + +const ( + configFilename = "flamenco-manager.yaml" + + latestConfigVersion = 3 + + // // relative to the Flamenco Server Base URL: + // jwtPublicKeysRelativeURL = "api/flamenco/jwt/public-keys" + + defaultShamanFilestorePath = "/shared/flamenco/file-store" + defaultJobStorage = "/shared/flamenco/jobs" +) + +var ( + // ErrMissingVariablePlatform is returned when a variable doesn't declare any valid platform for a certain value. + ErrMissingVariablePlatform = errors.New("variable's value is missing platform declaration") + // ErrBadDirection is returned when a direction doesn't match "oneway" or "twoway" + ErrBadDirection = errors.New("variable's direction is invalid") + + // Valid values for the "mode" config variable. + validModes = map[string]bool{ + "develop": true, + "production": true, + } + + // Valid values for the "audience" tag of a ConfV2 variable. + validAudiences = map[string]bool{ + "all": true, + "workers": true, + "users": true, + } + + // The default configuration, use DefaultConfig() to obtain a copy. + defaultConfig = Conf{ + Base: Base{ + Meta: ConfMeta{Version: latestConfigVersion}, + + Mode: "production", + ManagerName: "Flamenco Manager", + Listen: ":8080", + ListenHTTPS: ":8433", + DatabaseDSN: "host=localhost user=flamenco password=flamenco dbname=flamenco TimeZone=Europe/Amsterdam", + TaskLogsPath: "./task-logs", + // DownloadTaskSleep: 10 * time.Minute, + // DownloadTaskRecheckThrottle: 10 * time.Second, + // TaskUpdatePushMaxInterval: 5 * time.Second, + // TaskUpdatePushMaxCount: 3000, + // CancelTaskFetchInterval: 10 * time.Second, + ActiveTaskTimeoutInterval: 10 * time.Minute, + ActiveWorkerTimeoutInterval: 1 * time.Minute, + // FlamencoStr: defaultServerURL, + + // // Days are assumed to be 24 hours long. This is not exactly accurate, but should + // // be accurate enough for this type of cleanup. + // TaskCleanupMaxAge: 14 * 24 * time.Hour, + SSDPDiscovery: false, // Only enable after SSDP discovery has been improved (avoid finding printers). + SSDPDeviceUUID: "64ad4c21-6042-4378-9cdf-478f88b4f990", // UUID specific for Flamenco v3. + + BlacklistThreshold: 3, + TaskFailAfterSoftFailCount: 3, + + WorkerCleanupStatus: []string{string(api.WorkerStatusOffline)}, + + TestTasks: TestTasks{ + BlenderRender: BlenderRenderConfig{ + JobStorage: "{job_storage}/test-jobs", + RenderOutput: "{render}/test-renders", + }, + }, + + Shaman: ShamanConfig{ + Enabled: true, + FileStorePath: defaultShamanFilestorePath, + GarbageCollect: ShamanGarbageCollect{ + Period: 24 * time.Hour, + MaxAge: 31 * 24 * time.Hour, + ExtraCheckoutDirs: []string{}, + }, + }, + + // JWT: jwtauth.Config{ + // DownloadKeysInterval: 1 * time.Hour, + // }, + }, + + Variables: map[string]Variable{ + "blender": { + Direction: "oneway", + Values: VariableValues{ + VariableValue{Platform: "linux", Value: "/linux/path/to/blender --factory-startup --background"}, + VariableValue{Platform: "windows", Value: "C:/windows/path/to/blender.exe --factory-startup --background"}, + VariableValue{Platform: "darwin", Value: "/Volumes/Applications/Blender/blender --factory-startup --background"}, + }, + }, + "ffmpeg": { + Direction: "oneway", + Values: VariableValues{ + VariableValue{Platform: "linux", Value: "/usr/bin/ffmpeg"}, + VariableValue{Platform: "windows", Value: "C:/windows/path/to/ffmpeg.exe"}, + VariableValue{Platform: "darwin", Value: "/Volumes/Applications/FFmpeg/ffmpeg"}, + }, + }, + "job_storage": { + Direction: "twoway", + Values: VariableValues{ + VariableValue{Platform: "linux", Value: "/shared/flamenco/jobs"}, + VariableValue{Platform: "windows", Value: "S:/flamenco/jobs"}, + VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/jobs"}, + }, + }, + "render": { + Direction: "twoway", + Values: VariableValues{ + VariableValue{Platform: "linux", Value: "/shared/flamenco/render"}, + VariableValue{Platform: "windows", Value: "S:/flamenco/render"}, + VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/render"}, + }, + }, + }, + } +) + +// BlenderRenderConfig represents the configuration required for a test render. +type BlenderRenderConfig struct { + JobStorage string `yaml:"job_storage"` + RenderOutput string `yaml:"render_output"` +} + +// TestTasks represents the 'test_tasks' key in the Manager's configuration file. +type TestTasks struct { + BlenderRender BlenderRenderConfig `yaml:"test_blender_render"` +} + +// ConfMeta contains configuration file metadata. +type ConfMeta struct { + // Version of the config file structure. + Version int `yaml:"version"` +} + +// Base contains those settings that are shared by all configuration versions. +type Base struct { + Meta ConfMeta `yaml:"_meta"` + + Mode string `yaml:"mode"` // either "develop" or "production" + ManagerName string `yaml:"manager_name"` + DatabaseDSN string `yaml:"database_url"` + TaskLogsPath string `yaml:"task_logs_path"` + Listen string `yaml:"listen"` + ListenHTTPS string `yaml:"listen_https"` + OwnURL string `yaml:"own_url"` // sent to workers via SSDP/UPnP + + // TLS certificate management. TLSxxx has priority over ACME. + TLSKey string `yaml:"tlskey"` + TLSCert string `yaml:"tlscert"` + ACMEDomainName string `yaml:"acme_domain_name"` // for the ACME Let's Encrypt client + + ActiveTaskTimeoutInterval time.Duration `yaml:"active_task_timeout_interval"` + ActiveWorkerTimeoutInterval time.Duration `yaml:"active_worker_timeout_interval"` + + WorkerCleanupMaxAge time.Duration `yaml:"worker_cleanup_max_age"` + WorkerCleanupStatus []string `yaml:"worker_cleanup_status"` + + /* This many failures (on a given job+task type combination) will ban a worker + * from that task type on that job. */ + BlacklistThreshold int `yaml:"blacklist_threshold"` + + // When this many workers have tried the task and failed, it will be hard-failed + // (even when there are workers left that could technically retry the task). + TaskFailAfterSoftFailCount int `yaml:"task_fail_after_softfail_count"` + + SSDPDiscovery bool `yaml:"ssdp_discovery"` + SSDPDeviceUUID string `yaml:"ssdp_device_uuid"` + + TestTasks TestTasks `yaml:"test_tasks"` + + // Shaman configuration settings. + Shaman ShamanConfig `yaml:"shaman"` + + // Authentication settings. + // JWT jwtauth.Config `yaml:"user_authentication"` + WorkerRegistrationSecret string `yaml:"worker_registration_secret"` + + // Dynamic worker pools (Azure Batch, Google Compute, AWS, that sort). + // DynamicPoolPlatforms *dppoller.Config `yaml:"dynamic_pool_platforms,omitempty"` + + // Websetup *WebsetupConf `yaml:"websetup,omitempty"` +} + +type ShamanConfig struct { + Enabled bool `yaml:"enabled"` + FileStorePath string `yaml:"fileStorePath"` + GarbageCollect ShamanGarbageCollect `yaml:"garbageCollect"` +} + +// GarbageCollect contains the config options for the GC. +type ShamanGarbageCollect struct { + // How frequently garbage collection is performed on the file store: + Period time.Duration `yaml:"period"` + // How old files must be before they are GC'd: + MaxAge time.Duration `yaml:"maxAge"` + // Paths to check for symlinks before GC'ing files. + ExtraCheckoutDirs []string `yaml:"extraCheckoutPaths"` + + // Used by the -gc CLI arg to silently disable the garbage collector + // while we're performing a manual sweep. + SilentlyDisable bool `yaml:"-"` +} + +// Conf is the latest version of the configuration. +// Currently it is version 3. +type Conf struct { + Base `yaml:",inline"` + + // Variable name → Variable definition + Variables map[string]Variable `yaml:"variables"` + + // audience + platform + variable name → variable value. + // Used to look up variables for a given platform and audience. + // The 'audience' is never "all" or ""; only concrete audiences are stored here. + VariablesLookup map[string]map[string]map[string]string `yaml:"-"` +} + +// Variable defines a configuration variable. +type Variable struct { + // Either "oneway" or "twoway" + Direction string `yaml:"direction" json:"direction"` + // Mapping from variable value to audience/platform definition. + Values VariableValues `yaml:"values" json:"values"` +} + +// VariableValues is the list of values of a variable. +type VariableValues []VariableValue + +// VariableValue defines which audience and platform see which value. +type VariableValue struct { + // Audience defines who will use this variable, either "all", "workers", or "users". Empty string is "all". + Audience string `yaml:"audience,omitempty" json:"audience,omitempty"` + + // Platforms that use this value. Only one of "Platform" and "Platforms" may be set. + Platform string `yaml:"platform,omitempty" json:"platform,omitempty"` + Platforms []string `yaml:"platforms,omitempty,flow" json:"platforms,omitempty,flow"` + + // The actual value of the variable for this audience+platform. + Value string `yaml:"value" json:"value"` +} + +// WebsetupConf are settings used by the web setup mode. +// type WebsetupConf struct { +// // When true, the websetup will hide certain settings that are infrastructure-specific. +// // For example, it hides MongoDB choice, port numbers, task log directory, all kind of +// // hosting-specific things. This is used, for example, by the automated Azure deployment +// // to avoid messing up settings that are specific to that particular installation. +// HideInfraSettings bool `yaml:"hide_infra_settings"` +// } + +// getConf parses flamenco-manager.yaml and returns its contents as a Conf object. +func getConf() (Conf, error) { + return loadConf(configFilename) +} + +// DefaultConfig returns a copy of the default configuration. +func DefaultConfig() Conf { + c := defaultConfig + c.Meta.Version = latestConfigVersion + c.constructVariableLookupTable() + return c +} + +// loadConf parses the given file and returns its contents as a Conf object. +func loadConf(filename string) (Conf, error) { + yamlFile, err := os.ReadFile(filename) + if err != nil { + return DefaultConfig(), err + } + + // First parse attempt, find the version. + baseConf := Base{} + if err := yaml.Unmarshal(yamlFile, &baseConf); err != nil { + return Conf{}, fmt.Errorf("unable to parse %s: %w", filename, err) + } + + // Versioning was supported from Flamenco config v1 to v2, but not further. + if baseConf.Meta.Version != latestConfigVersion { + return Conf{}, fmt.Errorf( + "configuration file %s version %d, but only version %d is supported", + filename, baseConf.Meta.Version, latestConfigVersion) + } + + // Second parse attempt, based on the version found. + c := DefaultConfig() + if err := yaml.Unmarshal(yamlFile, &c); err != nil { + return c, fmt.Errorf("unable to parse %s: %w", filename, err) + } + + c.constructVariableLookupTable() + c.parseURLs() + c.checkMode(c.Mode) + c.checkDatabase() + c.checkVariables() + c.checkTLS() + + return c, nil +} + +func (c *Conf) constructVariableLookupTable() { + lookup := map[string]map[string]map[string]string{} + + // Construct a list of all audiences except "" and "all" + concreteAudiences := []string{} + isWildcard := map[string]bool{"": true, "all": true} + for audience := range validAudiences { + if isWildcard[audience] { + continue + } + concreteAudiences = append(concreteAudiences, audience) + } + log.Debug(). + Strs("concreteAudiences", concreteAudiences). + Interface("isWildcard", isWildcard). + Msg("constructing variable lookup table") + + // setValue expands wildcard audiences into concrete ones. + var setValue func(audience, platform, name, value string) + setValue = func(audience, platform, name, value string) { + if isWildcard[audience] { + for _, aud := range concreteAudiences { + setValue(aud, platform, name, value) + } + return + } + + if lookup[audience] == nil { + lookup[audience] = map[string]map[string]string{} + } + if lookup[audience][platform] == nil { + lookup[audience][platform] = map[string]string{} + } + log.Debug(). + Str("audience", audience). + Str("platform", platform). + Str("name", name). + Str("value", value). + Msg("setting variable") + lookup[audience][platform][name] = value + } + + // Construct the lookup table for each audience+platform+name + for name, variable := range c.Variables { + log.Debug(). + Str("name", name). + Interface("variable", variable). + Msg("handling variable") + for _, value := range variable.Values { + + // Two-way values should not end in path separator. + // Given a variable 'apps' with value '/path/to/apps', + // '/path/to/apps/blender' should be remapped to '{apps}/blender'. + if variable.Direction == "twoway" { + if strings.Contains(value.Value, "\\") { + log.Warn(). + Str("variable", name). + Str("audience", value.Audience). + Str("platform", value.Platform). + Str("value", value.Value). + Msg("Backslash found in variable value. Change paths to use forward slashes instead.") + } + value.Value = strings.TrimRight(value.Value, "/") + } + + if value.Platform != "" { + setValue(value.Audience, value.Platform, name, value.Value) + } + for _, platform := range value.Platforms { + setValue(value.Audience, platform, name, value.Value) + } + } + } + log.Debug(). + Interface("variables", c.Variables). + Interface("lookup", lookup). + Msg("constructed lookup table") + c.VariablesLookup = lookup +} + +// ExpandVariables converts "{variable name}" to the value that belongs to the given audience and platform. +func (c *Conf) ExpandVariables(valueToExpand, audience, platform string) string { + audienceMap := c.VariablesLookup[audience] + if audienceMap == nil { + log.Warn(). + Str("valueToExpand", valueToExpand). + Str("audience", audience). + Str("platform", platform). + Msg("no variables defined for this audience") + return valueToExpand + } + + platformMap := audienceMap[platform] + if platformMap == nil { + log.Warn(). + Str("valueToExpand", valueToExpand). + Str("audience", audience). + Str("platform", platform). + Msg("no variables defined for this platform given this audience") + return valueToExpand + } + + // Variable replacement + for varname, varvalue := range platformMap { + placeholder := fmt.Sprintf("{%s}", varname) + valueToExpand = strings.Replace(valueToExpand, placeholder, varvalue, -1) + } + + return valueToExpand +} + +// checkVariables performs some basic checks on variable definitions. +// Note that the returned error only reflects the last-found error. +// All errors are logged, though. +func (c *Conf) checkVariables() error { + var err error + + directionNames := []string{"oneway", "twoway"} + validDirections := map[string]bool{} + for _, direction := range directionNames { + validDirections[direction] = true + } + + for name, variable := range c.Variables { + if !validDirections[variable.Direction] { + log.Error(). + Str("name", name). + Str("direction", variable.Direction). + Strs("validChoices", directionNames). + Msg("variable has invalid direction") + err = ErrBadDirection + } + for valueIndex, value := range variable.Values { + // No platforms at all. + if value.Platform == "" && len(value.Platforms) == 0 { + log.Error(). + Str("name", name). + Interface("value", value). + Msg("variable has a platformless value") + err = ErrMissingVariablePlatform + continue + } + + // Both Platform and Platforms. + if value.Platform != "" && len(value.Platforms) > 0 { + log.Warn(). + Str("name", name). + Interface("value", value). + Str("platform", value.Platform). + Strs("platforms", value.Platforms). + Msg("variable has a both 'platform' and 'platforms' set") + value.Platforms = append(value.Platforms, value.Platform) + value.Platform = "" + } + + if value.Audience == "" { + value.Audience = "all" + } else if !validAudiences[value.Audience] { + log.Error(). + Str("name", name). + Interface("value", value). + Str("audience", value.Audience). + Msg("variable invalid audience") + } + + variable.Values[valueIndex] = value + } + } + + return err +} + +func (c *Conf) checkDatabase() { + c.DatabaseDSN = strings.TrimSpace(c.DatabaseDSN) +} + +// Overwrite stores this configuration object as flamenco-manager.yaml. +func (c *Conf) Overwrite() error { + tempFilename := configFilename + "~" + if err := c.Write(tempFilename); err != nil { + return fmt.Errorf("error writing config to %s: %w", tempFilename, err) + } + if err := os.Rename(tempFilename, configFilename); err != nil { + return fmt.Errorf("error moving %s to %s: %w", tempFilename, configFilename, err) + } + + log.Info().Str("filename", configFilename).Msg("saved configuration to file") + return nil +} + +// Write saves the current in-memory configuration to a YAML file. +func (c *Conf) Write(filename string) error { + data, err := yaml.Marshal(c) + if err != nil { + return err + } + + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + fmt.Fprintln(f, "# Configuration file for Flamenco Manager.") + fmt.Fprintln(f, "# For an explanation of the fields, refer to flamenco-manager-example.yaml") + fmt.Fprintln(f, "#") + fmt.Fprintln(f, "# NOTE: this file will be overwritten by Flamenco Manager's web-based configuration system.") + fmt.Fprintln(f, "#") + now := time.Now() + fmt.Fprintf(f, "# This file was written on %s\n\n", now.Format("2006-01-02 15:04:05 -07:00")) + + n, err := f.Write(data) + if err != nil { + return err + } + if n < len(data) { + return io.ErrShortWrite + } + if err = f.Close(); err != nil { + return err + } + + log.Debug().Str("filename", filename).Msg("config file written") + return nil +} + +// HasCustomTLS returns true if both the TLS certificate and key files are configured. +func (c *Conf) HasCustomTLS() bool { + return c.TLSCert != "" && c.TLSKey != "" +} + +// HasTLS returns true if either a custom certificate or ACME/Let's Encrypt is used. +func (c *Conf) HasTLS() bool { + return c.ACMEDomainName != "" || c.HasCustomTLS() +} + +// OverrideMode checks the mode parameter for validity and logs that it's being overridden. +func (c *Conf) OverrideMode(mode string) { + if mode == c.Mode { + log.Warn().Str("mode", mode).Msg("trying to override run mode with current value; ignoring") + return + } + c.checkMode(mode) + log.Warn(). + Str("configured_mode", c.Mode). + Str("current_mode", mode). + Msg("overriding run mode") + c.Mode = mode +} + +func (c *Conf) checkMode(mode string) { + // Check mode for validity + if !validModes[mode] { + keys := make([]string, 0, len(validModes)) + for k := range validModes { + keys = append(keys, k) + } + log.Error(). + Strs("valid_values", keys). + Str("current_value", mode). + Msg("bad value for 'mode' configuration parameter") + } +} + +func (c *Conf) checkTLS() { + hasTLS := c.HasCustomTLS() + + if hasTLS && c.ListenHTTPS == "" { + c.ListenHTTPS = c.Listen + c.Listen = "" + } + + if !hasTLS || c.ACMEDomainName == "" { + return + } + + log.Warn(). + Str("tlscert", c.TLSCert). + Str("tlskey", c.TLSKey). + Str("acme_domain_name", c.ACMEDomainName). + Msg("ACME/Let's Encrypt will not be used because custom certificate is specified") + c.ACMEDomainName = "" +} + +func (c *Conf) parseURLs() { + // var err error + // if jwtURL, err := c.Flamenco.Parse(jwtPublicKeysRelativeURL); err != nil { + // log.WithFields(log.Fields{ + // "url": c.Flamenco.String(), + // log.ErrorKey: err, + // }).Error("unable to construct URL to get JWT public keys") + // } else { + // c.JWT.PublicKeysURL = jwtURL.String() + // } +} + +// GetTestConfig returns the configuration for unit tests. +// The config is loaded from `test-flamenco-manager.yaml` in the directory +// containing the caller's source. +func GetTestConfig() Conf { + _, myFilename, _, _ := runtime.Caller(1) + myDir := path.Dir(myFilename) + + filepath := path.Join(myDir, "test-flamenco-manager.yaml") + conf, err := loadConf(filepath) + if err != nil { + log.Fatal().Err(err).Str("file", filepath).Msg("unable to load test config") + } + + return conf +} diff --git a/internal/manager/config/settings_test.go b/internal/manager/config/settings_test.go new file mode 100644 index 00000000..cb185acb --- /dev/null +++ b/internal/manager/config/settings_test.go @@ -0,0 +1,68 @@ +package config + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultSettings(t *testing.T) { + config, err := loadConf("nonexistant.yaml") + assert.NotNil(t, err) // should indicate an error to open the file. + + // The settings should contain the defaults, though. + assert.Equal(t, latestConfigVersion, config.Meta.Version) + assert.Equal(t, "./task-logs", config.TaskLogsPath) + assert.Equal(t, "64ad4c21-6042-4378-9cdf-478f88b4f990", config.SSDPDeviceUUID) + + assert.Contains(t, config.Variables, "job_storage") + assert.Contains(t, config.Variables, "render") + assert.Equal(t, "oneway", config.Variables["ffmpeg"].Direction) + assert.Equal(t, "/usr/bin/ffmpeg", config.Variables["ffmpeg"].Values[0].Value) + assert.Equal(t, "linux", config.Variables["ffmpeg"].Values[0].Platform) + + linuxPVars, ok := config.VariablesLookup["workers"]["linux"] + assert.True(t, ok, "workers/linux should have variables: %v", config.VariablesLookup) + assert.Equal(t, "/shared/flamenco/jobs", linuxPVars["job_storage"]) + + winPVars, ok := config.VariablesLookup["users"]["windows"] + assert.True(t, ok) + assert.Equal(t, "S:/flamenco/jobs", winPVars["job_storage"]) +} + +func TestVariableValidation(t *testing.T) { + c := DefaultConfig() + + platformless := c.Variables["blender"] + platformless.Values = VariableValues{ + VariableValue{Value: "/path/to/blender"}, + VariableValue{Platform: "linux", Value: "/valid/path/blender"}, + } + c.Variables["blender"] = platformless + + err := c.checkVariables() + assert.Equal(t, ErrMissingVariablePlatform, err) + + assert.Equal(t, c.Variables["blender"].Values[0].Value, "/path/to/blender") + assert.Equal(t, c.Variables["blender"].Values[1].Value, "/valid/path/blender") +}