Start of configuration/settings framework, including variable replacement

cmd/flamenco-manager-poc/main.go

@@ -37,6 +37,7 @@ import (
 
 	"gitlab.com/blender/flamenco-ng-poc/internal/appinfo"
 	"gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl"
+	"gitlab.com/blender/flamenco-ng-poc/internal/manager/config"
 	"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
 	"gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
 	"gitlab.com/blender/flamenco-ng-poc/internal/manager/swagger_ui"
@@ -58,6 +59,10 @@ func main() {
 	if cliArgs.version {
 		return
 	}
+
+	// Load configuration.
+	configService := config.NewService()
+
 	if cliArgs.initDB {
 		log.Info().Msg("creating databases")
 		err := persistence.InitialSetup()
@@ -88,7 +93,7 @@ func main() {
 		log.Fatal().Err(err).Msg("error loading job compilers")
 	}
 	logStorage := task_logs.NewStorage("./task-logs") // TODO: load job storage path from configuration.
-	flamenco := api_impl.NewFlamenco(compiler, persist, logStorage)
+	flamenco := api_impl.NewFlamenco(compiler, persist, logStorage, configService)
 	e := buildWebService(flamenco, persist)
 
 	// Start the web server.

internal/manager/api_impl/api_impl.go

@@ -36,10 +36,11 @@ type Flamenco struct {
 	jobCompiler JobCompiler
 	persist     PersistenceService
 	logStorage  LogStorage
+	config      ConfigService
 }
 
 // Generate mock implementations of these interfaces.
-//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage
+//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage,ConfigService
 
 type PersistenceService interface {
 	StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error
@@ -68,14 +69,19 @@ type LogStorage interface {
 	RotateFile(logger zerolog.Logger, jobID, taskID string)
 }
 
+type ConfigService interface {
+	ExpandVariables(valueToExpand, audience, platform string) string
+}
+
 var _ api.ServerInterface = (*Flamenco)(nil)
 
 // NewFlamenco creates a new Flamenco service, using the given JobCompiler.
-func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage) *Flamenco {
+func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage, cs ConfigService) *Flamenco {
 	return &Flamenco{
 		jobCompiler: jc,
 		persist:     jps,
 		logStorage:  ls,
+		config:      cs,
 	}
 }
 

internal/manager/api_impl/mocks/api_impl_mock.gen.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage)
+// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage,ConfigService)
 
 // Package mocks is a generated GoMock package.
 package mocks
@@ -268,3 +268,40 @@ func (mr *MockLogStorageMockRecorder) Write(arg0, arg1, arg2, arg3 interface{})
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLogStorage)(nil).Write), arg0, arg1, arg2, arg3)
 }
+
+// MockConfigService is a mock of ConfigService interface.
+type MockConfigService struct {
+	ctrl     *gomock.Controller
+	recorder *MockConfigServiceMockRecorder
+}
+
+// MockConfigServiceMockRecorder is the mock recorder for MockConfigService.
+type MockConfigServiceMockRecorder struct {
+	mock *MockConfigService
+}
+
+// NewMockConfigService creates a new mock instance.
+func NewMockConfigService(ctrl *gomock.Controller) *MockConfigService {
+	mock := &MockConfigService{ctrl: ctrl}
+	mock.recorder = &MockConfigServiceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockConfigService) EXPECT() *MockConfigServiceMockRecorder {
+	return m.recorder
+}
+
+// ExpandVariables mocks base method.
+func (m *MockConfigService) ExpandVariables(arg0, arg1, arg2 string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ExpandVariables", arg0, arg1, arg2)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// ExpandVariables indicates an expected call of ExpandVariables.
+func (mr *MockConfigServiceMockRecorder) ExpandVariables(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpandVariables", reflect.TypeOf((*MockConfigService)(nil).ExpandVariables), arg0, arg1, arg2)
+}

internal/manager/api_impl/test-flamenco-manager.yaml

@@ -0,0 +1,60 @@
+# This file is loaded by unit tests.
+_meta:
+  version: 3
+mode: develop
+listen: '[::0]:8083'
+own_url: http://192.168.3.108:8083/
+flamenco: http://localhost:51234/
+manager_id: 5852bc5198377351f95d103e
+manager_secret: SRVwA7wAxPRfudvqTDOLXwPn1cDRIlADz5Ef9kHk7d52Us
+
+task_logs_path: /tmp/flamenco-unittests
+blacklist_threshold: 3
+
+shaman:
+    enabled: false
+
+variables:
+  blender:
+    direction: oneway
+    values:
+      - audience: users
+        platform: linux
+        value: /linux/path/to/blender
+      - audience: workers
+        platform: linux
+        value: /opt/myblenderbuild/blender
+      - platform: windows
+        value: 'c:/temp/blender.exe'
+      - platform: darwin
+        value: /opt/myblenderbuild/blender
+  ffmpeg:
+    direction: oneway
+    values:
+      - platform: linux
+        value: /usr/bin/ffmpeg
+      - platform: windows
+        value: xxx
+      - platform: darwin
+        value: xxx
+  render_long:
+    direction: twoway
+    values:
+      - platform: windows
+        value: s:/flamenco/render/long
+      - platform: linux
+        value: /shared/flamenco/render/long
+      - platform: darwin
+        value: /Volume/shared/flamenco/render/long
+
+  job_storage:
+    direction: twoway
+    values:
+      - platform: windows
+        value: s:/flamenco/jobs
+      - platform: linux
+        value: /shared/flamenco/jobs
+      - platform: darwin
+        value: /Volume/shared/flamenco/jobs
+      - platform: autumn
+        value: hey

internal/manager/api_impl/test_support.go

@@ -39,18 +39,21 @@ type mockedFlamenco struct {
 	flamenco    *Flamenco
 	jobCompiler *mocks.MockJobCompiler
 	persistence *mocks.MockPersistenceService
+	config      *mocks.MockConfigService
 }
 
 func newMockedFlamenco(mockCtrl *gomock.Controller) mockedFlamenco {
 	jc := mocks.NewMockJobCompiler(mockCtrl)
 	ps := mocks.NewMockPersistenceService(mockCtrl)
 	ls := mocks.NewMockLogStorage(mockCtrl)
-	f := NewFlamenco(jc, ps, ls)
+	cs := mocks.NewMockConfigService(mockCtrl)
+	f := NewFlamenco(jc, ps, ls, cs)
 
 	return mockedFlamenco{
 		flamenco:    f,
 		jobCompiler: jc,
 		persistence: ps,
+		config:      cs,
 	}
 }
 

internal/manager/api_impl/varrepl.go

@@ -0,0 +1,59 @@
+package api_impl
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"reflect"
+
+	"gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
+)
+
+var stringType = reflect.TypeOf("somestring")
+
+type VariableReplacer interface {
+	ExpandVariables(valueToExpand, audience, platform string) string
+}
+
+// replaceTaskVariables performs variable replacement for worker tasks.
+func replaceTaskVariables(replacer VariableReplacer, task persistence.Task, worker persistence.Worker) persistence.Task {
+	repl := func(value string) string {
+		return replacer.ExpandVariables(value, "workers", worker.Platform)
+	}
+
+	for cmdIndex, cmd := range task.Commands {
+		for key, value := range cmd.Parameters {
+			switch v := value.(type) {
+			case string:
+				task.Commands[cmdIndex].Parameters[key] = repl(v)
+			case []string:
+				replaced := make([]string, len(v))
+				for idx := range v {
+					replaced[idx] = repl(v[idx])
+				}
+				task.Commands[cmdIndex].Parameters[key] = replaced
+			default:
+				continue
+			}
+		}
+	}
+
+	return task
+}

internal/manager/api_impl/varrepl_test.go

@@ -0,0 +1,113 @@
+package api_impl
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gitlab.com/blender/flamenco-ng-poc/internal/manager/config"
+	"gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
+)
+
+func varreplTestTask() persistence.Task {
+	return persistence.Task{
+		Commands: []persistence.Command{
+			{Name: "echo", Parameters: persistence.StringInterfaceMap{
+				"message": "Running Blender from {blender} {blender}"}},
+			{Name: "sleep", Parameters: persistence.StringInterfaceMap{
+				"{blender}": 3}},
+			{
+				Name: "blender_render",
+				Parameters: persistence.StringInterfaceMap{
+					"filepath":     "{job_storage}/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+					"exe":          "{blender}",
+					"otherpath":    "{hey}/haha",
+					"frames":       "47",
+					"cycles_chunk": 1.0,
+					"args":         []string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+				},
+			},
+		},
+	}
+}
+
+func TestReplaceVariables(t *testing.T) {
+	worker := persistence.Worker{Platform: "linux"}
+	task := varreplTestTask()
+	conf := config.GetTestConfig()
+	replacedTask := replaceTaskVariables(&conf, task, worker)
+
+	// Single string value.
+	assert.Equal(t,
+		"/opt/myblenderbuild/blender",
+		replacedTask.Commands[2].Parameters["exe"],
+	)
+
+	// Array value.
+	assert.Equal(t,
+		[]string{"--render-out", "/shared/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+		replacedTask.Commands[2].Parameters["args"],
+	)
+
+	// Substitution should happen as often as needed.
+	assert.Equal(t,
+		"Running Blender from /opt/myblenderbuild/blender /opt/myblenderbuild/blender",
+		replacedTask.Commands[0].Parameters["message"],
+	)
+
+	// No substitution should happen on keys, just on values.
+	assert.Equal(t, 3, replacedTask.Commands[1].Parameters["{blender}"])
+}
+
+func TestReplacePathsWindows(t *testing.T) {
+	worker := persistence.Worker{Platform: "windows"}
+	task := varreplTestTask()
+	conf := config.GetTestConfig()
+	replacedTask := replaceTaskVariables(&conf, task, worker)
+
+	assert.Equal(t,
+		"s:/flamenco/jobs/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+		replacedTask.Commands[2].Parameters["filepath"],
+	)
+	assert.Equal(t,
+		[]string{"--render-out", "s:/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+		replacedTask.Commands[2].Parameters["args"],
+	)
+	assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"])
+}
+
+func TestReplacePathsUnknownOS(t *testing.T) {
+	worker := persistence.Worker{Platform: "autumn"}
+	task := varreplTestTask()
+	conf := config.GetTestConfig()
+	replacedTask := replaceTaskVariables(&conf, task, worker)
+
+	assert.Equal(t,
+		"hey/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+		replacedTask.Commands[2].Parameters["filepath"],
+	)
+	assert.Equal(t,
+		[]string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+		replacedTask.Commands[2].Parameters["args"],
+	)
+	assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"])
+}

internal/manager/config/service.go

@@ -0,0 +1,47 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+// Service provides access to Flamenco Manager configuration.
+type Service struct {
+	config Conf
+}
+
+func NewService() *Service {
+	return &Service{}
+}
+
+func (s *Service) Load() error {
+	config, err := getConf()
+	if err != nil {
+		return err
+	}
+	s.config = config
+	return nil
+}
+
+func (s *Service) ExpandVariables(valueToExpand, audience, platform string) string {
+	return s.config.ExpandVariables(valueToExpand, audience, platform)
+}
+
+func (s *Service) Get() *Conf {
+	return &s.config
+}

internal/manager/config/settings.go

@@ -0,0 +1,650 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	yaml "gopkg.in/yaml.v2"
+
+	"gitlab.com/blender/flamenco-ng-poc/pkg/api"
+)
+
+const (
+	configFilename = "flamenco-manager.yaml"
+
+	latestConfigVersion = 3
+
+	// // relative to the Flamenco Server Base URL:
+	// jwtPublicKeysRelativeURL = "api/flamenco/jwt/public-keys"
+
+	defaultShamanFilestorePath = "/shared/flamenco/file-store"
+	defaultJobStorage          = "/shared/flamenco/jobs"
+)
+
+var (
+	// ErrMissingVariablePlatform is returned when a variable doesn't declare any valid platform for a certain value.
+	ErrMissingVariablePlatform = errors.New("variable's value is missing platform declaration")
+	// ErrBadDirection is returned when a direction doesn't match "oneway" or "twoway"
+	ErrBadDirection = errors.New("variable's direction is invalid")
+
+	// Valid values for the "mode" config variable.
+	validModes = map[string]bool{
+		"develop":    true,
+		"production": true,
+	}
+
+	// Valid values for the "audience" tag of a ConfV2 variable.
+	validAudiences = map[string]bool{
+		"all":     true,
+		"workers": true,
+		"users":   true,
+	}
+
+	// The default configuration, use DefaultConfig() to obtain a copy.
+	defaultConfig = Conf{
+		Base: Base{
+			Meta: ConfMeta{Version: latestConfigVersion},
+
+			Mode:         "production",
+			ManagerName:  "Flamenco Manager",
+			Listen:       ":8080",
+			ListenHTTPS:  ":8433",
+			DatabaseDSN:  "host=localhost user=flamenco password=flamenco dbname=flamenco TimeZone=Europe/Amsterdam",
+			TaskLogsPath: "./task-logs",
+			// DownloadTaskSleep:           10 * time.Minute,
+			// DownloadTaskRecheckThrottle: 10 * time.Second,
+			// TaskUpdatePushMaxInterval:   5 * time.Second,
+			// TaskUpdatePushMaxCount:      3000,
+			// CancelTaskFetchInterval:     10 * time.Second,
+			ActiveTaskTimeoutInterval:   10 * time.Minute,
+			ActiveWorkerTimeoutInterval: 1 * time.Minute,
+			// FlamencoStr:                 defaultServerURL,
+
+			// // Days are assumed to be 24 hours long. This is not exactly accurate, but should
+			// // be accurate enough for this type of cleanup.
+			// TaskCleanupMaxAge: 14 * 24 * time.Hour,
+			SSDPDiscovery:  false,                                  // Only enable after SSDP discovery has been improved (avoid finding printers).
+			SSDPDeviceUUID: "64ad4c21-6042-4378-9cdf-478f88b4f990", // UUID specific for Flamenco v3.
+
+			BlacklistThreshold:         3,
+			TaskFailAfterSoftFailCount: 3,
+
+			WorkerCleanupStatus: []string{string(api.WorkerStatusOffline)},
+
+			TestTasks: TestTasks{
+				BlenderRender: BlenderRenderConfig{
+					JobStorage:   "{job_storage}/test-jobs",
+					RenderOutput: "{render}/test-renders",
+				},
+			},
+
+			Shaman: ShamanConfig{
+				Enabled:       true,
+				FileStorePath: defaultShamanFilestorePath,
+				GarbageCollect: ShamanGarbageCollect{
+					Period:            24 * time.Hour,
+					MaxAge:            31 * 24 * time.Hour,
+					ExtraCheckoutDirs: []string{},
+				},
+			},
+
+			// JWT: jwtauth.Config{
+			// 	DownloadKeysInterval: 1 * time.Hour,
+			// },
+		},
+
+		Variables: map[string]Variable{
+			"blender": {
+				Direction: "oneway",
+				Values: VariableValues{
+					VariableValue{Platform: "linux", Value: "/linux/path/to/blender --factory-startup --background"},
+					VariableValue{Platform: "windows", Value: "C:/windows/path/to/blender.exe --factory-startup --background"},
+					VariableValue{Platform: "darwin", Value: "/Volumes/Applications/Blender/blender --factory-startup --background"},
+				},
+			},
+			"ffmpeg": {
+				Direction: "oneway",
+				Values: VariableValues{
+					VariableValue{Platform: "linux", Value: "/usr/bin/ffmpeg"},
+					VariableValue{Platform: "windows", Value: "C:/windows/path/to/ffmpeg.exe"},
+					VariableValue{Platform: "darwin", Value: "/Volumes/Applications/FFmpeg/ffmpeg"},
+				},
+			},
+			"job_storage": {
+				Direction: "twoway",
+				Values: VariableValues{
+					VariableValue{Platform: "linux", Value: "/shared/flamenco/jobs"},
+					VariableValue{Platform: "windows", Value: "S:/flamenco/jobs"},
+					VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/jobs"},
+				},
+			},
+			"render": {
+				Direction: "twoway",
+				Values: VariableValues{
+					VariableValue{Platform: "linux", Value: "/shared/flamenco/render"},
+					VariableValue{Platform: "windows", Value: "S:/flamenco/render"},
+					VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/render"},
+				},
+			},
+		},
+	}
+)
+
+// BlenderRenderConfig represents the configuration required for a test render.
+type BlenderRenderConfig struct {
+	JobStorage   string `yaml:"job_storage"`
+	RenderOutput string `yaml:"render_output"`
+}
+
+// TestTasks represents the 'test_tasks' key in the Manager's configuration file.
+type TestTasks struct {
+	BlenderRender BlenderRenderConfig `yaml:"test_blender_render"`
+}
+
+// ConfMeta contains configuration file metadata.
+type ConfMeta struct {
+	// Version of the config file structure.
+	Version int `yaml:"version"`
+}
+
+// Base contains those settings that are shared by all configuration versions.
+type Base struct {
+	Meta ConfMeta `yaml:"_meta"`
+
+	Mode         string `yaml:"mode"` // either "develop" or "production"
+	ManagerName  string `yaml:"manager_name"`
+	DatabaseDSN  string `yaml:"database_url"`
+	TaskLogsPath string `yaml:"task_logs_path"`
+	Listen       string `yaml:"listen"`
+	ListenHTTPS  string `yaml:"listen_https"`
+	OwnURL       string `yaml:"own_url"` // sent to workers via SSDP/UPnP
+
+	// TLS certificate management. TLSxxx has priority over ACME.
+	TLSKey         string `yaml:"tlskey"`
+	TLSCert        string `yaml:"tlscert"`
+	ACMEDomainName string `yaml:"acme_domain_name"` // for the ACME Let's Encrypt client
+
+	ActiveTaskTimeoutInterval   time.Duration `yaml:"active_task_timeout_interval"`
+	ActiveWorkerTimeoutInterval time.Duration `yaml:"active_worker_timeout_interval"`
+
+	WorkerCleanupMaxAge time.Duration `yaml:"worker_cleanup_max_age"`
+	WorkerCleanupStatus []string      `yaml:"worker_cleanup_status"`
+
+	/* This many failures (on a given job+task type combination) will ban a worker
+	 * from that task type on that job. */
+	BlacklistThreshold int `yaml:"blacklist_threshold"`
+
+	// When this many workers have tried the task and failed, it will be hard-failed
+	// (even when there are workers left that could technically retry the task).
+	TaskFailAfterSoftFailCount int `yaml:"task_fail_after_softfail_count"`
+
+	SSDPDiscovery  bool   `yaml:"ssdp_discovery"`
+	SSDPDeviceUUID string `yaml:"ssdp_device_uuid"`
+
+	TestTasks TestTasks `yaml:"test_tasks"`
+
+	// Shaman configuration settings.
+	Shaman ShamanConfig `yaml:"shaman"`
+
+	// Authentication settings.
+	// JWT                      jwtauth.Config `yaml:"user_authentication"`
+	WorkerRegistrationSecret string `yaml:"worker_registration_secret"`
+
+	// Dynamic worker pools (Azure Batch, Google Compute, AWS, that sort).
+	// DynamicPoolPlatforms *dppoller.Config `yaml:"dynamic_pool_platforms,omitempty"`
+
+	// Websetup *WebsetupConf `yaml:"websetup,omitempty"`
+}
+
+type ShamanConfig struct {
+	Enabled        bool                 `yaml:"enabled"`
+	FileStorePath  string               `yaml:"fileStorePath"`
+	GarbageCollect ShamanGarbageCollect `yaml:"garbageCollect"`
+}
+
+// GarbageCollect contains the config options for the GC.
+type ShamanGarbageCollect struct {
+	// How frequently garbage collection is performed on the file store:
+	Period time.Duration `yaml:"period"`
+	// How old files must be before they are GC'd:
+	MaxAge time.Duration `yaml:"maxAge"`
+	// Paths to check for symlinks before GC'ing files.
+	ExtraCheckoutDirs []string `yaml:"extraCheckoutPaths"`
+
+	// Used by the -gc CLI arg to silently disable the garbage collector
+	// while we're performing a manual sweep.
+	SilentlyDisable bool `yaml:"-"`
+}
+
+// Conf is the latest version of the configuration.
+// Currently it is version 3.
+type Conf struct {
+	Base `yaml:",inline"`
+
+	// Variable name → Variable definition
+	Variables map[string]Variable `yaml:"variables"`
+
+	// audience + platform + variable name → variable value.
+	// Used to look up variables for a given platform and audience.
+	// The 'audience' is never "all" or ""; only concrete audiences are stored here.
+	VariablesLookup map[string]map[string]map[string]string `yaml:"-"`
+}
+
+// Variable defines a configuration variable.
+type Variable struct {
+	// Either "oneway" or "twoway"
+	Direction string `yaml:"direction" json:"direction"`
+	// Mapping from variable value to audience/platform definition.
+	Values VariableValues `yaml:"values" json:"values"`
+}
+
+// VariableValues is the list of values of a variable.
+type VariableValues []VariableValue
+
+// VariableValue defines which audience and platform see which value.
+type VariableValue struct {
+	// Audience defines who will use this variable, either "all", "workers", or "users". Empty string is "all".
+	Audience string `yaml:"audience,omitempty" json:"audience,omitempty"`
+
+	// Platforms that use this value. Only one of "Platform" and "Platforms" may be set.
+	Platform  string   `yaml:"platform,omitempty" json:"platform,omitempty"`
+	Platforms []string `yaml:"platforms,omitempty,flow" json:"platforms,omitempty,flow"`
+
+	// The actual value of the variable for this audience+platform.
+	Value string `yaml:"value" json:"value"`
+}
+
+// WebsetupConf are settings used by the web setup mode.
+// type WebsetupConf struct {
+// 	// When true, the websetup will hide certain settings that are infrastructure-specific.
+// 	// For example, it hides MongoDB choice, port numbers, task log directory, all kind of
+// 	// hosting-specific things. This is used, for example, by the automated Azure deployment
+// 	// to avoid messing up settings that are specific to that particular installation.
+// 	HideInfraSettings bool `yaml:"hide_infra_settings"`
+// }
+
+// getConf parses flamenco-manager.yaml and returns its contents as a Conf object.
+func getConf() (Conf, error) {
+	return loadConf(configFilename)
+}
+
+// DefaultConfig returns a copy of the default configuration.
+func DefaultConfig() Conf {
+	c := defaultConfig
+	c.Meta.Version = latestConfigVersion
+	c.constructVariableLookupTable()
+	return c
+}
+
+// loadConf parses the given file and returns its contents as a Conf object.
+func loadConf(filename string) (Conf, error) {
+	yamlFile, err := os.ReadFile(filename)
+	if err != nil {
+		return DefaultConfig(), err
+	}
+
+	// First parse attempt, find the version.
+	baseConf := Base{}
+	if err := yaml.Unmarshal(yamlFile, &baseConf); err != nil {
+		return Conf{}, fmt.Errorf("unable to parse %s: %w", filename, err)
+	}
+
+	// Versioning was supported from Flamenco config v1 to v2, but not further.
+	if baseConf.Meta.Version != latestConfigVersion {
+		return Conf{}, fmt.Errorf(
+			"configuration file %s version %d, but only version %d is supported",
+			filename, baseConf.Meta.Version, latestConfigVersion)
+	}
+
+	// Second parse attempt, based on the version found.
+	c := DefaultConfig()
+	if err := yaml.Unmarshal(yamlFile, &c); err != nil {
+		return c, fmt.Errorf("unable to parse %s: %w", filename, err)
+	}
+
+	c.constructVariableLookupTable()
+	c.parseURLs()
+	c.checkMode(c.Mode)
+	c.checkDatabase()
+	c.checkVariables()
+	c.checkTLS()
+
+	return c, nil
+}
+
+func (c *Conf) constructVariableLookupTable() {
+	lookup := map[string]map[string]map[string]string{}
+
+	// Construct a list of all audiences except "" and "all"
+	concreteAudiences := []string{}
+	isWildcard := map[string]bool{"": true, "all": true}
+	for audience := range validAudiences {
+		if isWildcard[audience] {
+			continue
+		}
+		concreteAudiences = append(concreteAudiences, audience)
+	}
+	log.Debug().
+		Strs("concreteAudiences", concreteAudiences).
+		Interface("isWildcard", isWildcard).
+		Msg("constructing variable lookup table")
+
+	// setValue expands wildcard audiences into concrete ones.
+	var setValue func(audience, platform, name, value string)
+	setValue = func(audience, platform, name, value string) {
+		if isWildcard[audience] {
+			for _, aud := range concreteAudiences {
+				setValue(aud, platform, name, value)
+			}
+			return
+		}
+
+		if lookup[audience] == nil {
+			lookup[audience] = map[string]map[string]string{}
+		}
+		if lookup[audience][platform] == nil {
+			lookup[audience][platform] = map[string]string{}
+		}
+		log.Debug().
+			Str("audience", audience).
+			Str("platform", platform).
+			Str("name", name).
+			Str("value", value).
+			Msg("setting variable")
+		lookup[audience][platform][name] = value
+	}
+
+	// Construct the lookup table for each audience+platform+name
+	for name, variable := range c.Variables {
+		log.Debug().
+			Str("name", name).
+			Interface("variable", variable).
+			Msg("handling variable")
+		for _, value := range variable.Values {
+
+			// Two-way values should not end in path separator.
+			// Given a variable 'apps' with value '/path/to/apps',
+			// '/path/to/apps/blender' should be remapped to '{apps}/blender'.
+			if variable.Direction == "twoway" {
+				if strings.Contains(value.Value, "\\") {
+					log.Warn().
+						Str("variable", name).
+						Str("audience", value.Audience).
+						Str("platform", value.Platform).
+						Str("value", value.Value).
+						Msg("Backslash found in variable value. Change paths to use forward slashes instead.")
+				}
+				value.Value = strings.TrimRight(value.Value, "/")
+			}
+
+			if value.Platform != "" {
+				setValue(value.Audience, value.Platform, name, value.Value)
+			}
+			for _, platform := range value.Platforms {
+				setValue(value.Audience, platform, name, value.Value)
+			}
+		}
+	}
+	log.Debug().
+		Interface("variables", c.Variables).
+		Interface("lookup", lookup).
+		Msg("constructed lookup table")
+	c.VariablesLookup = lookup
+}
+
+// ExpandVariables converts "{variable name}" to the value that belongs to the given audience and platform.
+func (c *Conf) ExpandVariables(valueToExpand, audience, platform string) string {
+	audienceMap := c.VariablesLookup[audience]
+	if audienceMap == nil {
+		log.Warn().
+			Str("valueToExpand", valueToExpand).
+			Str("audience", audience).
+			Str("platform", platform).
+			Msg("no variables defined for this audience")
+		return valueToExpand
+	}
+
+	platformMap := audienceMap[platform]
+	if platformMap == nil {
+		log.Warn().
+			Str("valueToExpand", valueToExpand).
+			Str("audience", audience).
+			Str("platform", platform).
+			Msg("no variables defined for this platform given this audience")
+		return valueToExpand
+	}
+
+	// Variable replacement
+	for varname, varvalue := range platformMap {
+		placeholder := fmt.Sprintf("{%s}", varname)
+		valueToExpand = strings.Replace(valueToExpand, placeholder, varvalue, -1)
+	}
+
+	return valueToExpand
+}
+
+// checkVariables performs some basic checks on variable definitions.
+// Note that the returned error only reflects the last-found error.
+// All errors are logged, though.
+func (c *Conf) checkVariables() error {
+	var err error
+
+	directionNames := []string{"oneway", "twoway"}
+	validDirections := map[string]bool{}
+	for _, direction := range directionNames {
+		validDirections[direction] = true
+	}
+
+	for name, variable := range c.Variables {
+		if !validDirections[variable.Direction] {
+			log.Error().
+				Str("name", name).
+				Str("direction", variable.Direction).
+				Strs("validChoices", directionNames).
+				Msg("variable has invalid direction")
+			err = ErrBadDirection
+		}
+		for valueIndex, value := range variable.Values {
+			// No platforms at all.
+			if value.Platform == "" && len(value.Platforms) == 0 {
+				log.Error().
+					Str("name", name).
+					Interface("value", value).
+					Msg("variable has a platformless value")
+				err = ErrMissingVariablePlatform
+				continue
+			}
+
+			// Both Platform and Platforms.
+			if value.Platform != "" && len(value.Platforms) > 0 {
+				log.Warn().
+					Str("name", name).
+					Interface("value", value).
+					Str("platform", value.Platform).
+					Strs("platforms", value.Platforms).
+					Msg("variable has a both 'platform' and 'platforms' set")
+				value.Platforms = append(value.Platforms, value.Platform)
+				value.Platform = ""
+			}
+
+			if value.Audience == "" {
+				value.Audience = "all"
+			} else if !validAudiences[value.Audience] {
+				log.Error().
+					Str("name", name).
+					Interface("value", value).
+					Str("audience", value.Audience).
+					Msg("variable invalid audience")
+			}
+
+			variable.Values[valueIndex] = value
+		}
+	}
+
+	return err
+}
+
+func (c *Conf) checkDatabase() {
+	c.DatabaseDSN = strings.TrimSpace(c.DatabaseDSN)
+}
+
+// Overwrite stores this configuration object as flamenco-manager.yaml.
+func (c *Conf) Overwrite() error {
+	tempFilename := configFilename + "~"
+	if err := c.Write(tempFilename); err != nil {
+		return fmt.Errorf("error writing config to %s: %w", tempFilename, err)
+	}
+	if err := os.Rename(tempFilename, configFilename); err != nil {
+		return fmt.Errorf("error moving %s to %s: %w", tempFilename, configFilename, err)
+	}
+
+	log.Info().Str("filename", configFilename).Msg("saved configuration to file")
+	return nil
+}
+
+// Write saves the current in-memory configuration to a YAML file.
+func (c *Conf) Write(filename string) error {
+	data, err := yaml.Marshal(c)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintln(f, "# Configuration file for Flamenco Manager.")
+	fmt.Fprintln(f, "# For an explanation of the fields, refer to flamenco-manager-example.yaml")
+	fmt.Fprintln(f, "#")
+	fmt.Fprintln(f, "# NOTE: this file will be overwritten by Flamenco Manager's web-based configuration system.")
+	fmt.Fprintln(f, "#")
+	now := time.Now()
+	fmt.Fprintf(f, "# This file was written on %s\n\n", now.Format("2006-01-02 15:04:05 -07:00"))
+
+	n, err := f.Write(data)
+	if err != nil {
+		return err
+	}
+	if n < len(data) {
+		return io.ErrShortWrite
+	}
+	if err = f.Close(); err != nil {
+		return err
+	}
+
+	log.Debug().Str("filename", filename).Msg("config file written")
+	return nil
+}
+
+// HasCustomTLS returns true if both the TLS certificate and key files are configured.
+func (c *Conf) HasCustomTLS() bool {
+	return c.TLSCert != "" && c.TLSKey != ""
+}
+
+// HasTLS returns true if either a custom certificate or ACME/Let's Encrypt is used.
+func (c *Conf) HasTLS() bool {
+	return c.ACMEDomainName != "" || c.HasCustomTLS()
+}
+
+// OverrideMode checks the mode parameter for validity and logs that it's being overridden.
+func (c *Conf) OverrideMode(mode string) {
+	if mode == c.Mode {
+		log.Warn().Str("mode", mode).Msg("trying to override run mode with current value; ignoring")
+		return
+	}
+	c.checkMode(mode)
+	log.Warn().
+		Str("configured_mode", c.Mode).
+		Str("current_mode", mode).
+		Msg("overriding run mode")
+	c.Mode = mode
+}
+
+func (c *Conf) checkMode(mode string) {
+	// Check mode for validity
+	if !validModes[mode] {
+		keys := make([]string, 0, len(validModes))
+		for k := range validModes {
+			keys = append(keys, k)
+		}
+		log.Error().
+			Strs("valid_values", keys).
+			Str("current_value", mode).
+			Msg("bad value for 'mode' configuration parameter")
+	}
+}
+
+func (c *Conf) checkTLS() {
+	hasTLS := c.HasCustomTLS()
+
+	if hasTLS && c.ListenHTTPS == "" {
+		c.ListenHTTPS = c.Listen
+		c.Listen = ""
+	}
+
+	if !hasTLS || c.ACMEDomainName == "" {
+		return
+	}
+
+	log.Warn().
+		Str("tlscert", c.TLSCert).
+		Str("tlskey", c.TLSKey).
+		Str("acme_domain_name", c.ACMEDomainName).
+		Msg("ACME/Let's Encrypt will not be used because custom certificate is specified")
+	c.ACMEDomainName = ""
+}
+
+func (c *Conf) parseURLs() {
+	// var err error
+	// if jwtURL, err := c.Flamenco.Parse(jwtPublicKeysRelativeURL); err != nil {
+	// 	log.WithFields(log.Fields{
+	// 		"url":        c.Flamenco.String(),
+	// 		log.ErrorKey: err,
+	// 	}).Error("unable to construct URL to get JWT public keys")
+	// } else {
+	// 	c.JWT.PublicKeysURL = jwtURL.String()
+	// }
+}
+
+// GetTestConfig returns the configuration for unit tests.
+// The config is loaded from `test-flamenco-manager.yaml` in the directory
+// containing the caller's source.
+func GetTestConfig() Conf {
+	_, myFilename, _, _ := runtime.Caller(1)
+	myDir := path.Dir(myFilename)
+
+	filepath := path.Join(myDir, "test-flamenco-manager.yaml")
+	conf, err := loadConf(filepath)
+	if err != nil {
+		log.Fatal().Err(err).Str("file", filepath).Msg("unable to load test config")
+	}
+
+	return conf
+}

internal/manager/config/settings_test.go

@@ -0,0 +1,68 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultSettings(t *testing.T) {
+	config, err := loadConf("nonexistant.yaml")
+	assert.NotNil(t, err) // should indicate an error to open the file.
+
+	// The settings should contain the defaults, though.
+	assert.Equal(t, latestConfigVersion, config.Meta.Version)
+	assert.Equal(t, "./task-logs", config.TaskLogsPath)
+	assert.Equal(t, "64ad4c21-6042-4378-9cdf-478f88b4f990", config.SSDPDeviceUUID)
+
+	assert.Contains(t, config.Variables, "job_storage")
+	assert.Contains(t, config.Variables, "render")
+	assert.Equal(t, "oneway", config.Variables["ffmpeg"].Direction)
+	assert.Equal(t, "/usr/bin/ffmpeg", config.Variables["ffmpeg"].Values[0].Value)
+	assert.Equal(t, "linux", config.Variables["ffmpeg"].Values[0].Platform)
+
+	linuxPVars, ok := config.VariablesLookup["workers"]["linux"]
+	assert.True(t, ok, "workers/linux should have variables: %v", config.VariablesLookup)
+	assert.Equal(t, "/shared/flamenco/jobs", linuxPVars["job_storage"])
+
+	winPVars, ok := config.VariablesLookup["users"]["windows"]
+	assert.True(t, ok)
+	assert.Equal(t, "S:/flamenco/jobs", winPVars["job_storage"])
+}
+
+func TestVariableValidation(t *testing.T) {
+	c := DefaultConfig()
+
+	platformless := c.Variables["blender"]
+	platformless.Values = VariableValues{
+		VariableValue{Value: "/path/to/blender"},
+		VariableValue{Platform: "linux", Value: "/valid/path/blender"},
+	}
+	c.Variables["blender"] = platformless
+
+	err := c.checkVariables()
+	assert.Equal(t, ErrMissingVariablePlatform, err)
+
+	assert.Equal(t, c.Variables["blender"].Values[0].Value, "/path/to/blender")
+	assert.Equal(t, c.Variables["blender"].Values[1].Value, "/valid/path/blender")
+}