diff --git a/cmd/flamenco-worker/main.go b/cmd/flamenco-worker/main.go index ad95c5d0..1c523ae2 100644 --- a/cmd/flamenco-worker/main.go +++ b/cmd/flamenco-worker/main.go @@ -30,6 +30,8 @@ var ( shutdownComplete chan struct{} ) +const flamencoWorkerDatabase = "flamenco-worker.sqlite" + var cliArgs struct { // Do-and-quit flags. version bool @@ -231,8 +233,12 @@ func upstreamBufferOrDie(client worker.FlamencoClient, timeService clock.Clock) log.Fatal().Err(err).Msg("unable to create task update queue database") } - // TODO: make filename configurable? - if err := buffer.OpenDB(ctx, "flamenco-worker.sqlite"); err != nil { + dbPath, err := appinfo.InFlamencoHome(flamencoWorkerDatabase) + if err != nil { + log.Fatal().Err(err).Msg("unable to figure out where to save my database") + } + + if err := buffer.OpenDB(ctx, dbPath); err != nil { log.Fatal().Err(err).Msg("unable to open task update queue database") } diff --git a/go.mod b/go.mod index 0a5e1057..1b690c49 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( ) require ( + github.com/adrg/xdg v0.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/ghodss/yaml v1.0.0 // indirect diff --git a/go.sum b/go.sum index 033134ff..9fd03f7f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -217,6 +219,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/appinfo/xdg_paths.go b/internal/appinfo/xdg_paths.go new file mode 100644 index 00000000..114e505b --- /dev/null +++ b/internal/appinfo/xdg_paths.go @@ -0,0 +1,24 @@ +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "os" + "path" + "path/filepath" + + "github.com/adrg/xdg" +) + +// InFlamencoHome returns the filename in the 'flamenco home' dir, and ensures +// that the directory exists. +func InFlamencoHome(filename string) (string, error) { + flamencoHome := os.Getenv("FLAMENCO_HOME") + if flamencoHome == "" { + return xdg.DataFile(path.Join(xdgApplicationName, filename)) + } + if err := os.MkdirAll(flamencoHome, os.ModePerm); err != nil { + return "", err + } + return filepath.Join(flamencoHome, filename), nil +} diff --git a/internal/appinfo/xdg_paths_lowercase.go b/internal/appinfo/xdg_paths_lowercase.go new file mode 100644 index 00000000..6585dfda --- /dev/null +++ b/internal/appinfo/xdg_paths_lowercase.go @@ -0,0 +1,7 @@ +//go:build !(windows || darwin) + +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +const xdgApplicationName = "flamenco" diff --git a/internal/appinfo/xdg_paths_titlecase.go b/internal/appinfo/xdg_paths_titlecase.go new file mode 100644 index 00000000..6eb1a41d --- /dev/null +++ b/internal/appinfo/xdg_paths_titlecase.go @@ -0,0 +1,7 @@ +//go:build windows || darwin + +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +const xdgApplicationName = "Flamenco" diff --git a/internal/worker/config.go b/internal/worker/config.go index 5c9e2e77..00f9a102 100644 --- a/internal/worker/config.go +++ b/internal/worker/config.go @@ -9,9 +9,11 @@ import ( "io/fs" "net/url" "os" + "path/filepath" "strings" "time" + "git.blender.org/flamenco/internal/appinfo" "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v2" ) @@ -69,7 +71,12 @@ func (fcw *FileConfigWrangler) WorkerConfig() (WorkerConfig, error) { } wc := fcw.DefaultConfig() - err := fcw.loadConfig(configFilename, &wc) + filepath, err := appinfo.InFlamencoHome(configFilename) + if err != nil { + return wc, err + } + + err = fcw.loadConfig(filepath, &wc) if err != nil { switch { @@ -101,14 +108,19 @@ func (fcw *FileConfigWrangler) SaveConfig() error { } func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) { + filepath, err := appinfo.InFlamencoHome(credentialsFilename) + if err != nil { + return WorkerCredentials{}, err + } + var creds WorkerCredentials - err := fcw.loadConfig(credentialsFilename, &creds) + err = fcw.loadConfig(filepath, &creds) if err != nil { return WorkerCredentials{}, err } log.Info(). - Str("filename", credentialsFilename). + Str("filename", filepath). Msg("loaded credentials") return creds, nil } @@ -116,9 +128,14 @@ func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) { func (fcw *FileConfigWrangler) SaveCredentials(creds WorkerCredentials) error { fcw.creds = &creds - err := fcw.writeConfig(credentialsFilename, "Credentials", creds) + filepath, err := appinfo.InFlamencoHome(credentialsFilename) if err != nil { - return fmt.Errorf("writing to %s: %w", credentialsFilename, err) + return err + } + + err = fcw.writeConfig(filepath, "Credentials", creds) + if err != nil { + return fmt.Errorf("writing to %s: %w", filepath, err) } return nil } @@ -182,7 +199,16 @@ func (fcw FileConfigWrangler) writeConfig(filename string, filetype string, conf // LoadConfig loads a YAML configuration file into 'config' func (fcw FileConfigWrangler) loadConfig(filename string, config interface{}) error { - log.Debug().Str("filename", filename).Msg("loading config file") + // Log which directory the config is loaded from. + filepath, err := filepath.Abs(filename) + if err != nil { + log.Warn().Err(err).Str("filename", filename). + Msg("config loader: unable to find absolute path of config file") + log.Debug().Str("filename", filename).Msg("loading config file") + } else { + log.Debug().Str("path", filepath).Msg("loading config file") + } + f, err := os.OpenFile(filename, os.O_RDONLY, 0) if err != nil { return err diff --git a/web/flamenco-io-site/content/usage/getting-started/_index.md b/web/flamenco-io-site/content/usage/getting-started/_index.md index cc57fdcf..2a0413b8 100644 --- a/web/flamenco-io-site/content/usage/getting-started/_index.md +++ b/web/flamenco-io-site/content/usage/getting-started/_index.md @@ -1,5 +1,6 @@ --- -title: "Getting Started" +title: Getting Started +weight: 0 --- This will be written when a release of Flamenco 3 can actually be downloaded. diff --git a/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md b/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md new file mode 100644 index 00000000..6a147a01 --- /dev/null +++ b/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md @@ -0,0 +1,17 @@ +--- +title: Job Compiler Scripts +weight: 10 +--- + +TODO: write this :) + +## Task Types + +The following task types are defined by the standard job compiler scripts: + +- `blender`: any task that runs Blender. +- `ffmpeg`: any task that runs FFmpeg. +- `file-management`: moving or copying files, creating directories, moving old + renders out of the way to make place for new ones, etc. +- `misc`: tasks mostly used for testing things, such as "log a debug message" or + "do nothing for 5 seconds". diff --git a/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md b/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md index 6b07b268..f4255635 100644 --- a/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md +++ b/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md @@ -1,5 +1,6 @@ --- title: Jobs, Tasks, and Commands +weight: 5 --- TODO: write about the pipeline from job submission to command execution. diff --git a/web/flamenco-io-site/content/usage/worker-configuration/_index.md b/web/flamenco-io-site/content/usage/worker-configuration/_index.md new file mode 100644 index 00000000..108780e6 --- /dev/null +++ b/web/flamenco-io-site/content/usage/worker-configuration/_index.md @@ -0,0 +1,45 @@ +--- +title: Worker Configuration +weight: 2 +--- + +Flamenco Worker will read its configuration from `flamenco-worker.yaml` in the +*current working directory*. + +This is an example of such a configuration file: + +```yaml +manager_url: http://flamenco.local:8080/ +task_types: blender, ffmpeg, file-management, misc +``` + +The task types are determined by the [job compiler scripts][scripts]. The ones +listed here are in use by the default scripts. These determine which kind of +tasks this Worker will get. See [task types][task-types] for more info. + +[scripts]: {{< ref "usage/job-compiler-scripts/_index.md" >}} +[task-types]: {{< ref "usage/job-compiler-scripts/_index.md" >}}#task-types + +## Worker Local Files + +Apart from the above configuration file, which can be shared between Workers, +each Worker has a set of files that are specific to that Worker. These contain +the *worker credentials*, which are used to identify this worker to the Manager, +and a *database file* to queue task updates when the Manager is unreachable. + +These files are stored in a platform-specific location: + +| Platform | Default location | +| -- | -- | +| Linux | `$HOME/.local/share/flamenco` | +| Windows | `C:\Users\UserName\AppData\Local\Flamenco` | +| macOS | `$HOME/Library/Application Support/Flamenco` | + +## Configuration from Environment Variables + +Certain settings can be configured via environment variables. + +- `FLAMENCO_HOME`: Directory for [Worker local files](#worker-local-files). If + not given, the above defaults are used. +- `FLAMENCO_WORKER_NAME`: The name of the Worker. If not specified, the Worker + will use the hostname.