From 2f76df437bd2bb1bc71b3bac933efdcf0139db6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 19 Jul 2022 12:00:04 +0200 Subject: [PATCH] T99415: Worker: change default location for writing local files Change the location where the Worker writes its local files so that it follows the XDG specification (instead of writing to the current working directory). - Linux: `$HOME/.local/share/flamenco` - Windows: `C:\Users\UserName\AppData\Local\Flamenco` - macOS: `$HOME/Library/Application Support/Flamenco` NOTE: The old files will not be loaded any more. This means that if nothing is done and the new worker is run as-is, it will reregister as a brand new worker. Move `flamenco-worker-credentials.yaml` and `flamenco-worker.sqlite` to the new location to avoid this. --- cmd/flamenco-worker/main.go | 10 ++++- go.mod | 1 + go.sum | 3 ++ internal/appinfo/xdg_paths.go | 24 ++++++++++ internal/appinfo/xdg_paths_lowercase.go | 7 +++ internal/appinfo/xdg_paths_titlecase.go | 7 +++ internal/worker/config.go | 38 +++++++++++++--- .../content/usage/getting-started/_index.md | 3 +- .../usage/job-compiler-scripts/_index.md | 17 +++++++ .../usage/jobs-tasks-commands/_index.md | 1 + .../usage/worker-configuration/_index.md | 45 +++++++++++++++++++ 11 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 internal/appinfo/xdg_paths.go create mode 100644 internal/appinfo/xdg_paths_lowercase.go create mode 100644 internal/appinfo/xdg_paths_titlecase.go create mode 100644 web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md create mode 100644 web/flamenco-io-site/content/usage/worker-configuration/_index.md diff --git a/cmd/flamenco-worker/main.go b/cmd/flamenco-worker/main.go index ad95c5d0..1c523ae2 100644 --- a/cmd/flamenco-worker/main.go +++ b/cmd/flamenco-worker/main.go @@ -30,6 +30,8 @@ var ( shutdownComplete chan struct{} ) +const flamencoWorkerDatabase = "flamenco-worker.sqlite" + var cliArgs struct { // Do-and-quit flags. version bool @@ -231,8 +233,12 @@ func upstreamBufferOrDie(client worker.FlamencoClient, timeService clock.Clock) log.Fatal().Err(err).Msg("unable to create task update queue database") } - // TODO: make filename configurable? - if err := buffer.OpenDB(ctx, "flamenco-worker.sqlite"); err != nil { + dbPath, err := appinfo.InFlamencoHome(flamencoWorkerDatabase) + if err != nil { + log.Fatal().Err(err).Msg("unable to figure out where to save my database") + } + + if err := buffer.OpenDB(ctx, dbPath); err != nil { log.Fatal().Err(err).Msg("unable to open task update queue database") } diff --git a/go.mod b/go.mod index 0a5e1057..1b690c49 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( ) require ( + github.com/adrg/xdg v0.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/ghodss/yaml v1.0.0 // indirect diff --git a/go.sum b/go.sum index 033134ff..9fd03f7f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -217,6 +219,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/appinfo/xdg_paths.go b/internal/appinfo/xdg_paths.go new file mode 100644 index 00000000..114e505b --- /dev/null +++ b/internal/appinfo/xdg_paths.go @@ -0,0 +1,24 @@ +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "os" + "path" + "path/filepath" + + "github.com/adrg/xdg" +) + +// InFlamencoHome returns the filename in the 'flamenco home' dir, and ensures +// that the directory exists. +func InFlamencoHome(filename string) (string, error) { + flamencoHome := os.Getenv("FLAMENCO_HOME") + if flamencoHome == "" { + return xdg.DataFile(path.Join(xdgApplicationName, filename)) + } + if err := os.MkdirAll(flamencoHome, os.ModePerm); err != nil { + return "", err + } + return filepath.Join(flamencoHome, filename), nil +} diff --git a/internal/appinfo/xdg_paths_lowercase.go b/internal/appinfo/xdg_paths_lowercase.go new file mode 100644 index 00000000..6585dfda --- /dev/null +++ b/internal/appinfo/xdg_paths_lowercase.go @@ -0,0 +1,7 @@ +//go:build !(windows || darwin) + +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +const xdgApplicationName = "flamenco" diff --git a/internal/appinfo/xdg_paths_titlecase.go b/internal/appinfo/xdg_paths_titlecase.go new file mode 100644 index 00000000..6eb1a41d --- /dev/null +++ b/internal/appinfo/xdg_paths_titlecase.go @@ -0,0 +1,7 @@ +//go:build windows || darwin + +package appinfo + +// SPDX-License-Identifier: GPL-3.0-or-later + +const xdgApplicationName = "Flamenco" diff --git a/internal/worker/config.go b/internal/worker/config.go index 5c9e2e77..00f9a102 100644 --- a/internal/worker/config.go +++ b/internal/worker/config.go @@ -9,9 +9,11 @@ import ( "io/fs" "net/url" "os" + "path/filepath" "strings" "time" + "git.blender.org/flamenco/internal/appinfo" "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v2" ) @@ -69,7 +71,12 @@ func (fcw *FileConfigWrangler) WorkerConfig() (WorkerConfig, error) { } wc := fcw.DefaultConfig() - err := fcw.loadConfig(configFilename, &wc) + filepath, err := appinfo.InFlamencoHome(configFilename) + if err != nil { + return wc, err + } + + err = fcw.loadConfig(filepath, &wc) if err != nil { switch { @@ -101,14 +108,19 @@ func (fcw *FileConfigWrangler) SaveConfig() error { } func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) { + filepath, err := appinfo.InFlamencoHome(credentialsFilename) + if err != nil { + return WorkerCredentials{}, err + } + var creds WorkerCredentials - err := fcw.loadConfig(credentialsFilename, &creds) + err = fcw.loadConfig(filepath, &creds) if err != nil { return WorkerCredentials{}, err } log.Info(). - Str("filename", credentialsFilename). + Str("filename", filepath). Msg("loaded credentials") return creds, nil } @@ -116,9 +128,14 @@ func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) { func (fcw *FileConfigWrangler) SaveCredentials(creds WorkerCredentials) error { fcw.creds = &creds - err := fcw.writeConfig(credentialsFilename, "Credentials", creds) + filepath, err := appinfo.InFlamencoHome(credentialsFilename) if err != nil { - return fmt.Errorf("writing to %s: %w", credentialsFilename, err) + return err + } + + err = fcw.writeConfig(filepath, "Credentials", creds) + if err != nil { + return fmt.Errorf("writing to %s: %w", filepath, err) } return nil } @@ -182,7 +199,16 @@ func (fcw FileConfigWrangler) writeConfig(filename string, filetype string, conf // LoadConfig loads a YAML configuration file into 'config' func (fcw FileConfigWrangler) loadConfig(filename string, config interface{}) error { - log.Debug().Str("filename", filename).Msg("loading config file") + // Log which directory the config is loaded from. + filepath, err := filepath.Abs(filename) + if err != nil { + log.Warn().Err(err).Str("filename", filename). + Msg("config loader: unable to find absolute path of config file") + log.Debug().Str("filename", filename).Msg("loading config file") + } else { + log.Debug().Str("path", filepath).Msg("loading config file") + } + f, err := os.OpenFile(filename, os.O_RDONLY, 0) if err != nil { return err diff --git a/web/flamenco-io-site/content/usage/getting-started/_index.md b/web/flamenco-io-site/content/usage/getting-started/_index.md index cc57fdcf..2a0413b8 100644 --- a/web/flamenco-io-site/content/usage/getting-started/_index.md +++ b/web/flamenco-io-site/content/usage/getting-started/_index.md @@ -1,5 +1,6 @@ --- -title: "Getting Started" +title: Getting Started +weight: 0 --- This will be written when a release of Flamenco 3 can actually be downloaded. diff --git a/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md b/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md new file mode 100644 index 00000000..6a147a01 --- /dev/null +++ b/web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md @@ -0,0 +1,17 @@ +--- +title: Job Compiler Scripts +weight: 10 +--- + +TODO: write this :) + +## Task Types + +The following task types are defined by the standard job compiler scripts: + +- `blender`: any task that runs Blender. +- `ffmpeg`: any task that runs FFmpeg. +- `file-management`: moving or copying files, creating directories, moving old + renders out of the way to make place for new ones, etc. +- `misc`: tasks mostly used for testing things, such as "log a debug message" or + "do nothing for 5 seconds". diff --git a/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md b/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md index 6b07b268..f4255635 100644 --- a/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md +++ b/web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md @@ -1,5 +1,6 @@ --- title: Jobs, Tasks, and Commands +weight: 5 --- TODO: write about the pipeline from job submission to command execution. diff --git a/web/flamenco-io-site/content/usage/worker-configuration/_index.md b/web/flamenco-io-site/content/usage/worker-configuration/_index.md new file mode 100644 index 00000000..108780e6 --- /dev/null +++ b/web/flamenco-io-site/content/usage/worker-configuration/_index.md @@ -0,0 +1,45 @@ +--- +title: Worker Configuration +weight: 2 +--- + +Flamenco Worker will read its configuration from `flamenco-worker.yaml` in the +*current working directory*. + +This is an example of such a configuration file: + +```yaml +manager_url: http://flamenco.local:8080/ +task_types: blender, ffmpeg, file-management, misc +``` + +The task types are determined by the [job compiler scripts][scripts]. The ones +listed here are in use by the default scripts. These determine which kind of +tasks this Worker will get. See [task types][task-types] for more info. + +[scripts]: {{< ref "usage/job-compiler-scripts/_index.md" >}} +[task-types]: {{< ref "usage/job-compiler-scripts/_index.md" >}}#task-types + +## Worker Local Files + +Apart from the above configuration file, which can be shared between Workers, +each Worker has a set of files that are specific to that Worker. These contain +the *worker credentials*, which are used to identify this worker to the Manager, +and a *database file* to queue task updates when the Manager is unreachable. + +These files are stored in a platform-specific location: + +| Platform | Default location | +| -- | -- | +| Linux | `$HOME/.local/share/flamenco` | +| Windows | `C:\Users\UserName\AppData\Local\Flamenco` | +| macOS | `$HOME/Library/Application Support/Flamenco` | + +## Configuration from Environment Variables + +Certain settings can be configured via environment variables. + +- `FLAMENCO_HOME`: Directory for [Worker local files](#worker-local-files). If + not given, the above defaults are used. +- `FLAMENCO_WORKER_NAME`: The name of the Worker. If not specified, the Worker + will use the hostname.