T99415: Worker: change default location for writing local files

Change the location where the Worker writes its local files so that it
follows the XDG specification (instead of writing to the current working
directory).

- Linux:   `$HOME/.local/share/flamenco`
- Windows: `C:\Users\UserName\AppData\Local\Flamenco`
- macOS:   `$HOME/Library/Application Support/Flamenco`

NOTE: The old files will not be loaded any more. This means that if
nothing is done and the new worker is run as-is, it will reregister as a
brand new worker. Move `flamenco-worker-credentials.yaml` and
`flamenco-worker.sqlite` to the new location to avoid this.

cmd/flamenco-worker/main.go

@@ -30,6 +30,8 @@ var (
 	shutdownComplete chan struct{}
 )
 
+const flamencoWorkerDatabase = "flamenco-worker.sqlite"
+
 var cliArgs struct {
 	// Do-and-quit flags.
 	version bool
@@ -231,8 +233,12 @@ func upstreamBufferOrDie(client worker.FlamencoClient, timeService clock.Clock)
 		log.Fatal().Err(err).Msg("unable to create task update queue database")
 	}
 
-	// TODO: make filename configurable?
-	if err := buffer.OpenDB(ctx, "flamenco-worker.sqlite"); err != nil {
+	dbPath, err := appinfo.InFlamencoHome(flamencoWorkerDatabase)
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to figure out where to save my database")
+	}
+
+	if err := buffer.OpenDB(ctx, dbPath); err != nil {
 		log.Fatal().Err(err).Msg("unable to open task update queue database")
 	}
 

go.mod

@@ -31,6 +31,7 @@ require (
 )
 
 require (
+	github.com/adrg/xdg v0.4.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
+github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -217,6 +219,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/appinfo/xdg_paths.go

@@ -0,0 +1,24 @@
+package appinfo
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/adrg/xdg"
+)
+
+// InFlamencoHome returns the filename in the 'flamenco home' dir, and ensures
+// that the directory exists.
+func InFlamencoHome(filename string) (string, error) {
+	flamencoHome := os.Getenv("FLAMENCO_HOME")
+	if flamencoHome == "" {
+		return xdg.DataFile(path.Join(xdgApplicationName, filename))
+	}
+	if err := os.MkdirAll(flamencoHome, os.ModePerm); err != nil {
+		return "", err
+	}
+	return filepath.Join(flamencoHome, filename), nil
+}

internal/appinfo/xdg_paths_lowercase.go

@@ -0,0 +1,7 @@
+//go:build !(windows || darwin)
+
+package appinfo
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+const xdgApplicationName = "flamenco"

internal/appinfo/xdg_paths_titlecase.go

@@ -0,0 +1,7 @@
+//go:build windows || darwin
+
+package appinfo
+
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+const xdgApplicationName = "Flamenco"

internal/worker/config.go

@@ -9,9 +9,11 @@ import (
 	"io/fs"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"git.blender.org/flamenco/internal/appinfo"
 	"github.com/rs/zerolog/log"
 	yaml "gopkg.in/yaml.v2"
 )
@@ -69,7 +71,12 @@ func (fcw *FileConfigWrangler) WorkerConfig() (WorkerConfig, error) {
 	}
 
 	wc := fcw.DefaultConfig()
-	err := fcw.loadConfig(configFilename, &wc)
+	filepath, err := appinfo.InFlamencoHome(configFilename)
+	if err != nil {
+		return wc, err
+	}
+
+	err = fcw.loadConfig(filepath, &wc)
 
 	if err != nil {
 		switch {
@@ -101,14 +108,19 @@ func (fcw *FileConfigWrangler) SaveConfig() error {
 }
 
 func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) {
+	filepath, err := appinfo.InFlamencoHome(credentialsFilename)
+	if err != nil {
+		return WorkerCredentials{}, err
+	}
+
 	var creds WorkerCredentials
-	err := fcw.loadConfig(credentialsFilename, &creds)
+	err = fcw.loadConfig(filepath, &creds)
 	if err != nil {
 		return WorkerCredentials{}, err
 	}
 
 	log.Info().
-		Str("filename", credentialsFilename).
+		Str("filename", filepath).
 		Msg("loaded credentials")
 	return creds, nil
 }
@@ -116,9 +128,14 @@ func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) {
 func (fcw *FileConfigWrangler) SaveCredentials(creds WorkerCredentials) error {
 	fcw.creds = &creds
 
-	err := fcw.writeConfig(credentialsFilename, "Credentials", creds)
+	filepath, err := appinfo.InFlamencoHome(credentialsFilename)
 	if err != nil {
-		return fmt.Errorf("writing to %s: %w", credentialsFilename, err)
+		return err
+	}
+
+	err = fcw.writeConfig(filepath, "Credentials", creds)
+	if err != nil {
+		return fmt.Errorf("writing to %s: %w", filepath, err)
 	}
 	return nil
 }
@@ -182,7 +199,16 @@ func (fcw FileConfigWrangler) writeConfig(filename string, filetype string, conf
 
 // LoadConfig loads a YAML configuration file into 'config'
 func (fcw FileConfigWrangler) loadConfig(filename string, config interface{}) error {
+	// Log which directory the config is loaded from.
+	filepath, err := filepath.Abs(filename)
+	if err != nil {
+		log.Warn().Err(err).Str("filename", filename).
+			Msg("config loader: unable to find absolute path of config file")
 		log.Debug().Str("filename", filename).Msg("loading config file")
+	} else {
+		log.Debug().Str("path", filepath).Msg("loading config file")
+	}
+
 	f, err := os.OpenFile(filename, os.O_RDONLY, 0)
 	if err != nil {
 		return err

web/flamenco-io-site/content/usage/getting-started/_index.md

@@ -1,5 +1,6 @@
 ---
-title: "Getting Started"
+title: Getting Started
+weight: 0
 ---
 
 This will be written when a release of Flamenco 3 can actually be downloaded.

web/flamenco-io-site/content/usage/job-compiler-scripts/_index.md

@@ -0,0 +1,17 @@
+---
+title: Job Compiler Scripts
+weight: 10
+---
+
+TODO: write this :)
+
+## Task Types
+
+The following task types are defined by the standard job compiler scripts:
+
+- `blender`: any task that runs Blender.
+- `ffmpeg`: any task that runs FFmpeg.
+- `file-management`: moving or copying files, creating directories, moving old
+  renders out of the way to make place for new ones, etc.
+- `misc`: tasks mostly used for testing things, such as "log a debug message" or
+  "do nothing for 5 seconds".

web/flamenco-io-site/content/usage/jobs-tasks-commands/_index.md

@@ -1,5 +1,6 @@
 ---
 title: Jobs, Tasks, and Commands
+weight: 5
 ---
 
 TODO: write about the pipeline from job submission to command execution.

web/flamenco-io-site/content/usage/worker-configuration/_index.md

@@ -0,0 +1,45 @@
+---
+title: Worker Configuration
+weight: 2
+---
+
+Flamenco Worker will read its configuration from `flamenco-worker.yaml` in the
+*current working directory*.
+
+This is an example of such a configuration file:
+
+```yaml
+manager_url: http://flamenco.local:8080/
+task_types: blender, ffmpeg, file-management, misc
+```
+
+The task types are determined by the [job compiler scripts][scripts]. The ones
+listed here are in use by the default scripts. These determine which kind of
+tasks this Worker will get. See [task types][task-types] for more info.
+
+[scripts]: {{< ref "usage/job-compiler-scripts/_index.md" >}}
+[task-types]: {{< ref "usage/job-compiler-scripts/_index.md" >}}#task-types
+
+## Worker Local Files
+
+Apart from the above configuration file, which can be shared between Workers,
+each Worker has a set of files that are specific to that Worker. These contain
+the *worker credentials*, which are used to identify this worker to the Manager,
+and a *database file* to queue task updates when the Manager is unreachable.
+
+These files are stored in a platform-specific location:
+
+| Platform | Default location |
+| -- | -- |
+| Linux | `$HOME/.local/share/flamenco` |
+| Windows | `C:\Users\UserName\AppData\Local\Flamenco` |
+| macOS | `$HOME/Library/Application Support/Flamenco` |
+
+## Configuration from Environment Variables
+
+Certain settings can be configured via environment variables.
+
+- `FLAMENCO_HOME`: Directory for [Worker local files](#worker-local-files). If
+  not given, the above defaults are used.
+- `FLAMENCO_WORKER_NAME`: The name of the Worker. If not specified, the Worker
+  will use the hostname.