T99415: Worker: change default location for writing local files
Change the location where the Worker writes its local files so that it follows the XDG specification (instead of writing to the current working directory). - Linux: `$HOME/.local/share/flamenco` - Windows: `C:\Users\UserName\AppData\Local\Flamenco` - macOS: `$HOME/Library/Application Support/Flamenco` NOTE: The old files will not be loaded any more. This means that if nothing is done and the new worker is run as-is, it will reregister as a brand new worker. Move `flamenco-worker-credentials.yaml` and `flamenco-worker.sqlite` to the new location to avoid this.
This commit is contained in:
parent
fa600d6fc9
commit
2f76df437b
@ -30,6 +30,8 @@ var (
|
||||
shutdownComplete chan struct{}
|
||||
)
|
||||
|
||||
const flamencoWorkerDatabase = "flamenco-worker.sqlite"
|
||||
|
||||
var cliArgs struct {
|
||||
// Do-and-quit flags.
|
||||
version bool
|
||||
@ -231,8 +233,12 @@ func upstreamBufferOrDie(client worker.FlamencoClient, timeService clock.Clock)
|
||||
log.Fatal().Err(err).Msg("unable to create task update queue database")
|
||||
}
|
||||
|
||||
// TODO: make filename configurable?
|
||||
if err := buffer.OpenDB(ctx, "flamenco-worker.sqlite"); err != nil {
|
||||
dbPath, err := appinfo.InFlamencoHome(flamencoWorkerDatabase)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("unable to figure out where to save my database")
|
||||
}
|
||||
|
||||
if err := buffer.OpenDB(ctx, dbPath); err != nil {
|
||||
log.Fatal().Err(err).Msg("unable to open task update queue database")
|
||||
}
|
||||
|
||||
|
1
go.mod
1
go.mod
@ -31,6 +31,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/adrg/xdg v0.4.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
|
3
go.sum
3
go.sum
@ -1,3 +1,5 @@
|
||||
github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
|
||||
github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
|
||||
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
|
||||
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
@ -217,6 +219,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
24
internal/appinfo/xdg_paths.go
Normal file
24
internal/appinfo/xdg_paths.go
Normal file
@ -0,0 +1,24 @@
|
||||
package appinfo
|
||||
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
)
|
||||
|
||||
// InFlamencoHome returns the filename in the 'flamenco home' dir, and ensures
|
||||
// that the directory exists.
|
||||
func InFlamencoHome(filename string) (string, error) {
|
||||
flamencoHome := os.Getenv("FLAMENCO_HOME")
|
||||
if flamencoHome == "" {
|
||||
return xdg.DataFile(path.Join(xdgApplicationName, filename))
|
||||
}
|
||||
if err := os.MkdirAll(flamencoHome, os.ModePerm); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(flamencoHome, filename), nil
|
||||
}
|
7
internal/appinfo/xdg_paths_lowercase.go
Normal file
7
internal/appinfo/xdg_paths_lowercase.go
Normal file
@ -0,0 +1,7 @@
|
||||
//go:build !(windows || darwin)
|
||||
|
||||
package appinfo
|
||||
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
const xdgApplicationName = "flamenco"
|
7
internal/appinfo/xdg_paths_titlecase.go
Normal file
7
internal/appinfo/xdg_paths_titlecase.go
Normal file
@ -0,0 +1,7 @@
|
||||
//go:build windows || darwin
|
||||
|
||||
package appinfo
|
||||
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
const xdgApplicationName = "Flamenco"
|
@ -9,9 +9,11 @@ import (
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.blender.org/flamenco/internal/appinfo"
|
||||
"github.com/rs/zerolog/log"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
@ -69,7 +71,12 @@ func (fcw *FileConfigWrangler) WorkerConfig() (WorkerConfig, error) {
|
||||
}
|
||||
|
||||
wc := fcw.DefaultConfig()
|
||||
err := fcw.loadConfig(configFilename, &wc)
|
||||
filepath, err := appinfo.InFlamencoHome(configFilename)
|
||||
if err != nil {
|
||||
return wc, err
|
||||
}
|
||||
|
||||
err = fcw.loadConfig(filepath, &wc)
|
||||
|
||||
if err != nil {
|
||||
switch {
|
||||
@ -101,14 +108,19 @@ func (fcw *FileConfigWrangler) SaveConfig() error {
|
||||
}
|
||||
|
||||
func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) {
|
||||
filepath, err := appinfo.InFlamencoHome(credentialsFilename)
|
||||
if err != nil {
|
||||
return WorkerCredentials{}, err
|
||||
}
|
||||
|
||||
var creds WorkerCredentials
|
||||
err := fcw.loadConfig(credentialsFilename, &creds)
|
||||
err = fcw.loadConfig(filepath, &creds)
|
||||
if err != nil {
|
||||
return WorkerCredentials{}, err
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("filename", credentialsFilename).
|
||||
Str("filename", filepath).
|
||||
Msg("loaded credentials")
|
||||
return creds, nil
|
||||
}
|
||||
@ -116,9 +128,14 @@ func (fcw *FileConfigWrangler) WorkerCredentials() (WorkerCredentials, error) {
|
||||
func (fcw *FileConfigWrangler) SaveCredentials(creds WorkerCredentials) error {
|
||||
fcw.creds = &creds
|
||||
|
||||
err := fcw.writeConfig(credentialsFilename, "Credentials", creds)
|
||||
filepath, err := appinfo.InFlamencoHome(credentialsFilename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing to %s: %w", credentialsFilename, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = fcw.writeConfig(filepath, "Credentials", creds)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing to %s: %w", filepath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -182,7 +199,16 @@ func (fcw FileConfigWrangler) writeConfig(filename string, filetype string, conf
|
||||
|
||||
// LoadConfig loads a YAML configuration file into 'config'
|
||||
func (fcw FileConfigWrangler) loadConfig(filename string, config interface{}) error {
|
||||
log.Debug().Str("filename", filename).Msg("loading config file")
|
||||
// Log which directory the config is loaded from.
|
||||
filepath, err := filepath.Abs(filename)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("filename", filename).
|
||||
Msg("config loader: unable to find absolute path of config file")
|
||||
log.Debug().Str("filename", filename).Msg("loading config file")
|
||||
} else {
|
||||
log.Debug().Str("path", filepath).Msg("loading config file")
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -1,5 +1,6 @@
|
||||
---
|
||||
title: "Getting Started"
|
||||
title: Getting Started
|
||||
weight: 0
|
||||
---
|
||||
|
||||
This will be written when a release of Flamenco 3 can actually be downloaded.
|
||||
|
@ -0,0 +1,17 @@
|
||||
---
|
||||
title: Job Compiler Scripts
|
||||
weight: 10
|
||||
---
|
||||
|
||||
TODO: write this :)
|
||||
|
||||
## Task Types
|
||||
|
||||
The following task types are defined by the standard job compiler scripts:
|
||||
|
||||
- `blender`: any task that runs Blender.
|
||||
- `ffmpeg`: any task that runs FFmpeg.
|
||||
- `file-management`: moving or copying files, creating directories, moving old
|
||||
renders out of the way to make place for new ones, etc.
|
||||
- `misc`: tasks mostly used for testing things, such as "log a debug message" or
|
||||
"do nothing for 5 seconds".
|
@ -1,5 +1,6 @@
|
||||
---
|
||||
title: Jobs, Tasks, and Commands
|
||||
weight: 5
|
||||
---
|
||||
|
||||
TODO: write about the pipeline from job submission to command execution.
|
||||
|
@ -0,0 +1,45 @@
|
||||
---
|
||||
title: Worker Configuration
|
||||
weight: 2
|
||||
---
|
||||
|
||||
Flamenco Worker will read its configuration from `flamenco-worker.yaml` in the
|
||||
*current working directory*.
|
||||
|
||||
This is an example of such a configuration file:
|
||||
|
||||
```yaml
|
||||
manager_url: http://flamenco.local:8080/
|
||||
task_types: blender, ffmpeg, file-management, misc
|
||||
```
|
||||
|
||||
The task types are determined by the [job compiler scripts][scripts]. The ones
|
||||
listed here are in use by the default scripts. These determine which kind of
|
||||
tasks this Worker will get. See [task types][task-types] for more info.
|
||||
|
||||
[scripts]: {{< ref "usage/job-compiler-scripts/_index.md" >}}
|
||||
[task-types]: {{< ref "usage/job-compiler-scripts/_index.md" >}}#task-types
|
||||
|
||||
## Worker Local Files
|
||||
|
||||
Apart from the above configuration file, which can be shared between Workers,
|
||||
each Worker has a set of files that are specific to that Worker. These contain
|
||||
the *worker credentials*, which are used to identify this worker to the Manager,
|
||||
and a *database file* to queue task updates when the Manager is unreachable.
|
||||
|
||||
These files are stored in a platform-specific location:
|
||||
|
||||
| Platform | Default location |
|
||||
| -- | -- |
|
||||
| Linux | `$HOME/.local/share/flamenco` |
|
||||
| Windows | `C:\Users\UserName\AppData\Local\Flamenco` |
|
||||
| macOS | `$HOME/Library/Application Support/Flamenco` |
|
||||
|
||||
## Configuration from Environment Variables
|
||||
|
||||
Certain settings can be configured via environment variables.
|
||||
|
||||
- `FLAMENCO_HOME`: Directory for [Worker local files](#worker-local-files). If
|
||||
not given, the above defaults are used.
|
||||
- `FLAMENCO_WORKER_NAME`: The name of the Worker. If not specified, the Worker
|
||||
will use the hostname.
|
Loading…
Reference in New Issue
Block a user