From 1dcfd1e3298d5415d962e35f53d206b1f5845222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Wygoda?= Date: Fri, 30 Jul 2021 17:41:54 +0200 Subject: [PATCH] nixos/litestream: init --- nixos/modules/module-list.nix | 1 + .../litestream/default.nix | 100 ++++++++++++++++++ .../litestream/litestream.xml | 65 ++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/litestream.nix | 93 ++++++++++++++++ 5 files changed, 260 insertions(+) create mode 100644 nixos/modules/services/network-filesystems/litestream/default.nix create mode 100644 nixos/modules/services/network-filesystems/litestream/litestream.xml create mode 100644 nixos/tests/litestream.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 980e7027c98c..fd35dbb83af7 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -631,6 +631,7 @@ ./services/network-filesystems/glusterfs.nix ./services/network-filesystems/kbfs.nix ./services/network-filesystems/ipfs.nix + ./services/network-filesystems/litestream/default.nix ./services/network-filesystems/netatalk.nix ./services/network-filesystems/nfsd.nix ./services/network-filesystems/openafs/client.nix diff --git a/nixos/modules/services/network-filesystems/litestream/default.nix b/nixos/modules/services/network-filesystems/litestream/default.nix new file mode 100644 index 000000000000..f1806c5af0a9 --- /dev/null +++ b/nixos/modules/services/network-filesystems/litestream/default.nix @@ -0,0 +1,100 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.litestream; + settingsFormat = pkgs.formats.yaml {}; +in +{ + options.services.litestream = { + enable = mkEnableOption "litestream"; + + package = mkOption { + description = "Package to use."; + default = pkgs.litestream; + defaultText = "pkgs.litestream"; + type = types.package; + }; + + settings = mkOption { + description = '' + See the documentation. + ''; + type = settingsFormat.type; + example = { + dbs = [ + { + path = "/var/lib/db1"; + replicas = [ + { + url = "s3://mybkt.litestream.io/db1"; + } + ]; + } + ]; + }; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + example = "/run/secrets/litestream"; + description = '' + Environment file as defined in + systemd.exec5 + . + + Secrets may be passed to the service without adding them to the + world-readable Nix store, by specifying placeholder variables as + the option value in Nix and setting these variables accordingly in the + environment file. + + By default, Litestream will perform environment variable expansion + within the config file before reading it. Any references to ''$VAR or + ''${VAR} formatted variables will be replaced with their environment + variable values. If no value is set then it will be replaced with an + empty string. + + + # Content of the environment file + LITESTREAM_ACCESS_KEY_ID=AKIAxxxxxxxxxxxxxxxx + LITESTREAM_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxx + + + Note that this file needs to be available on the host on which + this exporter is running. + ''; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ cfg.package ]; + environment.etc = { + "litestream.yml" = { + source = settingsFormat.generate "litestream-config.yaml" cfg.settings; + }; + }; + + systemd.services.litestream = { + description = "Litestream"; + wantedBy = [ "multi-user.target" ]; + after = [ "networking.target" ]; + serviceConfig = { + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + ExecStart = "${cfg.package}/bin/litestream replicate"; + Restart = "always"; + User = "litestream"; + Group = "litestream"; + }; + }; + + users.users.litestream = { + description = "Litestream user"; + group = "litestream"; + isSystemUser = true; + }; + users.groups.litestream = {}; + }; + meta.doc = ./litestream.xml; +} diff --git a/nixos/modules/services/network-filesystems/litestream/litestream.xml b/nixos/modules/services/network-filesystems/litestream/litestream.xml new file mode 100644 index 000000000000..598f9be8cf63 --- /dev/null +++ b/nixos/modules/services/network-filesystems/litestream/litestream.xml @@ -0,0 +1,65 @@ + + Litestream + + Litestream is a standalone streaming + replication tool for SQLite. + + +
+ Configuration + + + Litestream service is managed by a dedicated user named litestream + which needs permission to the database file. Here's an example config which gives + required permissions to access + grafana database: + +{ pkgs, ... }: +{ + users.users.litestream.extraGroups = [ "grafana" ]; + + systemd.services.grafana.serviceConfig.ExecStartPost = "+" + pkgs.writeShellScript "grant-grafana-permissions" '' + timeout=10 + + while [ ! -f /var/lib/grafana/data/grafana.db ]; + do + if [ "$timeout" == 0 ]; then + echo "ERROR: Timeout while waiting for /var/lib/grafana/data/grafana.db." + exit 1 + fi + + sleep 1 + + ((timeout--)) + done + + find /var/lib/grafana -type d -exec chmod -v 775 {} \; + find /var/lib/grafana -type f -exec chmod -v 660 {} \; + ''; + + services.litestream = { + enable = true; + + environmentFile = "/run/secrets/litestream"; + + settings = { + dbs = [ + { + path = "/var/lib/grafana/data/grafana.db"; + replicas = [{ + url = "s3://mybkt.litestream.io/grafana"; + }]; + } + ]; + }; + }; +} + + +
+ + diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 741606732144..9eeb36b09751 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -224,6 +224,7 @@ in libreswan = handleTest ./libreswan.nix {}; lightdm = handleTest ./lightdm.nix {}; limesurvey = handleTest ./limesurvey.nix {}; + litestream = handleTest ./litestream.nix {}; locate = handleTest ./locate.nix {}; login = handleTest ./login.nix {}; loki = handleTest ./loki.nix {}; diff --git a/nixos/tests/litestream.nix b/nixos/tests/litestream.nix new file mode 100644 index 000000000000..886fbfef9cf5 --- /dev/null +++ b/nixos/tests/litestream.nix @@ -0,0 +1,93 @@ +import ./make-test-python.nix ({ pkgs, ...} : { + name = "litestream"; + meta = with pkgs.lib.maintainers; { + maintainers = [ jwygoda ]; + }; + + machine = + { pkgs, ... }: + { services.litestream = { + enable = true; + settings = { + dbs = [ + { + path = "/var/lib/grafana/data/grafana.db"; + replicas = [{ + url = "sftp://foo:bar@127.0.0.1:22/home/foo/grafana"; + }]; + } + ]; + }; + }; + systemd.services.grafana.serviceConfig.ExecStartPost = "+" + pkgs.writeShellScript "grant-grafana-permissions" '' + timeout=10 + + while [ ! -f /var/lib/grafana/data/grafana.db ]; + do + if [ "$timeout" == 0 ]; then + echo "ERROR: Timeout while waiting for /var/lib/grafana/data/grafana.db." + exit 1 + fi + + sleep 1 + + ((timeout--)) + done + + find /var/lib/grafana -type d -exec chmod -v 775 {} \; + find /var/lib/grafana -type f -exec chmod -v 660 {} \; + ''; + services.openssh = { + enable = true; + allowSFTP = true; + listenAddresses = [ { addr = "127.0.0.1"; port = 22; } ]; + }; + services.grafana = { + enable = true; + security = { + adminUser = "admin"; + adminPassword = "admin"; + }; + addr = "localhost"; + port = 3000; + extraOptions = { + DATABASE_URL = "sqlite3:///var/lib/grafana/data/grafana.db?cache=private&mode=rwc&_journal_mode=WAL"; + }; + }; + users.users.foo = { + isNormalUser = true; + password = "bar"; + }; + users.users.litestream.extraGroups = [ "grafana" ]; + }; + + testScript = '' + start_all() + machine.wait_until_succeeds("test -d /home/foo/grafana") + machine.wait_for_open_port(3000) + machine.succeed(""" + curl -sSfN -X PUT -H "Content-Type: application/json" -d '{ + "oldPassword": "admin", + "newPassword": "newpass", + "confirmNew": "newpass" + }' http://admin:admin@127.0.0.1:3000/api/user/password + """) + # https://litestream.io/guides/systemd/#simulating-a-disaster + machine.systemctl("stop litestream.service") + machine.succeed( + "rm -f /var/lib/grafana/data/grafana.db " + "/var/lib/grafana/data/grafana.db-shm " + "/var/lib/grafana/data/grafana.db-wal" + ) + machine.succeed( + "litestream restore /var/lib/grafana/data/grafana.db " + "&& chown grafana:grafana /var/lib/grafana/data/grafana.db " + "&& chmod 660 /var/lib/grafana/data/grafana.db" + ) + machine.systemctl("restart grafana.service") + machine.wait_for_open_port(3000) + machine.succeed( + "curl -sSfN -u admin:newpass http://127.0.0.1:3000/api/org/users | grep admin\@localhost" + ) + ''; +})