From ead6de5d3d71a935b0aa9d96d2c750296a79926f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Wed, 26 Aug 2020 10:51:10 +0200 Subject: [PATCH] tests/postgresql-wal-receiver: Port to Python ... and remove some weirdnesses. - Port to Python - Drop the extra pkgs, config, system args - Drop all `with` - Don't override the standard PostgreSQL directory - Use pkgs and lib from the test runner Tested with: - postgresql_12 - postgresql_11 - postgresql_10 - postgresql_9_6 - postgresql_9_5 Closes #96347 cc @flokli --- nixos/tests/postgresql-wal-receiver.nix | 202 ++++++++++++------------ 1 file changed, 105 insertions(+), 97 deletions(-) diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix index c50746aa838e..3be95187fe2c 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -1,103 +1,111 @@ -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } }: - -with import ../lib/testing.nix { inherit system pkgs; }; -with pkgs.lib; - let - makePostgresqlWalReceiverTest = subTestName: postgresqlPackage: let + # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. + makePostgresqlWalReceiverTest = postgresqlPackage: + { + name = postgresqlPackage; + value = + import ./make-test-python.nix ({ pkgs, lib, ... }: let - postgresqlDataDir = "/var/db/postgresql/test"; - replicationUser = "wal_receiver_user"; - replicationSlot = "wal_receiver_slot"; - replicationConn = "postgresql://${replicationUser}@localhost"; - baseBackupDir = "/tmp/pg_basebackup"; - walBackupDir = "/tmp/pg_wal"; - atLeast12 = versionAtLeast postgresqlPackage.version "12.0"; - restoreCommand = '' - restore_command = 'cp ${walBackupDir}/%f %p' - ''; - - recoveryFile = if atLeast12 - then pkgs.writeTextDir "recovery.signal" "" - else pkgs.writeTextDir "recovery.conf" "${restoreCommand}"; - - in makeTest { - name = "postgresql-wal-receiver-${subTestName}"; - meta.maintainers = with maintainers; [ pacien ]; - - machine = { ... }: { - # Needed because this test uses a non-default 'services.postgresql.dataDir'. - systemd.tmpfiles.rules = [ - "d /var/db/postgresql 0700 postgres postgres" - ]; - services.postgresql = { - package = postgresqlPackage; - enable = true; - dataDir = postgresqlDataDir; - extraConfig = '' - wal_level = archive # alias for replica on pg >= 9.6 - max_wal_senders = 10 - max_replication_slots = 10 - '' + optionalString atLeast12 '' - ${restoreCommand} - recovery_end_command = 'touch recovery.done' + pkg = pkgs."${postgresqlPackage}"; + postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; + replicationUser = "wal_receiver_user"; + replicationSlot = "wal_receiver_slot"; + replicationConn = "postgresql://${replicationUser}@localhost"; + baseBackupDir = "/tmp/pg_basebackup"; + walBackupDir = "/tmp/pg_wal"; + atLeast12 = lib.versionAtLeast pkg.version "12.0"; + restoreCommand = '' + restore_command = 'cp ${walBackupDir}/%f %p' ''; - authentication = '' - host replication ${replicationUser} all trust - ''; - initialScript = pkgs.writeText "init.sql" '' - create user ${replicationUser} replication; - select * from pg_create_physical_replication_slot('${replicationSlot}'); - ''; - }; - services.postgresqlWalReceiver.receivers.main = { - inherit postgresqlPackage; - connection = replicationConn; - slot = replicationSlot; - directory = walBackupDir; - }; - # This is only to speedup test, it isn't time racing. Service is set to autorestart always, - # default 60sec is fine for real system, but is too much for a test - systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = mkForce 5; + recoveryFile = if atLeast12 + then pkgs.writeTextDir "recovery.signal" "" + else pkgs.writeTextDir "recovery.conf" "${restoreCommand}"; + + in { + name = "postgresql-wal-receiver-${postgresqlPackage}"; + meta.maintainers = with lib.maintainers; [ pacien ]; + + machine = { ... }: { + services.postgresql = { + package = pkg; + enable = true; + extraConfig = '' + wal_level = archive # alias for replica on pg >= 9.6 + max_wal_senders = 10 + max_replication_slots = 10 + '' + lib.optionalString atLeast12 '' + ${restoreCommand} + recovery_end_command = 'touch recovery.done' + ''; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; + }; + + services.postgresqlWalReceiver.receivers.main = { + postgresqlPackage = pkg; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + # This is only to speedup test, it isn't time racing. Service is set to autorestart always, + # default 60sec is fine for real system, but is too much for a test + systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; + }; + + testScript = '' + # make an initial base backup + machine.wait_for_unit("postgresql") + machine.wait_for_unit("postgresql-wal-receiver-main") + # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other + # required only for 9.4 + machine.sleep(5) + machine.succeed( + "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" + ) + + # create a dummy table with 100 records + machine.succeed( + "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" + ) + + # stop postgres and destroy data + machine.systemctl("stop postgresql") + machine.systemctl("stop postgresql-wal-receiver-main") + machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") + + # restore the base backup + machine.succeed( + "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" + ) + + # prepare WAL and recovery + machine.succeed("chmod a+rX -R ${walBackupDir}") + machine.execute( + "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" + ) # make use of partial segments too + machine.succeed( + "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" + ) + + # replay WAL + machine.systemctl("start postgresql") + machine.wait_for_file("${postgresqlDataDir}/recovery.done") + machine.systemctl("restart postgresql") + machine.wait_for_unit("postgresql") + + # check that our records have been restored + machine.succeed( + "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" + ) + ''; + }); }; - testScript = '' - # make an initial base backup - $machine->waitForUnit('postgresql'); - $machine->waitForUnit('postgresql-wal-receiver-main'); - # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other - # required only for 9.4 - $machine->sleep(5); - $machine->succeed('${postgresqlPackage}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}'); - - # create a dummy table with 100 records - $machine->succeed('sudo -u postgres psql --command="create table dummy as select * from generate_series(1, 100) as val;"'); - - # stop postgres and destroy data - $machine->systemctl('stop postgresql'); - $machine->systemctl('stop postgresql-wal-receiver-main'); - $machine->succeed('rm -r ${postgresqlDataDir}/{base,global,pg_*}'); - - # restore the base backup - $machine->succeed('cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}'); - - # prepare WAL and recovery - $machine->succeed('chmod a+rX -R ${walBackupDir}'); - $machine->execute('for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done'); # make use of partial segments too - $machine->succeed('cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*'); - - # replay WAL - $machine->systemctl('start postgresql'); - $machine->waitForFile('${postgresqlDataDir}/recovery.done'); - $machine->systemctl('restart postgresql'); - $machine->waitForUnit('postgresql'); - - # check that our records have been restored - $machine->succeed('test $(sudo -u postgres psql --pset="pager=off" --tuples-only --command="select count(distinct val) from dummy;") -eq 100'); - ''; - }; - -in mapAttrs makePostgresqlWalReceiverTest (import ../../pkgs/servers/sql/postgresql pkgs) +# Maps the generic function over all attributes of PostgreSQL packages +in builtins.listToAttrs (map makePostgresqlWalReceiverTest (builtins.attrNames (import ../../pkgs/servers/sql/postgresql { })))