Merge pull request #246354 from minijackson/netbox-fixes

netbox: 3.5.6 -> 3.5.7 + migration fixes + upgrade NixOS test

nixos/modules/services/web-apps/netbox.nix

@@ -1,7 +1,5 @@
 { config, lib, pkgs, ... }:
 
-with lib;
-
 let
   cfg = config.services.netbox;
   pythonFmt = pkgs.formats.pythonVars {};
@@ -17,7 +15,7 @@ let
   pkg = (cfg.package.overrideAttrs (old: {
     installPhase = old.installPhase + ''
       ln -s ${configFile} $out/opt/netbox/netbox/netbox/configuration.py
-    '' + optionalString cfg.enableLdap ''
+    '' + lib.optionalString cfg.enableLdap ''
       ln -s ${cfg.ldapConfigPath} $out/opt/netbox/netbox/netbox/ldap_config.py
     '';
   })).override {
@@ -31,7 +29,7 @@ let
 
 in {
   options.services.netbox = {
-    enable = mkOption {
+    enable = lib.mkOption {
       type = lib.types.bool;
       default = false;
       description = lib.mdDoc ''
@@ -66,18 +64,18 @@ in {
       };
     };
 
-    listenAddress = mkOption {
-      type = types.str;
+    listenAddress = lib.mkOption {
+      type = lib.types.str;
       default = "[::1]";
       description = lib.mdDoc ''
         Address the server will listen on.
       '';
     };
 
-    package = mkOption {
-      type = types.package;
-      default = if versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3;
-      defaultText = literalExpression ''
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = if lib.versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3;
+      defaultText = lib.literalExpression ''
         if versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3;
       '';
       description = lib.mdDoc ''
@@ -85,18 +83,18 @@ in {
       '';
     };
 
-    port = mkOption {
-      type = types.port;
+    port = lib.mkOption {
+      type = lib.types.port;
       default = 8001;
       description = lib.mdDoc ''
         Port the server will listen on.
       '';
     };
 
-    plugins = mkOption {
-      type = types.functionTo (types.listOf types.package);
+    plugins = lib.mkOption {
+      type = with lib.types; functionTo (listOf package);
       default = _: [];
-      defaultText = literalExpression ''
+      defaultText = lib.literalExpression ''
         python3Packages: with python3Packages; [];
       '';
       description = lib.mdDoc ''
@@ -104,23 +102,23 @@ in {
       '';
     };
 
-    dataDir = mkOption {
-      type = types.str;
+    dataDir = lib.mkOption {
+      type = lib.types.str;
       default = "/var/lib/netbox";
       description = lib.mdDoc ''
         Storage path of netbox.
       '';
     };
 
-    secretKeyFile = mkOption {
-      type = types.path;
+    secretKeyFile = lib.mkOption {
+      type = lib.types.path;
       description = lib.mdDoc ''
         Path to a file containing the secret key.
       '';
     };
 
-    extraConfig = mkOption {
-      type = types.lines;
+    extraConfig = lib.mkOption {
+      type = lib.types.lines;
       default = "";
       description = lib.mdDoc ''
         Additional lines of configuration appended to the `configuration.py`.
@@ -128,8 +126,8 @@ in {
       '';
     };
 
-    enableLdap = mkOption {
-      type = types.bool;
+    enableLdap = lib.mkOption {
+      type = lib.types.bool;
       default = false;
       description = lib.mdDoc ''
         Enable LDAP-Authentication for Netbox.
@@ -138,8 +136,8 @@ in {
       '';
     };
 
-    ldapConfigPath = mkOption {
-      type = types.path;
+    ldapConfigPath = lib.mkOption {
+      type = lib.types.path;
       default = "";
       description = lib.mdDoc ''
         Path to the Configuration-File for LDAP-Authentication, will be loaded as `ldap_config.py`.
@@ -173,15 +171,17 @@ in {
     };
   };
 
-  config = mkIf cfg.enable {
+  config = lib.mkIf cfg.enable {
     services.netbox = {
-      plugins = mkIf cfg.enableLdap (ps: [ ps.django-auth-ldap ]);
+      plugins = lib.mkIf cfg.enableLdap (ps: [ ps.django-auth-ldap ]);
       settings = {
         STATIC_ROOT = staticDir;
         MEDIA_ROOT = "${cfg.dataDir}/media";
         REPORTS_ROOT = "${cfg.dataDir}/reports";
         SCRIPTS_ROOT = "${cfg.dataDir}/scripts";
 
+        GIT_PATH = "${pkgs.gitMinimal}/bin/git";
+
         DATABASE = {
           NAME = "netbox";
           USER = "netbox";
@@ -264,39 +264,39 @@ in {
         RestartSec = 30;
       };
     in {
-      netbox-migration = {
-        description = "NetBox migrations";
-        wantedBy = [ "netbox.target" ];
-
-        environment = {
-          PYTHONPATH = pkg.pythonPath;
-        };
-
-        serviceConfig = defaultServiceConfig // {
-          Type = "oneshot";
-          ExecStart = ''
-            ${pkg}/bin/netbox migrate
-          '';
-          PrivateTmp = true;
-        };
-      };
-
       netbox = {
         description = "NetBox WSGI Service";
         documentation = [ "https://docs.netbox.dev/" ];
 
         wantedBy = [ "netbox.target" ];
 
-        after = [ "network-online.target" "netbox-migration.service" ];
+        after = [ "network-online.target" ];
         wants = [ "network-online.target" ];
 
+        environment.PYTHONPATH = pkg.pythonPath;
+
         preStart = ''
+          # On the first run, or on upgrade / downgrade, run migrations and related.
+          # This mostly correspond to upstream NetBox's 'upgrade.sh' script.
+          versionFile="${cfg.dataDir}/version"
+
+          if [[ -e "$versionFile" && "$(cat "$versionFile")" == "${cfg.package.version}" ]]; then
+            exit 0
+          fi
+
+          ${pkg}/bin/netbox migrate
           ${pkg}/bin/netbox trace_paths --no-input
           ${pkg}/bin/netbox collectstatic --no-input
           ${pkg}/bin/netbox remove_stale_contenttypes --no-input
-        '';
+          # TODO: remove the condition when we remove netbox_3_3
+          ${lib.optionalString
+            (lib.versionAtLeast cfg.package.version "3.5.0")
+            "${pkg}/bin/netbox reindex --lazy"}
+          ${pkg}/bin/netbox clearsessions
+          ${pkg}/bin/netbox clearcache
 
-        environment.PYTHONPATH = pkg.pythonPath;
+          echo "${cfg.package.version}" > "$versionFile"
+        '';
 
         serviceConfig = defaultServiceConfig // {
           ExecStart = ''
@@ -331,7 +331,7 @@ in {
 
         wantedBy = [ "multi-user.target" ];
 
-        after = [ "network-online.target" ];
+        after = [ "network-online.target" "netbox.service" ];
         wants = [ "network-online.target" ];
 
         environment.PYTHONPATH = pkg.pythonPath;
@@ -351,7 +351,7 @@ in {
 
       wantedBy = [ "multi-user.target" ];
 
-      after = [ "network-online.target" ];
+      after = [ "network-online.target" "netbox.service" ];
       wants = [ "network-online.target" ];
 
       timerConfig = {

nixos/tests/all-tests.nix

@@ -525,6 +525,7 @@ in {
   networking.scripted = handleTest ./networking.nix { networkd = false; };
   netbox = handleTest ./web-apps/netbox.nix { inherit (pkgs) netbox; };
   netbox_3_3 = handleTest ./web-apps/netbox.nix { netbox = pkgs.netbox_3_3; };
+  netbox-upgrade = handleTest ./web-apps/netbox-upgrade.nix {};
   # TODO: put in networking.nix after the test becomes more complete
   networkingProxy = handleTest ./networking-proxy.nix {};
   nextcloud = handleTest ./nextcloud {};

nixos/tests/web-apps/netbox-upgrade.nix

@@ -0,0 +1,85 @@
+import ../make-test-python.nix ({ lib, pkgs, ... }: let
+  oldNetbox = pkgs.netbox_3_3;
+in {
+  name = "netbox-upgrade";
+
+  meta = with lib.maintainers; {
+    maintainers = [ minijackson ];
+  };
+
+  nodes.machine = { config, ... }: {
+    services.netbox = {
+      enable = true;
+      package = oldNetbox;
+      secretKeyFile = pkgs.writeText "secret" ''
+        abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+      '';
+    };
+
+    services.nginx = {
+      enable = true;
+
+      recommendedProxySettings = true;
+
+      virtualHosts.netbox = {
+        default = true;
+        locations."/".proxyPass = "http://localhost:${toString config.services.netbox.port}";
+        locations."/static/".alias = "/var/lib/netbox/static/";
+      };
+    };
+
+    users.users.nginx.extraGroups = [ "netbox" ];
+
+    networking.firewall.allowedTCPPorts = [ 80 ];
+
+    specialisation.upgrade.configuration.services.netbox.package = lib.mkForce pkgs.netbox;
+  };
+
+  testScript = { nodes, ... }:
+    let
+      apiVersion = version: lib.pipe version [
+        (lib.splitString ".")
+        (lib.take 2)
+        (lib.concatStringsSep ".")
+      ];
+      oldApiVersion = apiVersion oldNetbox.version;
+      newApiVersion = apiVersion pkgs.netbox.version;
+    in
+    ''
+      start_all()
+      machine.wait_for_unit("netbox.target")
+      machine.wait_for_unit("nginx.service")
+      machine.wait_until_succeeds("journalctl --since -1m --unit netbox --grep Listening")
+
+      def api_version(headers):
+          header = [header for header in headers.splitlines() if header.startswith("API-Version:")][0]
+          return header.split()[1]
+
+      def check_api_version(version):
+          headers = machine.succeed(
+            "curl -sSfL http://localhost/api/ --head -H 'Content-Type: application/json'"
+          )
+          assert api_version(headers) == version
+
+      with subtest("NetBox version is the old one"):
+          check_api_version("${oldApiVersion}")
+
+      # Somehow, even though netbox-housekeeping.service has After=netbox.service,
+      # netbox-housekeeping.service and netbox.service still get started at the
+      # same time, making netbox-housekeeping fail (can't really do some house
+      # keeping job if the database is not correctly formed).
+      #
+      # So we don't check that the upgrade went well, we just check that
+      # netbox.service is active, and that netbox-housekeeping can be run
+      # successfully afterwards.
+      #
+      # This is not good UX, but the system should be working nonetheless.
+      machine.execute("${nodes.machine.system.build.toplevel}/specialisation/upgrade/bin/switch-to-configuration test >&2")
+
+      machine.wait_for_unit("netbox.service")
+      machine.succeed("systemctl start netbox-housekeeping.service")
+
+      with subtest("NetBox version is the new one"):
+          check_api_version("${newApiVersion}")
+    '';
+})

pkgs/servers/web-apps/netbox/default.nix

@@ -17,20 +17,23 @@ in
       })
     ];
 
-    tests.netbox = nixosTests.netbox_3_3;
+    tests = {
+      netbox = nixosTests.netbox_3_3;
+      inherit (nixosTests) netbox-upgrade;
+    };
     maintainers = with lib.maintainers; [ n0emis raitobezarius ];
     eol = true;
   };
 
   netbox = callPackage generic {
-    version = "3.5.6";
-    hash = "sha256-n5EJQcC5uVoL5KjGzF7bLF8c4Wke/YBJpx2V9KZz5Qo=";
+    version = "3.5.7";
+    hash = "sha256-R5P4FOhn7rE6e9H9U3JlE3ms4Svv6ps4c3+ZjE1KwNM=";
     extraPatches = [
       # Allow setting the STATIC_ROOT from within the configuration and setting a custom redis URL
       ./config.patch
     ];
     tests = {
-      inherit (nixosTests) netbox;
+      inherit (nixosTests) netbox netbox-upgrade;
     };
 
     maintainers = with lib.maintainers; [ minijackson n0emis raitobezarius ];