Merge pull request #277633 from nikstur/image-repart-improvements

Image repart improvements

nixos/modules/image/repart-image.nix

@@ -10,6 +10,8 @@
 , systemd
 , fakeroot
 , util-linux
+
+  # filesystem tools
 , dosfstools
 , mtools
 , e2fsprogs
@@ -18,8 +20,13 @@
 , btrfs-progs
 , xfsprogs
 
+  # compression tools
+, zstd
+, xz
+
   # arguments
-, name
+, imageFileBasename
+, compression
 , fileSystems
 , partitions
 , split
@@ -52,14 +59,25 @@ let
   };
 
   fileSystemTools = builtins.concatMap (f: fileSystemToolMapping."${f}") fileSystems;
+
+  compressionPkg = {
+    "zstd" = zstd;
+    "xz" = xz;
+  }."${compression.algorithm}";
+
+  compressionCommand = {
+    "zstd" = "zstd --no-progress --threads=0 -${toString compression.level}";
+    "xz" = "xz --keep --verbose --threads=0 -${toString compression.level}";
+  }."${compression.algorithm}";
 in
 
-runCommand name
+runCommand imageFileBasename
 {
   nativeBuildInputs = [
     systemd
     fakeroot
     util-linux
+    compressionPkg
   ] ++ fileSystemTools;
 } ''
   amendedRepartDefinitions=$(${amendRepartDefinitions} ${partitions} ${definitionsDirectory})
@@ -67,6 +85,7 @@ runCommand name
   mkdir -p $out
   cd $out
 
+  echo "Building image with systemd-repart..."
   unshare --map-root-user fakeroot systemd-repart \
     --dry-run=no \
     --empty=create \
@@ -75,6 +94,17 @@ runCommand name
     --definitions="$amendedRepartDefinitions" \
     --split="${lib.boolToString split}" \
     --json=pretty \
-    image.raw \
+    ${imageFileBasename}.raw \
     | tee repart-output.json
+
+  # Compression is implemented in the same derivation as opposed to in a
+  # separate derivation to allow users to save disk space. Disk images are
+  # already very space intensive so we want to allow users to mitigate this.
+  if ${lib.boolToString compression.enable}; then
+    for f in ${imageFileBasename}*; do
+      echo "Compressing $f with ${compression.algorithm}..."
+      # Keep the original file when compressing and only delete it afterwards
+      ${compressionCommand} $f && rm $f
+    done
+  fi
 ''

nixos/modules/image/repart.nix

@@ -66,7 +66,53 @@ in
 
     name = lib.mkOption {
       type = lib.types.str;
-      description = lib.mdDoc "The name of the image.";
+      description = lib.mdDoc ''
+        Name of the image.
+
+        If this option is unset but config.system.image.id is set,
+        config.system.image.id is used as the default value.
+      '';
+    };
+
+    version = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = config.system.image.version;
+      defaultText = lib.literalExpression "config.system.image.version";
+      description = lib.mdDoc "Version of the image";
+    };
+
+    imageFileBasename = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      description = lib.mdDoc ''
+        Basename of the image filename without any extension (e.g. `image_1`).
+      '';
+    };
+
+    imageFile = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      description = lib.mdDoc ''
+        Filename of the image including all extensions (e.g `image_1.raw` or
+        `image_1.raw.zst`).
+      '';
+    };
+
+    compression = {
+      enable = lib.mkEnableOption (lib.mdDoc "Image compression");
+
+      algorithm = lib.mkOption {
+        type = lib.types.enum [ "zstd" "xz" ];
+        default = "zstd";
+        description = lib.mdDoc "Compression algorithm";
+      };
+
+      level = lib.mkOption {
+        type = lib.types.int;
+        description = lib.mdDoc ''
+          Compression level. The available range depends on the used algorithm.
+        '';
+      };
     };
 
     seed = lib.mkOption {
@@ -131,6 +177,32 @@ in
 
   config = {
 
+    image.repart =
+      let
+        version = config.image.repart.version;
+        versionInfix = if version != null then "_${version}" else "";
+        compressionSuffix = lib.optionalString cfg.compression.enable
+          {
+            "zstd" = ".zst";
+            "xz" = ".xz";
+          }."${cfg.compression.algorithm}";
+      in
+      {
+        name = lib.mkIf (config.system.image.id != null) (lib.mkOptionDefault config.system.image.id);
+        imageFileBasename = cfg.name + versionInfix;
+        imageFile = cfg.imageFileBasename + ".raw" + compressionSuffix;
+
+        compression = {
+          # Generally default to slightly faster than default compression
+          # levels under the assumption that most of the building will be done
+          # for development and release builds will be customized.
+          level = lib.mkOptionDefault {
+            "zstd" = 3;
+            "xz" = 3;
+          }."${cfg.compression.algorithm}";
+        };
+      };
+
     system.build.image =
       let
         fileSystems = lib.filter
@@ -160,7 +232,7 @@ in
       in
       pkgs.callPackage ./repart-image.nix {
         systemd = cfg.package;
-        inherit (cfg) name split seed;
+        inherit (cfg) imageFileBasename compression split seed;
         inherit fileSystems definitionsDirectory partitions;
       };
 

nixos/modules/misc/version.nix

@@ -28,6 +28,8 @@ let
     DOCUMENTATION_URL = lib.optionalString (cfg.distroId == "nixos") "https://nixos.org/learn.html";
     SUPPORT_URL = lib.optionalString (cfg.distroId == "nixos") "https://nixos.org/community.html";
     BUG_REPORT_URL = lib.optionalString (cfg.distroId == "nixos") "https://github.com/NixOS/nixpkgs/issues";
+    IMAGE_ID = lib.optionalString (config.system.image.id != null) config.system.image.id;
+    IMAGE_VERSION = lib.optionalString (config.system.image.version != null) config.system.image.version;
   } // lib.optionalAttrs (cfg.variant_id != null) {
     VARIANT_ID = cfg.variant_id;
   };
@@ -110,6 +112,38 @@ in
       example = "installer";
     };
 
+    image = {
+
+      id = lib.mkOption {
+        type = types.nullOr (types.strMatching "^[a-z0-9._-]+$");
+        default = null;
+        description = lib.mdDoc ''
+          Image identifier.
+
+          This corresponds to the IMAGE_ID field in os-release. See the
+          upstream docs for more details on valid characters for this field:
+          https://www.freedesktop.org/software/systemd/man/latest/os-release.html#IMAGE_ID=
+
+          You would only want to set this option if you're build NixOS appliance images.
+        '';
+      };
+
+      version = lib.mkOption {
+        type = types.nullOr (types.strMatching "^[a-z0-9._-]+$");
+        default = null;
+        description = lib.mdDoc ''
+          Image version.
+
+          This corresponds to the IMAGE_VERSION field in os-release. See the
+          upstream docs for more details on valid characters for this field:
+          https://www.freedesktop.org/software/systemd/man/latest/os-release.html#IMAGE_VERSION=
+
+          You would only want to set this option if you're build NixOS appliance images.
+        '';
+      };
+
+    };
+
     stateVersion = mkOption {
       type = types.str;
       # TODO Remove this and drop the default of the option so people are forced to set it.

nixos/tests/appliance-repart-image.nix

@@ -8,6 +8,9 @@
 let
   rootPartitionLabel = "root";
 
+  imageId = "nixos-appliance";
+  imageVersion = "1-rc1";
+
   bootLoaderConfigPath = "/loader/entries/nixos.conf";
   kernelPath = "/EFI/nixos/kernel.efi";
   initrdPath = "/EFI/nixos/initrd.efi";
@@ -29,6 +32,9 @@ in
     # TODO(raitobezarius): revisit this when #244907 lands
     boot.loader.grub.enable = false;
 
+    system.image.id = imageId;
+    system.image.version = imageVersion;
+
     virtualisation.fileSystems = lib.mkForce {
       "/" = {
         device = "/dev/disk/by-partlabel/${rootPartitionLabel}";
@@ -99,7 +105,7 @@ in
       "-f",
       "qcow2",
       "-b",
-      "${nodes.machine.system.build.image}/image.raw",
+      "${nodes.machine.system.build.image}/${nodes.machine.image.repart.imageFile}",
       "-F",
       "raw",
       tmp_disk_image.name,
@@ -108,6 +114,10 @@ in
     # Set NIX_DISK_IMAGE so that the qemu script finds the right disk image.
     os.environ['NIX_DISK_IMAGE'] = tmp_disk_image.name
 
+    os_release = machine.succeed("cat /etc/os-release")
+    assert 'IMAGE_ID="${imageId}"' in os_release
+    assert 'IMAGE_VERSION="${imageVersion}"' in os_release
+
     bootctl_status = machine.succeed("bootctl status")
     assert "${bootLoaderConfigPath}" in bootctl_status
     assert "${kernelPath}" in bootctl_status