From 48321bb555c44a4fbc07fbfa1e9e30e018d66452 Mon Sep 17 00:00:00 2001
From: Andrew Kidd <contact@andrewiankidd.co.uk>
Date: Sat, 4 Jan 2025 16:30:34 +0000
Subject: [PATCH] Add 'NetImage' config option for Pi Netboot (#1)

* Add wip netboot image gen

* rem firmware dir

* additional logging

* nixosConfigurations

* config.fileSystems

* rework rootfs output dir

* nfs/boot/firmware

* boot.initrd.network.enable = true;

* nfsOptions

* neededForBoot, supportedFilesystems
---
 flake.nix                  |   6 +
 net-image/default.nix      |  83 +++++++++++++
 net-image/make-root-fs.nix |  52 ++++++++
 net-image/net-image.nix    | 245 +++++++++++++++++++++++++++++++++++++
 4 files changed, 386 insertions(+)
 create mode 100644 net-image/default.nix
 create mode 100644 net-image/make-root-fs.nix
 create mode 100644 net-image/net-image.nix

diff --git a/flake.nix b/flake.nix
index c0acdc3..dd27468 100644
--- a/flake.nix
+++ b/flake.nix
@@ -60,12 +60,17 @@
           libcamera-overlay = self.overlays.libcamera;
         };
         sd-image = import ./sd-image;
+        net-image = import ./net-image;
       };
       nixosConfigurations = {
         rpi-example = srcs.nixpkgs.lib.nixosSystem {
           system = "aarch64-linux";
           modules = [ self.nixosModules.raspberry-pi self.nixosModules.sd-image ./example ];
         };
+        rpi-net-example = srcs.nixpkgs.lib.nixosSystem {
+          system = "aarch64-linux";
+          modules = [ self.nixosModules.raspberry-pi self.nixosModules.net-image ./example ];
+        };
       };
       checks.aarch64-linux = self.packages.aarch64-linux;
       packages.aarch64-linux = with pinned.lib;
@@ -82,6 +87,7 @@
         in
         {
           example-sd-image = self.nixosConfigurations.rpi-example.config.system.build.sdImage;
+          example-net-image = self.nixosConfigurations.rpi-net-example.config.system.build.netImage;
           firmware = pinned.raspberrypifw;
           libcamera = pinned.libcamera;
           wireless-firmware = pinned.raspberrypiWirelessFirmware;
diff --git a/net-image/default.nix b/net-image/default.nix
new file mode 100644
index 0000000..2dbdb2c
--- /dev/null
+++ b/net-image/default.nix
@@ -0,0 +1,83 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ./net-image.nix ];
+
+  config = {
+    boot.loader.grub.enable = false;
+
+    boot.consoleLogLevel = lib.mkDefault 8;
+
+    boot.kernelParams = [
+        # Read-only root filesystem
+        "ro"
+        # NFS root filesystem location
+        "nfsroot=${config.netImage.nfsRoot},v3"
+        # Root filesystem device
+        "root=/dev/nfs"
+        # Wait for root filesystem
+        "rootwait"
+        # I/O scheduler
+        "elevator=deadline"
+        # Enable systemd debug shell
+        "systemd.debug_shell=1"
+        # Set systemd log level to debug
+        "systemd.log_level=info"
+        # Disable splash screen
+        "disable_splash"
+        # Early printk to serial console
+        "earlyprintk=serial,ttyS0,115200"
+        # Enable initcall debugging
+        "initcall_debug"
+        # Print timestamps in printk messages
+        "printk.time=1"
+    ];
+
+    netImage =
+      let
+        kernel-params = pkgs.writeTextFile {
+          name = "cmdline.txt";
+          text = ''
+            ${lib.strings.concatStringsSep " " config.boot.kernelParams}
+          '';
+        };
+        cfg = config.raspberry-pi-nix;
+        version = cfg.kernel-version;
+        board = cfg.board;
+        kernel = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
+        initrd = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
+        populate-kernel =
+          if cfg.uboot.enable
+          then ''
+            cp ${cfg.uboot.package}/u-boot.bin ./u-boot-rpi-arm64.bin
+          ''
+          else ''
+            cp "${kernel}" ./kernel.img
+            cp "${initrd}" ./initrd
+            cp "${kernel-params}" ./cmdline.txt
+          '';
+      in
+      {
+        populateFirmwareCommands = ''
+          ${populate-kernel}
+          cp -r ${pkgs.raspberrypifw}/share/raspberrypi/boot/{start*.elf,*.dtb,bootcode.bin,fixup*.dat,overlays} ./
+          cp ${config.hardware.raspberry-pi.config-output} ./config.txt
+        '';
+        populateRootCommands =
+          if cfg.uboot.enable
+          then ''
+            mkdir -p ./files/boot
+            ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d ./files/boot
+          ''
+          else ''
+            mkdir -p ./files/sbin
+            content="$(
+              echo "#!${pkgs.bash}/bin/bash"
+              echo "exec ${config.system.build.toplevel}/init"
+            )"
+            echo "$content" > ./files/sbin/init
+            chmod 744 ./files/sbin/init
+          '';
+      };
+  };
+}
diff --git a/net-image/make-root-fs.nix b/net-image/make-root-fs.nix
new file mode 100644
index 0000000..ee7c721
--- /dev/null
+++ b/net-image/make-root-fs.nix
@@ -0,0 +1,52 @@
+# Builds a directory containing a populated /nix/store with the closure
+# of store paths passed in the storePaths parameter, in addition to the
+# contents of a directory that can be populated with commands.
+{
+  pkgs,
+  lib,
+  # List of derivations to be included
+  storePaths,
+  # Shell commands to populate the ./files directory.
+  # All files in that directory are copied to the root of the FS.
+  populateRootCommands ? "",
+  perl
+}:
+
+let
+  netbootClosureInfo = pkgs.buildPackages.closureInfo { rootPaths = storePaths; };
+in
+pkgs.stdenv.mkDerivation {
+  name = "root-fs";
+
+  nativeBuildInputs = [
+    perl
+  ];
+
+  buildCommand = ''
+    echo "Populating image with command: ${populateRootCommands}"
+    mkdir -p ./files
+    ${populateRootCommands}
+
+    echo "Preparing store paths for image..."
+    # Create nix/store before copying path
+    mkdir -p ./rootImage/nix/store
+
+    xargs -I % cp -a --reflink=auto % -t ./rootImage/nix/store/ < ${netbootClosureInfo}/store-paths
+    (
+      GLOBIGNORE=".:.."
+      shopt -u dotglob
+
+      for f in ./files/*; do
+          cp -a --reflink=auto -t ./rootImage/ "$f"
+      done
+    )
+
+    # Also include a manifest of the closures in a format suitable for nix-store --load-db
+    cp ${netbootClosureInfo}/registration ./rootImage/nix-path-registration
+
+    # done
+    echo "Image populated."
+    ls -aR ./rootImage
+    cp -r ./rootImage/ $out
+  '';
+}
diff --git a/net-image/net-image.nix b/net-image/net-image.nix
new file mode 100644
index 0000000..9a84b9b
--- /dev/null
+++ b/net-image/net-image.nix
@@ -0,0 +1,245 @@
+# This module creates the files necessary containing the given NixOS
+# configuration. The generated directories consists of a `boot` directory
+# (for TFTP boot) and a `root` directory (for NFS root). The goal is to
+# allow the system to boot over the network, using TFTP to retrieve the
+# boot files and NFS to mount the root filesystem, enabling fully
+# headless deployment of the NixOS system.
+
+# The generated files consists of two directories:
+# - `boot`: Contains the necessary bootloader, kernel, and initrd files
+#   required for booting the system. These files will be served via TFTP
+#   to the target machine.
+# - `root`: Contains the root filesystem that will be mounted by the
+#   target machine over NFS. This is typically an ext4 root partition
+#   populated with the necessary NixOS configuration.
+
+# The image does not include a bootable SD card but instead prepares the
+# filesystem and boot files for network-based booting. The NixOS
+# configuration will be automatically applied when the system boots.
+
+# Note: This module assumes that you have already set up the TFTP and
+# NFS servers on your network, and the target machine is configured
+# for network booting.
+
+{ modulesPath, config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  rootfsImage = pkgs.callPackage (builtins.path { path = ./make-root-fs.nix; }) ({
+    inherit (config.netImage) storePaths;
+    populateRootCommands = config.netImage.populateRootCommands;
+  });
+in
+{
+  imports = [ ];
+
+  options.netImage = {
+    rootDirectoryName = mkOption {
+      default =
+        "${config.netImage.directoryBaseName}-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
+      description = ''
+        Name of the generated root directory.
+      '';
+    };
+
+    directoryBaseName = mkOption {
+      default = "nixos-net-image";
+      description = ''
+        Prefix of the name of the generated root directory.
+      '';
+    };
+
+    storePaths = mkOption {
+      type = with types; listOf package;
+      example = literalExpression "[ pkgs.stdenv ]";
+      description = ''
+        Derivations to be included in the Nix store in the generated Netboot image.
+      '';
+    };
+
+    nfsRoot = mkOption {
+      type = types.str;
+      default = "192.168.0.108:/mnt/nfsshare/${config.netImage.directoryBaseName}-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
+      description = ''
+        cmdline.txt nfs parameter for the root filesystem.
+      '';
+    };
+
+    nfsOptions = mkOption {
+      type = with types; listOf str;
+      default = [
+          # Disable file locking
+          "nolock"
+          # Mount the filesystem read-write
+          "rw"
+          # Use NFS version 3
+          "vers=3"
+          # Set the read buffer size to 131072 bytes
+          "rsize=131072"
+          # Set the write buffer size to 131072 bytes
+          "wsize=131072"
+          # Set the maximum filename length to 255 characters
+          "namlen=255"
+          # Use hard mounts (retry indefinitely on failure)
+          "hard"
+          # Disable Access Control Lists
+          "noacl"
+          # Use TCP as the transport protocol
+          "proto=tcp"
+          # Set the NFS timeout to 11 tenths of a second
+          "timeo=11"
+          # Set the number of NFS retransmissions to 3
+          "retrans=3"
+          # Use the 'sys' security flavor
+          "sec=sys"
+          # Use NFS mount protocol version 3
+          "mountvers=3"
+          # Use TCP for the mount protocol
+          "mountproto=tcp"
+          # Enable local locking
+          "local_lock=all"
+          # Do not update inode access times on reads
+          "noatime"
+          # Do not update directory inode access times on reads
+          "nodiratime"
+        ];
+      description = ''
+        NFS options to use when mounting the root filesystem.
+      '';
+    };
+
+    populateFirmwareCommands = mkOption {
+      example =
+        literalExpression "'' cp \${pkgs.myBootLoader}/u-boot.bin ./ ''";
+      description = ''
+        Shell commands to populate the ./ directory.
+        All files in that directory are copied to the
+        tftp files on the Netboot image.
+      '';
+    };
+
+    populateRootCommands = mkOption {
+      example = literalExpression
+        "''\${config.boot.loader.generic-extlinux-compatible.populateCmd} -c \${config.system.build.toplevel} -d ./files/boot''";
+      description = ''
+        Shell commands to populate the ./files directory.
+        All files in that directory are copied to the
+        root (/) partition on the Netboot image. Use this to
+        populate the ./files/boot (/boot) directory.
+      '';
+    };
+  };
+
+  config = {
+    # net
+    networking.useDHCP = lib.mkForce true;
+    networking.interfaces.eth0.useDHCP = lib.mkForce true;
+    networking.interfaces.wlan0.useDHCP = lib.mkForce false;
+
+    # boot
+    boot.initrd.network.enable = lib.mkForce true;
+    boot.initrd.network.flushBeforeStage2 = lib.mkForce false;
+    boot.initrd.supportedFilesystems = [
+        # Network File System (NFS) support for mounting root over the network
+        "nfs"
+        # Overlay filesystem for layering file systems
+        "overlay"
+    ];
+
+    boot.initrd.availableKernelModules = [
+        # Network File System (NFS) module
+        "nfs"
+        # Overlay filesystem module
+        "overlay"
+        # Broadcom PHY library for Ethernet device support
+        "bcm_phy_lib"
+        # Broadcom-specific driver module
+        "broadcom"
+        # Broadcom GENET Ethernet controller driver
+        "genet"
+    ];
+
+    boot.initrd.kernelModules = [
+        # Network File System (NFS) module
+        "nfs"
+        # Overlay filesystem module
+        "overlay"
+        # Broadcom PHY library for Ethernet device support
+        "bcm_phy_lib"
+        # Broadcom-specific driver module
+        "broadcom"
+        # Broadcom GENET Ethernet controller driver
+        "genet"
+    ];
+
+
+    # fileSystems
+    fileSystems = {
+      "/boot/firmware" = {
+        device = "${config.netImage.nfsRoot}/boot/firmware";
+        fsType = "nfs";
+        options = config.netImage.nfsOptions;
+        neededForBoot = lib.mkForce true;
+      };
+      "/" = {
+        device = "${config.netImage.nfsRoot}";
+        fsType = "nfs";
+        options = config.netImage.nfsOptions;
+        neededForBoot = lib.mkForce true;
+      };
+    };
+
+    netImage.storePaths = [ config.system.build.toplevel ];
+
+    system.build.netImage = pkgs.callPackage
+      ({ stdenv, util-linux }:
+        stdenv.mkDerivation {
+          name = config.netImage.rootDirectoryName;
+
+          nativeBuildInputs = [ util-linux ];
+
+          buildCommand = ''
+            set -e
+            set -x
+            mkdir -p $out/nix-support $out/net-image
+            export rootfs=$out/net-image/os/${config.netImage.rootDirectoryName}
+            export bootfs=$out/net-image/boot
+
+            echo "${pkgs.stdenv.buildPlatform.system}" > $out/nix-support/system
+
+            # Populate the files intended for NFS
+            echo "Exporting rootfs image"
+            mkdir -p $rootfs
+            cp -r ${rootfsImage}/* $rootfs
+
+            # Populate the files intended for TFTP
+            echo "Exporting rootfs image"
+            ${config.netImage.populateFirmwareCommands}
+            mkdir -p $bootfs
+            cp -r . $bootfs
+            mkdir -p $rootfs/boot/firmware
+            cp -r . $rootfs/boot/firmware
+          '';
+        })
+      { };
+
+    boot.postBootCommands = ''
+      # On the first boot do some maintenance tasks
+      if [ -f /nix-path-registration ]; then
+        set -euo pipefail
+        set -x
+
+        # Register the contents of the initial Nix store
+        ${config.nix.package.out}/bin/nix-store --load-db < /nix-path-registration
+
+        # nixos-rebuild also requires a "system" profile and an /etc/NIXOS tag.
+        touch /etc/NIXOS
+        ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+
+        # Prevents this from running on later boots.
+        rm -f /nix-path-registration
+      fi
+    '';
+  };
+}