diff --git a/deploy/nixos/README.md b/deploy/nixos/README.md index 062cd5f..55eb522 100644 --- a/deploy/nixos/README.md +++ b/deploy/nixos/README.md @@ -19,7 +19,7 @@ deploy/nixos/ │ └── 99-bitspire-hardware.rules # additional udev rules (loaded via configuration.nix) ├── provision-atm.sh # Push LNbits credentials to a deployed ATM via SSH ├── atm-transactions.sh # Operator query tool — reads /var/lib/bitspire/state.db -├── flash-douro-usb.sh # Helper for flashing a douro live USB +├── flash-douro-usb.sh # Helper for flashing a douro LIVE ISO (not the -usb disk image) └── build-iso.sh # Convenience wrapper for `nix build .#iso-` ``` @@ -33,19 +33,37 @@ Each ATM model has two flake outputs: | `nixosConfigurations.-installed` | installed | full GPT + systemd-boot install, ext4 root, supports `nixos-rebuild switch` | | `packages.x86_64-linux.iso-` | ISO | ISO image of the live variant | | `packages.x86_64-linux.disk-image-` | raw image | dd-able full disk image of the installed variant | -| `nixosConfigurations.-usb` | installed, on a stick | `-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off (`batm3`, `douro` only) | +| `nixosConfigurations.-usb` | installed, on a stick | `-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off, `uas` blacklisted | | `packages.x86_64-linux.disk-image--usb` | raw image | dd-able image of the `-usb` variant — flash, plug in, boot; no installer step | -Models: `douro`, `tejo`, `sintra`, `batm3`. +Models: `douro`, `tejo`, `sintra`, `batm3`. All four have `-usb` outputs. -**Run-from-USB deployments** (`batm3` today, `douro` since the LNbits cutover) -skip the Alpine + dd-to-internal-disk procedure below entirely: the stick *is* -the system. Flash `disk-image--usb` with balenaEtcher (it verifies the +**Run-from-USB deployments** skip the Alpine + dd-to-internal-disk procedure +below entirely: the stick *is* the system. That is how `batm3` runs today, how +`douro` runs since the LNbits cutover, and how `tejo` is being brought up — its +internal storage still holds the factory Debian (`ubilinux4`) and is never +touched. Flash `disk-image--usb` with balenaEtcher (it verifies the write — a truncated or bad copy fails stage-1 fsck on first boot) onto a stick of 16 GB or more, plug it in, power on. Updates go in-place against the `-usb` config (`nix copy` the toplevel + `switch-to-configuration`), which keeps pairing and `/var/lib/bitspire`. +### Two bootloader shapes, picked by firmware + +The `-usb` images are not interchangeable between models, because the fleet's +firmware is not: + +| Models | Partition table | Bootloader | Why | +|---|---|---|---| +| `batm3`, `douro` | `efi` (GPT + ESP) | systemd-boot | Their firmware UEFI-USB-boots fine via the ESP's removable `/EFI/BOOT/BOOTX64.EFI` fallback | +| `tejo`, `sintra` | `hybrid` (GPT + `bios_grub` + ESP) | GRUB, BIOS **and** UEFI | Aaeon UP Board firmware USB-boots in Legacy/BIOS mode — it boots the live ISO via isolinux, not the ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot stick isn't recognised as bootable at all. The hybrid image boots either way | + +Both shapes come out of `mkUsbDiskImage` in `flake.nix`; the GRUB override is +`usbGrubHybridModule`. A UP Board `-usb` config installs GRUB with +`devices = [ "nodev" ]` so an in-place `switch-to-configuration` on a live +stick only regenerates `grub.cfg` — the image build is the one place the BIOS +stage gets written to an MBR. + ```bash # Build a Sintra disk image nix build .#disk-image-sintra diff --git a/deploy/nixos/hardware/batm3.nix b/deploy/nixos/hardware/batm3.nix index ca57f0d..dcf8692 100644 --- a/deploy/nixos/hardware/batm3.nix +++ b/deploy/nixos/hardware/batm3.nix @@ -223,6 +223,5 @@ }; }; - # WireGuard VPN address - networking.wireguard.interfaces.wg0.ips = [ "10.0.0.5/24" ]; + # WireGuard VPN address → wireguardIpForModel in flake.nix. } diff --git a/deploy/nixos/hardware/douro.nix b/deploy/nixos/hardware/douro.nix index e2e138c..83a11cf 100644 --- a/deploy/nixos/hardware/douro.nix +++ b/deploy/nixos/hardware/douro.nix @@ -93,8 +93,7 @@ hybrid-sleep.enable = false; }; - # WireGuard VPN address - networking.wireguard.interfaces.wg0.ips = [ "10.0.0.4/24" ]; + # WireGuard VPN address → wireguardIpForModel in flake.nix. # Serial port access for bill validator/dispenser services.udev.extraRules = lib.mkAfter '' diff --git a/flake.nix b/flake.nix index e6ddaaf..4037d3d 100644 --- a/flake.nix +++ b/flake.nix @@ -159,6 +159,36 @@ sintra = true; # HID Global OMNIKEY 5022 }; + # WireGuard address on the 10.0.0.0/24 management tunnel to the VPS + # (peer + listenPort live in configuration.nix; only the address is + # per-machine). Same keying caveat as the three tables above. + # + # This cannot live in a hardware file for the UP Board models, and the + # reason it now lives here for ALL of them is tejo: hardware/upboard.nix + # is shared by tejo and sintra, so an address set there would be claimed + # by both machines on the same /24. tejo had no address at all as a + # result — `wg0.ips = [ ]` brings the interface up with no IP and the + # tunnel is dead, which is a silent way to lose remote access to a + # machine that has no other route in. Keeping douro's and batm3's + # addresses here too means there is one list to read when allocating the + # next one, rather than three files plus the VPS peer config. + # + # An unlisted model gets no address and no tunnel. That is deliberate for + # sintra, which is reachable on the LAN (192.168.0.252) and has never had + # a tunnel address. + # + # NOTE: the address is only half of it. The VPS maps peer PUBLIC KEY to + # pragma: allowlist secret + # tunnel IP, so a machine also needs its private key at + # /var/lib/wireguard/wg0.key — carried over from the machine's previous + # install, or newly generated with its pubkey added to the VPS peer list. + # The key is operator-provisioned and deliberately not in the image. + wireguardIpForModel = { + tejo = "10.0.0.3/24"; + douro = "10.0.0.4/24"; + batm3 = "10.0.0.5/24"; + }; + lib = nixpkgs.lib; # Helper to create a live USB NixOS config for a specific machine model @@ -218,6 +248,11 @@ nfc.enable = nfcReaderForModel.${machineModel} or false; }; + # Management-tunnel address; see wireguardIpForModel. + networking.wireguard.interfaces.wg0.ips = + lib.optional (wireguardIpForModel ? ${machineModel}) + wireguardIpForModel.${machineModel}; + # Operator TUI and CLI tools environment.systemPackages = [ atm-tui.packages.${system}.default @@ -391,19 +426,92 @@ system.autoUpgrade.enable = lib.mkForce false; }; + # Bus hardening that makes a USB stick a reliable boot medium: keep the + # flash drive off the flaky UAS driver (many bridges advertise UAS then + # drop off the bus under sustained write load — "device offline error, + # dev sdb"), and stop USB autosuspend cutting power mid-I/O. douro.nix + # and batm3.nix carry this in their hardware files because those machines + # boot from USB exclusively; hardware/upboard.nix is SHARED with sintra's + # eMMC install, so for the UP Board models it is scoped to the -usb + # config here rather than changing a production machine's cmdline. + usbBusHardening = { + boot.blacklistedKernelModules = [ "uas" ]; + boot.kernelParams = [ "usbcore.autosuspend=-1" ]; + }; + + # Aaeon UP Board firmware (tejo, sintra) USB-boots in Legacy/BIOS mode — + # it boots the live ISO via its isolinux (BIOS) El Torito image, not the + # UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot image is not + # recognised as bootable at all. Switch the UP Board USB configs to GRUB + # with BOTH BIOS (MBR + bios_grub partition, via mkUsbDiskImage's + # partitionTableType = "hybrid") and UEFI (removable + # /EFI/BOOT/BOOTX64.EFI) — mirroring the live ISO's dual boot — so the + # stick boots on Legacy and UEFI alike. Scoped to the USB configs; the + # eMMC installs keep systemd-boot. + # + # devices = [ "nodev" ] here, NOT the image's disk. This config is also + # what in-place updates (`nix copy` + switch-to-configuration) run against + # on a LIVE stick, where the build VM's /dev/vda does not exist and a BIOS + # grub-install against it would fail the switch. "nodev" regenerates + # grub.cfg and skips the MBR write, which is the correct behaviour for an + # update: GRUB's embedded core.img reads grub.cfg off the partition, so + # the MBR stage never needs rewriting per generation. mkUsbDiskImage's + # grubBiosDevice overrides this for the image build, where the BIOS stage + # genuinely has to be written. + usbGrubHybridModule = { lib, ... }: { + boot.loader.systemd-boot.enable = lib.mkForce false; + boot.loader.efi.canTouchEfiVariables = lib.mkForce false; + boot.loader.grub = { + enable = lib.mkForce true; + efiSupport = true; + efiInstallAsRemovable = true; + # Plain definition, NOT mkForce: grubBiosDevice overrides it with + # mkForce, and two mkForce list definitions would merge (both + # priority 50) into [ "/dev/vda" "nodev" ] instead of replacing. + # Nothing else in the module stack defines grub.devices. + devices = [ "nodev" ]; + }; + }; + # dd-able USB image of a -usb config. make-disk-image gives the # ext4 root the nixos-usb label directly (-L) but hardcodes the ESP FAT # label to "ESP", so the volume is relabelled to ESP-USB afterwards — # volume label only; bootloader files are untouched and UEFI loads - # /EFI/BOOT/BOOTX64.EFI regardless. Keeps systemd-boot: both the batm3 - # and douro firmware UEFI-USB-boot fine via that removable fallback. - mkUsbDiskImage = machineModel: usbConfig: + # /EFI/BOOT/BOOTX64.EFI regardless. + # + # partitionTableType: "efi" (GPT + ESP, systemd-boot) for batm3 and douro, + # whose firmware UEFI-USB-boots fine via that removable fallback; + # "hybrid" (GPT + bios_grub + ESP) for the UP Board models, paired with + # usbGrubHybridModule. In BOTH layouts the ESP is partition 1 — the hybrid + # table creates the ESP first and the bios_grub partition second — so the + # parted/mlabel relabel below is layout-independent. + # + # grubBiosDevice: the build VM's disk, for hybrid images only. GRUB must + # write its BIOS stage to that disk's MBR at image-build time, while the + # config itself says "nodev" so in-place updates on a live stick work; + # see usbGrubHybridModule. + mkUsbDiskImage = + { machineModel + , usbConfig + , partitionTableType ? "efi" + , grubBiosDevice ? null + }: let + imageConfig = + if grubBiosDevice == null then + usbConfig + else + usbConfig.extendModules { + modules = [ + ({ lib, ... }: { + boot.loader.grub.devices = lib.mkForce [ grubBiosDevice ]; + }) + ]; + }; baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") { - inherit pkgs lib; - config = usbConfig.config; + inherit pkgs lib partitionTableType; + config = imageConfig.config; format = "raw"; - partitionTableType = "efi"; diskSize = "auto"; label = "nixos-usb"; # ext4 root label (make-disk-image -L) }; @@ -469,6 +577,24 @@ douro-usb = self.nixosConfigurations.douro-installed.extendModules { modules = [ usbBootModule ]; }; + # UP Board models get the same run-from-USB shape plus two things the + # Bay Trail / OptiPlex boxes don't need: GRUB on a hybrid table, because + # the Aaeon firmware USB-boots in Legacy/BIOS mode (usbGrubHybridModule), + # and the uas/autosuspend hardening from here instead of + # hardware/upboard.nix, which sintra's eMMC install also reads. + # + # tejo: the unit still runs its factory Debian (ubilinux4) on internal + # storage, so the stick has to BE the system, same as douro. Its + # internal install is not NixOS and carries no nixos/ESP labels, which + # makes the label disambiguation moot there today — but the hardened + # /boot and the bus settings are what make a stick a reliable boot + # medium, so it takes the identical module set as sintra. + tejo-usb = self.nixosConfigurations.tejo-installed.extendModules { + modules = [ usbBootModule usbBusHardening usbGrubHybridModule ]; + }; + sintra-usb = self.nixosConfigurations.sintra-installed.extendModules { + modules = [ usbBootModule usbBusHardening usbGrubHybridModule ]; + }; }; # ── Standalone NixOS module ─────────────────────────────────── @@ -536,71 +662,6 @@ diskSize = "auto"; }; - # USB-bootable Sintra image with DISTINCT partition labels - # (nixos-usb / ESP-USB) so the stick can be booted on a Sintra whose - # eMMC already holds a nixos/ESP-labelled install without a by-label - # collision — stage-1 would otherwise race between the two roots and - # likely mount the eMMC. Auto-upgrade is disabled: this is a portable - # test / hand-off image, not a managed fleet member, and disabling it - # also removes the scheduled bootloader writes that could otherwise - # land on the eMMC's ESP. - disk-image-sintra-usb = - let - cfg = self.nixosConfigurations.sintra-installed.extendModules { - modules = [ - ({ lib, ... }: { - fileSystems."/".device = lib.mkForce "/dev/disk/by-label/nixos-usb"; - fileSystems."/boot".device = lib.mkForce "/dev/disk/by-label/ESP-USB"; - system.autoUpgrade.enable = lib.mkForce false; - - # The Sintra's Aaeon firmware USB-boots in Legacy/BIOS mode — it - # boots the live ISO via its isolinux (BIOS) El Torito image, not - # the UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot - # image isn't recognised as bootable. Switch THIS USB image to - # GRUB with BOTH BIOS (MBR + bios_grub partition, via the "hybrid" - # table below) and UEFI (removable /EFI/BOOT/BOOTX64.EFI) — mirroring - # the live ISO's dual boot — so it boots on Legacy and UEFI alike. - # Scoped to the USB image; the eMMC install keeps systemd-boot. - boot.loader.systemd-boot.enable = lib.mkForce false; - boot.loader.efi.canTouchEfiVariables = lib.mkForce false; - boot.loader.grub = { - enable = lib.mkForce true; - efiSupport = true; - efiInstallAsRemovable = true; - # make-disk-image's build VM exposes the image as /dev/vda; - # GRUB installs its BIOS stage to that disk's MBR. - devices = lib.mkForce [ "/dev/vda" ]; - }; - }) - ]; - }; - baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") { - inherit pkgs lib; - config = cfg.config; - format = "raw"; - # hybrid = GPT + bios_grub partition + ESP → BIOS + UEFI bootable. - partitionTableType = "hybrid"; - diskSize = "auto"; - label = "nixos-usb"; # ext4 root label (make-disk-image -L) - }; - in - pkgs.runCommand "nixos-disk-image-sintra-usb" - { nativeBuildInputs = [ pkgs.parted pkgs.mtools ]; } - '' - mkdir -p $out - cp --sparse=always ${baseImage}/nixos.img $out/nixos.img - chmod +w $out/nixos.img - # make-disk-image hardcodes the ESP FAT label to "ESP"; relabel the - # volume to ESP-USB so /boot (by-label/ESP-USB) doesn't collide with - # the eMMC's ESP. Volume label only — bootloader files are untouched, - # and UEFI loads /EFI/BOOT/BOOTX64.EFI regardless of the label. - espStart=$(parted -sm "$out/nixos.img" unit B print | awk -F: '$1==1 {gsub("B","",$2); print $2}') - echo "ESP partition starts at byte $espStart — relabelling to ESP-USB" - export MTOOLS_SKIP_CHECK=1 - mlabel -i "$out/nixos.img@@$espStart" ::ESP-USB - printf 'verify ESP label: '; mlabel -i "$out/nixos.img@@$espStart" -s :: || true - ''; - # USB-bootable images (see usbBootModule / mkUsbDiskImage in the let # block). Flash with dd or balenaEtcher, boot the stick, done — no # installer step. The plain disk-image- reuses the generic @@ -609,8 +670,32 @@ # stage-1's by-label/nixos resolve to the internal drive instead of the # stick — the stage-2 init path baked into the USB's boot entry isn't on # that root, so stage 1 aborts. These variants can't hit that. - disk-image-batm3-usb = mkUsbDiskImage "batm3" self.nixosConfigurations.batm3-usb; - disk-image-douro-usb = mkUsbDiskImage "douro" self.nixosConfigurations.douro-usb; + disk-image-batm3-usb = mkUsbDiskImage { + machineModel = "batm3"; + usbConfig = self.nixosConfigurations.batm3-usb; + }; + disk-image-douro-usb = mkUsbDiskImage { + machineModel = "douro"; + usbConfig = self.nixosConfigurations.douro-usb; + }; + + # UP Board models: hybrid table + GRUB, so one stick boots on the Aaeon + # firmware's Legacy/BIOS USB path as well as UEFI. Distinct labels also + # matter more here than on douro — a sintra's eMMC already holds a + # nixos/ESP-labelled install, and stage-1 would otherwise race the two + # roots and likely mount the eMMC. + disk-image-tejo-usb = mkUsbDiskImage { + machineModel = "tejo"; + usbConfig = self.nixosConfigurations.tejo-usb; + partitionTableType = "hybrid"; + grubBiosDevice = "/dev/vda"; + }; + disk-image-sintra-usb = mkUsbDiskImage { + machineModel = "sintra"; + usbConfig = self.nixosConfigurations.sintra-usb; + partitionTableType = "hybrid"; + grubBiosDevice = "/dev/vda"; + }; # Backwards compat iso = self.nixosConfigurations.douro.config.system.build.isoImage;