diff --git a/deploy/nixos/README.md b/deploy/nixos/README.md index 55eb522..062cd5f 100644 --- a/deploy/nixos/README.md +++ b/deploy/nixos/README.md @@ -19,7 +19,7 @@ deploy/nixos/ │ └── 99-bitspire-hardware.rules # additional udev rules (loaded via configuration.nix) ├── provision-atm.sh # Push LNbits credentials to a deployed ATM via SSH ├── atm-transactions.sh # Operator query tool — reads /var/lib/bitspire/state.db -├── flash-douro-usb.sh # Helper for flashing a douro LIVE ISO (not the -usb disk image) +├── flash-douro-usb.sh # Helper for flashing a douro live USB └── build-iso.sh # Convenience wrapper for `nix build .#iso-` ``` @@ -33,37 +33,19 @@ Each ATM model has two flake outputs: | `nixosConfigurations.-installed` | installed | full GPT + systemd-boot install, ext4 root, supports `nixos-rebuild switch` | | `packages.x86_64-linux.iso-` | ISO | ISO image of the live variant | | `packages.x86_64-linux.disk-image-` | raw image | dd-able full disk image of the installed variant | -| `nixosConfigurations.-usb` | installed, on a stick | `-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off, `uas` blacklisted | +| `nixosConfigurations.-usb` | installed, on a stick | `-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off (`batm3`, `douro` only) | | `packages.x86_64-linux.disk-image--usb` | raw image | dd-able image of the `-usb` variant — flash, plug in, boot; no installer step | -Models: `douro`, `tejo`, `sintra`, `batm3`. All four have `-usb` outputs. +Models: `douro`, `tejo`, `sintra`, `batm3`. -**Run-from-USB deployments** skip the Alpine + dd-to-internal-disk procedure -below entirely: the stick *is* the system. That is how `batm3` runs today, how -`douro` runs since the LNbits cutover, and how `tejo` is being brought up — its -internal storage still holds the factory Debian (`ubilinux4`) and is never -touched. Flash `disk-image--usb` with balenaEtcher (it verifies the +**Run-from-USB deployments** (`batm3` today, `douro` since the LNbits cutover) +skip the Alpine + dd-to-internal-disk procedure below entirely: the stick *is* +the system. Flash `disk-image--usb` with balenaEtcher (it verifies the write — a truncated or bad copy fails stage-1 fsck on first boot) onto a stick of 16 GB or more, plug it in, power on. Updates go in-place against the `-usb` config (`nix copy` the toplevel + `switch-to-configuration`), which keeps pairing and `/var/lib/bitspire`. -### Two bootloader shapes, picked by firmware - -The `-usb` images are not interchangeable between models, because the fleet's -firmware is not: - -| Models | Partition table | Bootloader | Why | -|---|---|---|---| -| `batm3`, `douro` | `efi` (GPT + ESP) | systemd-boot | Their firmware UEFI-USB-boots fine via the ESP's removable `/EFI/BOOT/BOOTX64.EFI` fallback | -| `tejo`, `sintra` | `hybrid` (GPT + `bios_grub` + ESP) | GRUB, BIOS **and** UEFI | Aaeon UP Board firmware USB-boots in Legacy/BIOS mode — it boots the live ISO via isolinux, not the ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot stick isn't recognised as bootable at all. The hybrid image boots either way | - -Both shapes come out of `mkUsbDiskImage` in `flake.nix`; the GRUB override is -`usbGrubHybridModule`. A UP Board `-usb` config installs GRUB with -`devices = [ "nodev" ]` so an in-place `switch-to-configuration` on a live -stick only regenerates `grub.cfg` — the image build is the one place the BIOS -stage gets written to an MBR. - ```bash # Build a Sintra disk image nix build .#disk-image-sintra diff --git a/deploy/nixos/hardware/batm3.nix b/deploy/nixos/hardware/batm3.nix index dcf8692..ca57f0d 100644 --- a/deploy/nixos/hardware/batm3.nix +++ b/deploy/nixos/hardware/batm3.nix @@ -223,5 +223,6 @@ }; }; - # WireGuard VPN address → wireguardIpForModel in flake.nix. + # WireGuard VPN address + networking.wireguard.interfaces.wg0.ips = [ "10.0.0.5/24" ]; } diff --git a/deploy/nixos/hardware/douro.nix b/deploy/nixos/hardware/douro.nix index 83a11cf..e2e138c 100644 --- a/deploy/nixos/hardware/douro.nix +++ b/deploy/nixos/hardware/douro.nix @@ -93,7 +93,8 @@ hybrid-sleep.enable = false; }; - # WireGuard VPN address → wireguardIpForModel in flake.nix. + # WireGuard VPN address + networking.wireguard.interfaces.wg0.ips = [ "10.0.0.4/24" ]; # Serial port access for bill validator/dispenser services.udev.extraRules = lib.mkAfter '' diff --git a/flake.nix b/flake.nix index 4037d3d..e6ddaaf 100644 --- a/flake.nix +++ b/flake.nix @@ -159,36 +159,6 @@ sintra = true; # HID Global OMNIKEY 5022 }; - # WireGuard address on the 10.0.0.0/24 management tunnel to the VPS - # (peer + listenPort live in configuration.nix; only the address is - # per-machine). Same keying caveat as the three tables above. - # - # This cannot live in a hardware file for the UP Board models, and the - # reason it now lives here for ALL of them is tejo: hardware/upboard.nix - # is shared by tejo and sintra, so an address set there would be claimed - # by both machines on the same /24. tejo had no address at all as a - # result — `wg0.ips = [ ]` brings the interface up with no IP and the - # tunnel is dead, which is a silent way to lose remote access to a - # machine that has no other route in. Keeping douro's and batm3's - # addresses here too means there is one list to read when allocating the - # next one, rather than three files plus the VPS peer config. - # - # An unlisted model gets no address and no tunnel. That is deliberate for - # sintra, which is reachable on the LAN (192.168.0.252) and has never had - # a tunnel address. - # - # NOTE: the address is only half of it. The VPS maps peer PUBLIC KEY to - # pragma: allowlist secret - # tunnel IP, so a machine also needs its private key at - # /var/lib/wireguard/wg0.key — carried over from the machine's previous - # install, or newly generated with its pubkey added to the VPS peer list. - # The key is operator-provisioned and deliberately not in the image. - wireguardIpForModel = { - tejo = "10.0.0.3/24"; - douro = "10.0.0.4/24"; - batm3 = "10.0.0.5/24"; - }; - lib = nixpkgs.lib; # Helper to create a live USB NixOS config for a specific machine model @@ -248,11 +218,6 @@ nfc.enable = nfcReaderForModel.${machineModel} or false; }; - # Management-tunnel address; see wireguardIpForModel. - networking.wireguard.interfaces.wg0.ips = - lib.optional (wireguardIpForModel ? ${machineModel}) - wireguardIpForModel.${machineModel}; - # Operator TUI and CLI tools environment.systemPackages = [ atm-tui.packages.${system}.default @@ -426,92 +391,19 @@ system.autoUpgrade.enable = lib.mkForce false; }; - # Bus hardening that makes a USB stick a reliable boot medium: keep the - # flash drive off the flaky UAS driver (many bridges advertise UAS then - # drop off the bus under sustained write load — "device offline error, - # dev sdb"), and stop USB autosuspend cutting power mid-I/O. douro.nix - # and batm3.nix carry this in their hardware files because those machines - # boot from USB exclusively; hardware/upboard.nix is SHARED with sintra's - # eMMC install, so for the UP Board models it is scoped to the -usb - # config here rather than changing a production machine's cmdline. - usbBusHardening = { - boot.blacklistedKernelModules = [ "uas" ]; - boot.kernelParams = [ "usbcore.autosuspend=-1" ]; - }; - - # Aaeon UP Board firmware (tejo, sintra) USB-boots in Legacy/BIOS mode — - # it boots the live ISO via its isolinux (BIOS) El Torito image, not the - # UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot image is not - # recognised as bootable at all. Switch the UP Board USB configs to GRUB - # with BOTH BIOS (MBR + bios_grub partition, via mkUsbDiskImage's - # partitionTableType = "hybrid") and UEFI (removable - # /EFI/BOOT/BOOTX64.EFI) — mirroring the live ISO's dual boot — so the - # stick boots on Legacy and UEFI alike. Scoped to the USB configs; the - # eMMC installs keep systemd-boot. - # - # devices = [ "nodev" ] here, NOT the image's disk. This config is also - # what in-place updates (`nix copy` + switch-to-configuration) run against - # on a LIVE stick, where the build VM's /dev/vda does not exist and a BIOS - # grub-install against it would fail the switch. "nodev" regenerates - # grub.cfg and skips the MBR write, which is the correct behaviour for an - # update: GRUB's embedded core.img reads grub.cfg off the partition, so - # the MBR stage never needs rewriting per generation. mkUsbDiskImage's - # grubBiosDevice overrides this for the image build, where the BIOS stage - # genuinely has to be written. - usbGrubHybridModule = { lib, ... }: { - boot.loader.systemd-boot.enable = lib.mkForce false; - boot.loader.efi.canTouchEfiVariables = lib.mkForce false; - boot.loader.grub = { - enable = lib.mkForce true; - efiSupport = true; - efiInstallAsRemovable = true; - # Plain definition, NOT mkForce: grubBiosDevice overrides it with - # mkForce, and two mkForce list definitions would merge (both - # priority 50) into [ "/dev/vda" "nodev" ] instead of replacing. - # Nothing else in the module stack defines grub.devices. - devices = [ "nodev" ]; - }; - }; - # dd-able USB image of a -usb config. make-disk-image gives the # ext4 root the nixos-usb label directly (-L) but hardcodes the ESP FAT # label to "ESP", so the volume is relabelled to ESP-USB afterwards — # volume label only; bootloader files are untouched and UEFI loads - # /EFI/BOOT/BOOTX64.EFI regardless. - # - # partitionTableType: "efi" (GPT + ESP, systemd-boot) for batm3 and douro, - # whose firmware UEFI-USB-boots fine via that removable fallback; - # "hybrid" (GPT + bios_grub + ESP) for the UP Board models, paired with - # usbGrubHybridModule. In BOTH layouts the ESP is partition 1 — the hybrid - # table creates the ESP first and the bios_grub partition second — so the - # parted/mlabel relabel below is layout-independent. - # - # grubBiosDevice: the build VM's disk, for hybrid images only. GRUB must - # write its BIOS stage to that disk's MBR at image-build time, while the - # config itself says "nodev" so in-place updates on a live stick work; - # see usbGrubHybridModule. - mkUsbDiskImage = - { machineModel - , usbConfig - , partitionTableType ? "efi" - , grubBiosDevice ? null - }: + # /EFI/BOOT/BOOTX64.EFI regardless. Keeps systemd-boot: both the batm3 + # and douro firmware UEFI-USB-boot fine via that removable fallback. + mkUsbDiskImage = machineModel: usbConfig: let - imageConfig = - if grubBiosDevice == null then - usbConfig - else - usbConfig.extendModules { - modules = [ - ({ lib, ... }: { - boot.loader.grub.devices = lib.mkForce [ grubBiosDevice ]; - }) - ]; - }; baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") { - inherit pkgs lib partitionTableType; - config = imageConfig.config; + inherit pkgs lib; + config = usbConfig.config; format = "raw"; + partitionTableType = "efi"; diskSize = "auto"; label = "nixos-usb"; # ext4 root label (make-disk-image -L) }; @@ -577,24 +469,6 @@ douro-usb = self.nixosConfigurations.douro-installed.extendModules { modules = [ usbBootModule ]; }; - # UP Board models get the same run-from-USB shape plus two things the - # Bay Trail / OptiPlex boxes don't need: GRUB on a hybrid table, because - # the Aaeon firmware USB-boots in Legacy/BIOS mode (usbGrubHybridModule), - # and the uas/autosuspend hardening from here instead of - # hardware/upboard.nix, which sintra's eMMC install also reads. - # - # tejo: the unit still runs its factory Debian (ubilinux4) on internal - # storage, so the stick has to BE the system, same as douro. Its - # internal install is not NixOS and carries no nixos/ESP labels, which - # makes the label disambiguation moot there today — but the hardened - # /boot and the bus settings are what make a stick a reliable boot - # medium, so it takes the identical module set as sintra. - tejo-usb = self.nixosConfigurations.tejo-installed.extendModules { - modules = [ usbBootModule usbBusHardening usbGrubHybridModule ]; - }; - sintra-usb = self.nixosConfigurations.sintra-installed.extendModules { - modules = [ usbBootModule usbBusHardening usbGrubHybridModule ]; - }; }; # ── Standalone NixOS module ─────────────────────────────────── @@ -662,6 +536,71 @@ diskSize = "auto"; }; + # USB-bootable Sintra image with DISTINCT partition labels + # (nixos-usb / ESP-USB) so the stick can be booted on a Sintra whose + # eMMC already holds a nixos/ESP-labelled install without a by-label + # collision — stage-1 would otherwise race between the two roots and + # likely mount the eMMC. Auto-upgrade is disabled: this is a portable + # test / hand-off image, not a managed fleet member, and disabling it + # also removes the scheduled bootloader writes that could otherwise + # land on the eMMC's ESP. + disk-image-sintra-usb = + let + cfg = self.nixosConfigurations.sintra-installed.extendModules { + modules = [ + ({ lib, ... }: { + fileSystems."/".device = lib.mkForce "/dev/disk/by-label/nixos-usb"; + fileSystems."/boot".device = lib.mkForce "/dev/disk/by-label/ESP-USB"; + system.autoUpgrade.enable = lib.mkForce false; + + # The Sintra's Aaeon firmware USB-boots in Legacy/BIOS mode — it + # boots the live ISO via its isolinux (BIOS) El Torito image, not + # the UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot + # image isn't recognised as bootable. Switch THIS USB image to + # GRUB with BOTH BIOS (MBR + bios_grub partition, via the "hybrid" + # table below) and UEFI (removable /EFI/BOOT/BOOTX64.EFI) — mirroring + # the live ISO's dual boot — so it boots on Legacy and UEFI alike. + # Scoped to the USB image; the eMMC install keeps systemd-boot. + boot.loader.systemd-boot.enable = lib.mkForce false; + boot.loader.efi.canTouchEfiVariables = lib.mkForce false; + boot.loader.grub = { + enable = lib.mkForce true; + efiSupport = true; + efiInstallAsRemovable = true; + # make-disk-image's build VM exposes the image as /dev/vda; + # GRUB installs its BIOS stage to that disk's MBR. + devices = lib.mkForce [ "/dev/vda" ]; + }; + }) + ]; + }; + baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") { + inherit pkgs lib; + config = cfg.config; + format = "raw"; + # hybrid = GPT + bios_grub partition + ESP → BIOS + UEFI bootable. + partitionTableType = "hybrid"; + diskSize = "auto"; + label = "nixos-usb"; # ext4 root label (make-disk-image -L) + }; + in + pkgs.runCommand "nixos-disk-image-sintra-usb" + { nativeBuildInputs = [ pkgs.parted pkgs.mtools ]; } + '' + mkdir -p $out + cp --sparse=always ${baseImage}/nixos.img $out/nixos.img + chmod +w $out/nixos.img + # make-disk-image hardcodes the ESP FAT label to "ESP"; relabel the + # volume to ESP-USB so /boot (by-label/ESP-USB) doesn't collide with + # the eMMC's ESP. Volume label only — bootloader files are untouched, + # and UEFI loads /EFI/BOOT/BOOTX64.EFI regardless of the label. + espStart=$(parted -sm "$out/nixos.img" unit B print | awk -F: '$1==1 {gsub("B","",$2); print $2}') + echo "ESP partition starts at byte $espStart — relabelling to ESP-USB" + export MTOOLS_SKIP_CHECK=1 + mlabel -i "$out/nixos.img@@$espStart" ::ESP-USB + printf 'verify ESP label: '; mlabel -i "$out/nixos.img@@$espStart" -s :: || true + ''; + # USB-bootable images (see usbBootModule / mkUsbDiskImage in the let # block). Flash with dd or balenaEtcher, boot the stick, done — no # installer step. The plain disk-image- reuses the generic @@ -670,32 +609,8 @@ # stage-1's by-label/nixos resolve to the internal drive instead of the # stick — the stage-2 init path baked into the USB's boot entry isn't on # that root, so stage 1 aborts. These variants can't hit that. - disk-image-batm3-usb = mkUsbDiskImage { - machineModel = "batm3"; - usbConfig = self.nixosConfigurations.batm3-usb; - }; - disk-image-douro-usb = mkUsbDiskImage { - machineModel = "douro"; - usbConfig = self.nixosConfigurations.douro-usb; - }; - - # UP Board models: hybrid table + GRUB, so one stick boots on the Aaeon - # firmware's Legacy/BIOS USB path as well as UEFI. Distinct labels also - # matter more here than on douro — a sintra's eMMC already holds a - # nixos/ESP-labelled install, and stage-1 would otherwise race the two - # roots and likely mount the eMMC. - disk-image-tejo-usb = mkUsbDiskImage { - machineModel = "tejo"; - usbConfig = self.nixosConfigurations.tejo-usb; - partitionTableType = "hybrid"; - grubBiosDevice = "/dev/vda"; - }; - disk-image-sintra-usb = mkUsbDiskImage { - machineModel = "sintra"; - usbConfig = self.nixosConfigurations.sintra-usb; - partitionTableType = "hybrid"; - grubBiosDevice = "/dev/vda"; - }; + disk-image-batm3-usb = mkUsbDiskImage "batm3" self.nixosConfigurations.batm3-usb; + disk-image-douro-usb = mkUsbDiskImage "douro" self.nixosConfigurations.douro-usb; # Backwards compat iso = self.nixosConfigurations.douro.config.system.build.isoImage;