deploy/nixos/README.md

@@ -19,7 +19,7 @@ deploy/nixos/
 │   └── 99-bitspire-hardware.rules   # additional udev rules (loaded via configuration.nix)
 ├── provision-atm.sh        # Push LNbits credentials to a deployed ATM via SSH
 ├── atm-transactions.sh     # Operator query tool — reads /var/lib/bitspire/state.db
-├── flash-douro-usb.sh      # Helper for flashing a douro live USB
+├── flash-douro-usb.sh      # Helper for flashing a douro LIVE ISO (not the -usb disk image)
 └── build-iso.sh            # Convenience wrapper for `nix build .#iso-<model>`
 ```
 
@@ -33,19 +33,37 @@ Each ATM model has two flake outputs:
 | `nixosConfigurations.<model>-installed` | installed | full GPT + systemd-boot install, ext4 root, supports `nixos-rebuild switch` |
 | `packages.x86_64-linux.iso-<model>` | ISO | ISO image of the live variant |
 | `packages.x86_64-linux.disk-image-<model>` | raw image | dd-able full disk image of the installed variant |
-| `nixosConfigurations.<model>-usb` | installed, on a stick | `<model>-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off (`batm3`, `douro` only) |
+| `nixosConfigurations.<model>-usb` | installed, on a stick | `<model>-installed` hardened to run from a USB stick: `nixos-usb`/`ESP-USB` labels, `nofail` `/boot`, no partition growing, auto-upgrade off, `uas` blacklisted |
 | `packages.x86_64-linux.disk-image-<model>-usb` | raw image | dd-able image of the `-usb` variant — flash, plug in, boot; no installer step |
 
-Models: `douro`, `tejo`, `sintra`, `batm3`.
+Models: `douro`, `tejo`, `sintra`, `batm3`. All four have `-usb` outputs.
 
-**Run-from-USB deployments** (`batm3` today, `douro` since the LNbits cutover)
+**Run-from-USB deployments** skip the Alpine + dd-to-internal-disk procedure
-skip the Alpine + dd-to-internal-disk procedure below entirely: the stick *is*
+below entirely: the stick *is* the system. That is how `batm3` runs today, how
-the system. Flash `disk-image-<model>-usb` with balenaEtcher (it verifies the
+`douro` runs since the LNbits cutover, and how `tejo` is being brought up — its
+internal storage still holds the factory Debian (`ubilinux4`) and is never
+touched. Flash `disk-image-<model>-usb` with balenaEtcher (it verifies the
 write — a truncated or bad copy fails stage-1 fsck on first boot) onto a stick
 of 16 GB or more, plug it in, power on. Updates go in-place against the
 `<model>-usb` config (`nix copy` the toplevel + `switch-to-configuration`),
 which keeps pairing and `/var/lib/bitspire`.
 
+### Two bootloader shapes, picked by firmware
+
+The `-usb` images are not interchangeable between models, because the fleet's
+firmware is not:
+
+| Models | Partition table | Bootloader | Why |
+|---|---|---|---|
+| `batm3`, `douro` | `efi` (GPT + ESP) | systemd-boot | Their firmware UEFI-USB-boots fine via the ESP's removable `/EFI/BOOT/BOOTX64.EFI` fallback |
+| `tejo`, `sintra` | `hybrid` (GPT + `bios_grub` + ESP) | GRUB, BIOS **and** UEFI | Aaeon UP Board firmware USB-boots in Legacy/BIOS mode — it boots the live ISO via isolinux, not the ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot stick isn't recognised as bootable at all. The hybrid image boots either way |
+
+Both shapes come out of `mkUsbDiskImage` in `flake.nix`; the GRUB override is
+`usbGrubHybridModule`. A UP Board `-usb` config installs GRUB with
+`devices = [ "nodev" ]` so an in-place `switch-to-configuration` on a live
+stick only regenerates `grub.cfg` — the image build is the one place the BIOS
+stage gets written to an MBR.
+
 ```bash
 # Build a Sintra disk image
 nix build .#disk-image-sintra

deploy/nixos/hardware/batm3.nix

@@ -223,6 +223,5 @@
     };
   };
 
-  # WireGuard VPN address
+  # WireGuard VPN address → wireguardIpForModel in flake.nix.
-  networking.wireguard.interfaces.wg0.ips = [ "10.0.0.5/24" ];
 }

deploy/nixos/hardware/douro.nix

@@ -93,8 +93,7 @@
     hybrid-sleep.enable = false;
   };
 
-  # WireGuard VPN address
+  # WireGuard VPN address → wireguardIpForModel in flake.nix.
-  networking.wireguard.interfaces.wg0.ips = [ "10.0.0.4/24" ];
 
   # Serial port access for bill validator/dispenser
   services.udev.extraRules = lib.mkAfter ''

flake.nix

@@ -159,6 +159,36 @@
         sintra = true; # HID Global OMNIKEY 5022
       };
 
+      # WireGuard address on the 10.0.0.0/24 management tunnel to the VPS
+      # (peer + listenPort live in configuration.nix; only the address is
+      # per-machine). Same keying caveat as the three tables above.
+      #
+      # This cannot live in a hardware file for the UP Board models, and the
+      # reason it now lives here for ALL of them is tejo: hardware/upboard.nix
+      # is shared by tejo and sintra, so an address set there would be claimed
+      # by both machines on the same /24. tejo had no address at all as a
+      # result — `wg0.ips = [ ]` brings the interface up with no IP and the
+      # tunnel is dead, which is a silent way to lose remote access to a
+      # machine that has no other route in. Keeping douro's and batm3's
+      # addresses here too means there is one list to read when allocating the
+      # next one, rather than three files plus the VPS peer config.
+      #
+      # An unlisted model gets no address and no tunnel. That is deliberate for
+      # sintra, which is reachable on the LAN (192.168.0.252) and has never had
+      # a tunnel address.
+      #
+      # NOTE: the address is only half of it. The VPS maps peer PUBLIC KEY to
+      # pragma: allowlist secret
+      # tunnel IP, so a machine also needs its private key at
+      # /var/lib/wireguard/wg0.key — carried over from the machine's previous
+      # install, or newly generated with its pubkey added to the VPS peer list.
+      # The key is operator-provisioned and deliberately not in the image.
+      wireguardIpForModel = {
+        tejo = "10.0.0.3/24";
+        douro = "10.0.0.4/24";
+        batm3 = "10.0.0.5/24";
+      };
+
       lib = nixpkgs.lib;
 
       # Helper to create a live USB NixOS config for a specific machine model
@@ -218,6 +248,11 @@
                 nfc.enable = nfcReaderForModel.${machineModel} or false;
               };
 
+              # Management-tunnel address; see wireguardIpForModel.
+              networking.wireguard.interfaces.wg0.ips =
+                lib.optional (wireguardIpForModel ? ${machineModel})
+                  wireguardIpForModel.${machineModel};
+
               # Operator TUI and CLI tools
               environment.systemPackages = [
                 atm-tui.packages.${system}.default
@@ -391,19 +426,92 @@
         system.autoUpgrade.enable = lib.mkForce false;
       };
 
+      # Bus hardening that makes a USB stick a reliable boot medium: keep the
+      # flash drive off the flaky UAS driver (many bridges advertise UAS then
+      # drop off the bus under sustained write load — "device offline error,
+      # dev sdb"), and stop USB autosuspend cutting power mid-I/O. douro.nix
+      # and batm3.nix carry this in their hardware files because those machines
+      # boot from USB exclusively; hardware/upboard.nix is SHARED with sintra's
+      # eMMC install, so for the UP Board models it is scoped to the -usb
+      # config here rather than changing a production machine's cmdline.
+      usbBusHardening = {
+        boot.blacklistedKernelModules = [ "uas" ];
+        boot.kernelParams = [ "usbcore.autosuspend=-1" ];
+      };
+
+      # Aaeon UP Board firmware (tejo, sintra) USB-boots in Legacy/BIOS mode —
+      # it boots the live ISO via its isolinux (BIOS) El Torito image, not the
+      # UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot image is not
+      # recognised as bootable at all. Switch the UP Board USB configs to GRUB
+      # with BOTH BIOS (MBR + bios_grub partition, via mkUsbDiskImage's
+      # partitionTableType = "hybrid") and UEFI (removable
+      # /EFI/BOOT/BOOTX64.EFI) — mirroring the live ISO's dual boot — so the
+      # stick boots on Legacy and UEFI alike. Scoped to the USB configs; the
+      # eMMC installs keep systemd-boot.
+      #
+      # devices = [ "nodev" ] here, NOT the image's disk. This config is also
+      # what in-place updates (`nix copy` + switch-to-configuration) run against
+      # on a LIVE stick, where the build VM's /dev/vda does not exist and a BIOS
+      # grub-install against it would fail the switch. "nodev" regenerates
+      # grub.cfg and skips the MBR write, which is the correct behaviour for an
+      # update: GRUB's embedded core.img reads grub.cfg off the partition, so
+      # the MBR stage never needs rewriting per generation. mkUsbDiskImage's
+      # grubBiosDevice overrides this for the image build, where the BIOS stage
+      # genuinely has to be written.
+      usbGrubHybridModule = { lib, ... }: {
+        boot.loader.systemd-boot.enable = lib.mkForce false;
+        boot.loader.efi.canTouchEfiVariables = lib.mkForce false;
+        boot.loader.grub = {
+          enable = lib.mkForce true;
+          efiSupport = true;
+          efiInstallAsRemovable = true;
+          # Plain definition, NOT mkForce: grubBiosDevice overrides it with
+          # mkForce, and two mkForce list definitions would merge (both
+          # priority 50) into [ "/dev/vda" "nodev" ] instead of replacing.
+          # Nothing else in the module stack defines grub.devices.
+          devices = [ "nodev" ];
+        };
+      };
+
       # dd-able USB image of a <model>-usb config. make-disk-image gives the
       # ext4 root the nixos-usb label directly (-L) but hardcodes the ESP FAT
       # label to "ESP", so the volume is relabelled to ESP-USB afterwards —
       # volume label only; bootloader files are untouched and UEFI loads
-      # /EFI/BOOT/BOOTX64.EFI regardless. Keeps systemd-boot: both the batm3
+      # /EFI/BOOT/BOOTX64.EFI regardless.
-      # and douro firmware UEFI-USB-boot fine via that removable fallback.
+      #
-      mkUsbDiskImage = machineModel: usbConfig:
+      # partitionTableType: "efi" (GPT + ESP, systemd-boot) for batm3 and douro,
+      # whose firmware UEFI-USB-boots fine via that removable fallback;
+      # "hybrid" (GPT + bios_grub + ESP) for the UP Board models, paired with
+      # usbGrubHybridModule. In BOTH layouts the ESP is partition 1 — the hybrid
+      # table creates the ESP first and the bios_grub partition second — so the
+      # parted/mlabel relabel below is layout-independent.
+      #
+      # grubBiosDevice: the build VM's disk, for hybrid images only. GRUB must
+      # write its BIOS stage to that disk's MBR at image-build time, while the
+      # config itself says "nodev" so in-place updates on a live stick work;
+      # see usbGrubHybridModule.
+      mkUsbDiskImage =
+        { machineModel
+        , usbConfig
+        , partitionTableType ? "efi"
+        , grubBiosDevice ? null
+        }:
         let
+          imageConfig =
+            if grubBiosDevice == null then
+              usbConfig
+            else
+              usbConfig.extendModules {
+                modules = [
+                  ({ lib, ... }: {
+                    boot.loader.grub.devices = lib.mkForce [ grubBiosDevice ];
+                  })
+                ];
+              };
           baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") {
-            inherit pkgs lib;
+            inherit pkgs lib partitionTableType;
-            config = usbConfig.config;
+            config = imageConfig.config;
             format = "raw";
-            partitionTableType = "efi";
             diskSize = "auto";
             label = "nixos-usb"; # ext4 root label (make-disk-image -L)
           };
@@ -469,6 +577,24 @@
         douro-usb = self.nixosConfigurations.douro-installed.extendModules {
           modules = [ usbBootModule ];
         };
+        # UP Board models get the same run-from-USB shape plus two things the
+        # Bay Trail / OptiPlex boxes don't need: GRUB on a hybrid table, because
+        # the Aaeon firmware USB-boots in Legacy/BIOS mode (usbGrubHybridModule),
+        # and the uas/autosuspend hardening from here instead of
+        # hardware/upboard.nix, which sintra's eMMC install also reads.
+        #
+        # tejo: the unit still runs its factory Debian (ubilinux4) on internal
+        # storage, so the stick has to BE the system, same as douro. Its
+        # internal install is not NixOS and carries no nixos/ESP labels, which
+        # makes the label disambiguation moot there today — but the hardened
+        # /boot and the bus settings are what make a stick a reliable boot
+        # medium, so it takes the identical module set as sintra.
+        tejo-usb = self.nixosConfigurations.tejo-installed.extendModules {
+          modules = [ usbBootModule usbBusHardening usbGrubHybridModule ];
+        };
+        sintra-usb = self.nixosConfigurations.sintra-installed.extendModules {
+          modules = [ usbBootModule usbBusHardening usbGrubHybridModule ];
+        };
       };
 
       # ── Standalone NixOS module ───────────────────────────────────
@@ -536,71 +662,6 @@
             diskSize = "auto";
           };
 
-        # USB-bootable Sintra image with DISTINCT partition labels
-        # (nixos-usb / ESP-USB) so the stick can be booted on a Sintra whose
-        # eMMC already holds a nixos/ESP-labelled install without a by-label
-        # collision — stage-1 would otherwise race between the two roots and
-        # likely mount the eMMC. Auto-upgrade is disabled: this is a portable
-        # test / hand-off image, not a managed fleet member, and disabling it
-        # also removes the scheduled bootloader writes that could otherwise
-        # land on the eMMC's ESP.
-        disk-image-sintra-usb =
-          let
-            cfg = self.nixosConfigurations.sintra-installed.extendModules {
-              modules = [
-                ({ lib, ... }: {
-                  fileSystems."/".device = lib.mkForce "/dev/disk/by-label/nixos-usb";
-                  fileSystems."/boot".device = lib.mkForce "/dev/disk/by-label/ESP-USB";
-                  system.autoUpgrade.enable = lib.mkForce false;
-
-                  # The Sintra's Aaeon firmware USB-boots in Legacy/BIOS mode — it
-                  # boots the live ISO via its isolinux (BIOS) El Torito image, not
-                  # the UEFI ESP. systemd-boot is UEFI-only, so a dd'd systemd-boot
-                  # image isn't recognised as bootable. Switch THIS USB image to
-                  # GRUB with BOTH BIOS (MBR + bios_grub partition, via the "hybrid"
-                  # table below) and UEFI (removable /EFI/BOOT/BOOTX64.EFI) — mirroring
-                  # the live ISO's dual boot — so it boots on Legacy and UEFI alike.
-                  # Scoped to the USB image; the eMMC install keeps systemd-boot.
-                  boot.loader.systemd-boot.enable = lib.mkForce false;
-                  boot.loader.efi.canTouchEfiVariables = lib.mkForce false;
-                  boot.loader.grub = {
-                    enable = lib.mkForce true;
-                    efiSupport = true;
-                    efiInstallAsRemovable = true;
-                    # make-disk-image's build VM exposes the image as /dev/vda;
-                    # GRUB installs its BIOS stage to that disk's MBR.
-                    devices = lib.mkForce [ "/dev/vda" ];
-                  };
-                })
-              ];
-            };
-            baseImage = import (nixpkgs + "/nixos/lib/make-disk-image.nix") {
-              inherit pkgs lib;
-              config = cfg.config;
-              format = "raw";
-              # hybrid = GPT + bios_grub partition + ESP → BIOS + UEFI bootable.
-              partitionTableType = "hybrid";
-              diskSize = "auto";
-              label = "nixos-usb"; # ext4 root label (make-disk-image -L)
-            };
-          in
-          pkgs.runCommand "nixos-disk-image-sintra-usb"
-            { nativeBuildInputs = [ pkgs.parted pkgs.mtools ]; }
-            ''
-              mkdir -p $out
-              cp --sparse=always ${baseImage}/nixos.img $out/nixos.img
-              chmod +w $out/nixos.img
-              # make-disk-image hardcodes the ESP FAT label to "ESP"; relabel the
-              # volume to ESP-USB so /boot (by-label/ESP-USB) doesn't collide with
-              # the eMMC's ESP. Volume label only — bootloader files are untouched,
-              # and UEFI loads /EFI/BOOT/BOOTX64.EFI regardless of the label.
-              espStart=$(parted -sm "$out/nixos.img" unit B print | awk -F: '$1==1 {gsub("B","",$2); print $2}')
-              echo "ESP partition starts at byte $espStart — relabelling to ESP-USB"
-              export MTOOLS_SKIP_CHECK=1
-              mlabel -i "$out/nixos.img@@$espStart" ::ESP-USB
-              printf 'verify ESP label: '; mlabel -i "$out/nixos.img@@$espStart" -s :: || true
-            '';
-
         # USB-bootable images (see usbBootModule / mkUsbDiskImage in the let
         # block). Flash with dd or balenaEtcher, boot the stick, done — no
         # installer step. The plain disk-image-<model> reuses the generic
@@ -609,8 +670,32 @@
         # stage-1's by-label/nixos resolve to the internal drive instead of the
         # stick — the stage-2 init path baked into the USB's boot entry isn't on
         # that root, so stage 1 aborts. These variants can't hit that.
-        disk-image-batm3-usb = mkUsbDiskImage "batm3" self.nixosConfigurations.batm3-usb;
+        disk-image-batm3-usb = mkUsbDiskImage {
-        disk-image-douro-usb = mkUsbDiskImage "douro" self.nixosConfigurations.douro-usb;
+          machineModel = "batm3";
+          usbConfig = self.nixosConfigurations.batm3-usb;
+        };
+        disk-image-douro-usb = mkUsbDiskImage {
+          machineModel = "douro";
+          usbConfig = self.nixosConfigurations.douro-usb;
+        };
+
+        # UP Board models: hybrid table + GRUB, so one stick boots on the Aaeon
+        # firmware's Legacy/BIOS USB path as well as UEFI. Distinct labels also
+        # matter more here than on douro — a sintra's eMMC already holds a
+        # nixos/ESP-labelled install, and stage-1 would otherwise race the two
+        # roots and likely mount the eMMC.
+        disk-image-tejo-usb = mkUsbDiskImage {
+          machineModel = "tejo";
+          usbConfig = self.nixosConfigurations.tejo-usb;
+          partitionTableType = "hybrid";
+          grubBiosDevice = "/dev/vda";
+        };
+        disk-image-sintra-usb = mkUsbDiskImage {
+          machineModel = "sintra";
+          usbConfig = self.nixosConfigurations.sintra-usb;
+          partitionTableType = "hybrid";
+          grubBiosDevice = "/dev/vda";
+        };
 
         # Backwards compat
         iso = self.nixosConfigurations.douro.config.system.build.isoImage;