bitspire/deploy/nixos/configuration.nix
Padreug 9b07880925 fix(deploy): the .env migration called sed and grep by bare name on the activation PATH
NixOS activation scripts run with a minimal PATH that has coreutils but
not gnugrep or gnused. On sintra the snippet printed its success line and
then failed with "sed: command not found" (127) — the keys were never
renamed, the new app booted on fallback config, and switch-to-
configuration exited 2. Both binaries are now referenced by store path.
2026-10-10 22:08:48 +02:00

531 lines
23 KiB
Nix

# bitSpire ATM NixOS Configuration
# Base system configuration for ATM kiosk
{ config, lib, pkgs, pkgs-unstable, ... }:
let
# ── Firmware pruning (bitspire#70 sizing) ────────────────────────────
# hardware.enableRedistributableFirmware installs the entire linux-firmware
# tree: 752MB compressed, 16% of the image and its single largest item. The
# fleet is four fixed Intel boards. The other ~640MB is firmware for
# Qualcomm, Mellanox, NVIDIA, Marvell, AMD and MediaTek parts that will
# never appear in one of these machines.
#
# Keep only what a bitSpire board can plausibly load. Entries are paths
# inside lib/firmware; nothing outside this list is copied.
firmwareKeep = [
# Intel GPU. Gen9 (Apollo Lake) loads DMC from here. Bay Trail and
# Haswell load nothing, but 9.6MB is cheap insurance against a board swap.
"i915"
# Intel WiFi, 89MB and the bulk of what survives, covering every Intel
# card since 2008. This is the conservative half of the trade: losing the
# network on a deployed ATM is not remotely recoverable. Narrow it to the
# specific generation once each machine's card is known, via
# `lspci -k | grep -A3 Network` on the box.
"intel/iwlwifi"
"rtl_nic" # Realtek GbE (r8169) — the UP Board's onboard NIC
"rtw88" # Realtek WiFi, the usual M.2 or USB retrofit
"rtw89"
"brcm" # Broadcom WiFi, the other usual retrofit
# Intel Smart Sound Technology DSP, 420KB. Cherry Trail boards (sintra,
# tejo) probe intel_sst_acpi at boot whether or not anything will use the
# audio, and without the blob every boot logs
# Direct firmware load for intel/fw_sst_22a8.bin failed with error -2
# Found by pruning, rebooting sintra and reading dmesg. The audio stack is
# gone so this changes no behaviour, but a recurring error in a payment
# terminal's boot log is worth 420KB to remove: an error people learn to
# ignore is one they will ignore when it matters.
"intel/fw_sst_0f28.bin"
"intel/fw_sst_0f28_ssp0.bin"
"intel/fw_sst_22a8.bin"
];
# Prune the tree rather than hand-pick files, so a firmware bump can't
# silently drop a blob we depend on. Left UNCOMPRESSED on purpose: NixOS
# compresses each hardware.firmware entry itself, zstd or xz depending on
# what the machine's kernel understands, and douro's 5.15 predates zstd
# firmware support. Pre-compressing here would hand douro a tree it cannot
# read.
bitspireFirmware = pkgs.runCommand "linux-firmware-bitspire"
{
inherit (pkgs.linux-firmware) version;
meta = pkgs.linux-firmware.meta // {
description = "linux-firmware pruned to the hardware bitSpire ships on";
};
}
''
src=${pkgs.linux-firmware}/lib/firmware
dst=$out/lib/firmware
mkdir -p "$dst"
for p in ${lib.escapeShellArgs firmwareKeep}; do
if [ ! -e "$src/$p" ]; then
echo "ERROR: firmwareKeep entry '$p' is not in linux-firmware" >&2
exit 1
fi
mkdir -p "$dst/$(dirname "$p")"
cp -a "$src/$p" "$dst/$p"
done
# A kept directory can contain symlinks pointing at blobs OUTSIDE it:
# brcm/brcmfmac*.bin are links into cypress/, for instance. Left dangling
# they fail nixpkgs' firmware compression step, and silently deleting
# them would quietly drop firmware a device needs. So pull the targets in
# instead. Looped because a resolved target can itself be a link.
for _pass in 1 2 3; do
_pulled=0
while IFS= read -r link; do
tgt=$(readlink -m "$link")
case "$tgt" in
"$dst"/*) rel=''${tgt#"$dst"/} ;;
*) continue ;;
esac
if [ ! -e "$dst/$rel" ] && [ -e "$src/$rel" ]; then
mkdir -p "$dst/$(dirname "$rel")"
cp -a "$src/$rel" "$dst/$rel"
_pulled=1
fi
done < <(find "$dst" -xtype l)
[ "$_pulled" -eq 0 ] && break
done
# Anything still dangling is not in linux-firmware at all. Fail loudly
# rather than ship a tree with holes in it.
if find "$dst" -xtype l | grep -q .; then
echo "ERROR: dangling firmware symlinks after resolution:" >&2
find "$dst" -xtype l >&2
exit 1
fi
# linux-firmware stores many blobs under a vendor directory and leaves a
# flat top-level symlink pointing at them, e.g.
# iwlwifi-cc-a0-77.ucode -> intel/iwlwifi/iwlwifi-cc-a0-77.ucode. The
# kernel requests the flat name, so a kept blob is useless without its
# link. Recreate every top-level link whose target survived the prune.
( cd "$src"
find . -maxdepth 1 -type l -printf '%f\t%l\n' \
| while IFS="$(printf '\t')" read -r link target; do
# if/then, not `[ ... ] && ln`: the latter makes the loop's exit
# status depend on whether the LAST candidate matched, and a
# non-match returns 1, which set -e turns into a build failure.
# Whether it fails is then a function of readdir order.
if [ -e "$dst/$target" ]; then
ln -s "$target" "$dst/$link"
fi
done
)
echo "firmware kept: $(find "$dst" -type f | wc -l) files, \
$(find "$dst" -type l | wc -l) links, $(du -sh "$dst" | cut -f1) uncompressed"
'';
in
{
# System basics
system.stateVersion = "24.05";
# ── Image slimming (bitspire#70 sizing) ──────────────────────────────
# This is a single-purpose Electron kiosk; strip the desktop/multimedia
# baggage NixOS pulls in by default so the disk image stays lean.
# - speechd: text-to-speech (speech-dispatcher → espeak-ng → mbrola, ~1GB).
# An ATM does not talk.
# - documentation: man/info/NixOS manual — no one reads them on a kiosk.
# - pipewire: the audio stack (+ WirePlumber, ALSA, the PulseAudio shim),
# ~353MB. The app has never played a sound — nothing under apps/machine or
# packages/ constructs an Audio element or ships an audio file.
#
# All three need mkForce, not just an absent/false assignment: enabling
# services.xserver pulls in NixOS's `graphical-desktop` module, which
# mkDefault-enables speechd AND pipewire (services/misc/graphical-desktop.nix).
# Dropping our own `enable = true` simply falls back to that default — the
# 353MB stayed until this was forced off. Re-enable pipewire (with alsa +
# pulse) and security.rtkit if transaction sounds are ever added.
services.speechd.enable = lib.mkForce false;
services.pipewire.enable = lib.mkForce false;
documentation.enable = false;
documentation.nixos.enable = false;
# Ship the pruned firmware tree instead of all of linux-firmware. mkForce
# because every hardware/*.nix sets enableRedistributableFirmware = true;
# overriding once here keeps the four machines in step. Turning that option
# off also drops the extras it bundles (sof-firmware, libreelec-dvb,
# alsa-firmware, intel2200BG, zd1211fw and friends), none of which applies to
# a soundless kiosk on a wired Intel board. The regulatory database is
# normally implied by the same option, so ask for it explicitly: without it
# WiFi is pinned to the most restrictive channel set.
hardware.enableRedistributableFirmware = lib.mkForce false;
hardware.wirelessRegulatoryDatabase = true;
hardware.firmware = [ bitspireFirmware ];
# Make the prune stick. Without this, a nixpkgs bump or a stray module
# setting enableRedistributableFirmware back to true silently re-adds 750MB
# and nobody notices until an eMMC runs out of room at 04:00. The regex
# matches the upstream package's versioned name (linux-firmware-20260519)
# and deliberately not ours (linux-firmware-bitspire), so the pruned tree
# passes and the full one fails the build with a readable error.
system.forbiddenDependenciesRegexes = [ "linux-firmware-[0-9]" ];
# Mesa without an LLVM-backed rasterizer.
#
# nixpkgs builds Mesa with 21 gallium drivers. Two of them, llvmpipe and
# radeonsi, link LLVM, and that RPATH pulls llvm-21-lib into the system
# closure: 540MB, a ninth of the image, on a kiosk with a soldered Intel GPU.
#
# The driver list has to span three Intel generations:
# crocus EVERY machine in the fleet. Surveyed, not assumed: sintra
# and tejo are Braswell [8086:22b0], batm3 is Haswell GT2
# [8086:0412], douro is Bay Trail. sintra and batm3 were read
# straight off their running X logs; both say crocus.
# i915 pre-Gen4, insurance against an older board turning up.
# softpipe the software rasterizer that does NOT use LLVM. Kept so a
# board whose KMS driver fails still brings up X, slowly,
# rather than dying headless in the field.
#
# ── IRIS IS DELIBERATELY ABSENT AND RE-ADDING IT COSTS 540MB ────────
# iris covers Gen8+ big-core Intel, which nothing here has. Its absence is
# what lets -Dllvm=disabled below work: mesa's meson puts
# with_gallium_iris in with_driver_using_cl and then
# with_llvm.enable_if(with_clc, 'CLC requires LLVM')
# so asking for iris drags in the OpenCL frontend and the whole of
# llvm-lib. Mesa's closure is 88MB without iris, 633MB with.
#
# A newer x86 board — a modern NUC, the "build it from these parts" kiosk
# — WILL need iris. Until one exists, such a board falls back to softpipe
# and renders in software: it boots, it displays, it looks fine, and it is
# very slow. Check `DRI driver:` in /var/log/X.0.log on any new hardware
# rather than assuming this list still covers it.
# i915 pre-Gen4, insurance against an older board turning up
# softpipe the software rasterizer that does NOT use LLVM. Kept so a
# board whose KMS driver fails still brings up X, slowly,
# rather than dying headless in the field. This is the role
# llvmpipe was playing, for 540MB.
#
# Vulkan is emptied because nothing here uses it, and its software ICD
# (lavapipe) is the other LLVM consumer. The VDPAU and VA state trackers
# have to go with it: meson refuses to build them unless one of the AMD or
# NVIDIA gallium drivers is present. Intel VA-API is unaffected, it comes
# from intel-media-driver in hardware/*.nix.
hardware.graphics.package =
(pkgs.mesa.override {
galliumDrivers = [ "crocus" "i915" "softpipe" ];
vulkanDrivers = [ ];
vulkanLayers = [ ];
}).overrideAttrs
(old: {
mesonFlags = old.mesonFlags ++ [
# Severs LLVM outright. Only possible because iris is out of the
# driver list above; with iris present meson refuses this flag.
# Verified with patchelf: libgallium.so ends up with no libLLVM in
# its DT_NEEDED, not merely absent from the closure listing.
# Dropping llvmpipe alone never achieved this.
(lib.mesonEnable "llvm" false)
(lib.mesonBool "gallium-rusticl" false)
# nixpkgs builds the asahi/panfrost cross tools and installs
# mesa-clc on native builds. Both reference prog_mesa_clc, which
# exists only when CLC is on, so they go with LLVM. An x86 kiosk
# has no use for either.
(lib.mesonOption "tools" "")
(lib.mesonBool "install-mesa-clc" false)
(lib.mesonBool "install-precomp-compiler" false)
(lib.mesonEnable "gallium-vdpau" false)
(lib.mesonEnable "gallium-va" false)
(lib.mesonEnable "intel-rt" false)
];
# Mesa declares spirv2dxil and cross_tools as outputs unconditionally,
# but they only receive files when the d3d12, asahi or panfrost gallium
# drivers are built, and none of those are in the list above. Nix fails
# a build that leaves a declared output unproduced, so create them
# empty. (Mesa sets __structuredAttrs, so $outputs is a bash array and
# a plain `for o in $outputs` loop silently does nothing here.)
postInstall = (old.postInstall or "") + ''
mkdir -p "$spirv2dxil" "$cross_tools" "$opencl"
'';
# With rusticl off there is no libRusticlOpenCL.so, and Mesa's
# postFixup patchelfs it unconditionally. Drop just that argument.
# The assert makes a nixpkgs bump that reshapes this line fail loudly
# here rather than silently stop removing LLVM.
postFixup =
let
marker = " $opencl/lib/libRusticlOpenCL.so";
in
assert lib.assertMsg (lib.hasInfix marker old.postFixup)
"mesa postFixup no longer patchelfs libRusticlOpenCL.so; revisit this override";
lib.replaceStrings [ marker ] [ "" ] old.postFixup;
});
# Networking
networking = {
hostName = "bitspire";
# Use NetworkManager for easy WiFi configuration
networkmanager.enable = true;
# Firewall - minimal exposure
firewall = {
enable = true;
allowedTCPPorts = [ ]; # ATM initiates all connections
allowedUDPPorts = [ 51820 ]; # WireGuard
};
# WireGuard VPN tunnel to VPS for remote SSH access
# IP address set per-machine in hardware/*.nix via networking.wireguard.interfaces.wg0.ips
wireguard.interfaces.wg0 = {
listenPort = 51820;
privateKeyFile = "/var/lib/wireguard/wg0.key";
peers = [{
publicKey = "R6uB4o5ELEKEHCvK+llRYbzdkZGDHegVmS0f08aRtWM=";
endpoint = "170.75.161.21:51820";
allowedIPs = [ "10.0.0.0/24" ];
persistentKeepalive = 25;
}];
};
};
# Timezone - set to your location
time.timeZone = "America/Guatemala";
# Locale
i18n.defaultLocale = "en_US.UTF-8";
# Users
users.groups.bitspire = { };
users.users.bitspire = {
isNormalUser = true;
group = "bitspire";
description = "bitSpire ATM";
home = "/home/bitspire";
extraGroups = [
"wheel" # For admin access
"video" # GPU access
"audio" # Sound
"dialout" # Serial ports
"plugdev" # USB devices
"networkmanager" # Network config
];
# No password - kiosk mode
initialPassword = "bitspire"; # pragma: allowlist secret
};
# Kiosk display configuration
services.xserver = {
enable = true;
# No desktop environment - just the ATM app
desktopManager.xterm.enable = false;
# Basic window manager for Electron
windowManager.openbox.enable = true;
# Disable screen blanking
serverFlagsSection = ''
Option "BlankTime" "0"
Option "StandbyTime" "0"
Option "SuspendTime" "0"
Option "OffTime" "0"
'';
# Intel driver
videoDrivers = [ "modesetting" ];
};
# Display manager - auto-login (top-level since NixOS 24.11+)
services.displayManager.autoLogin = {
enable = true;
user = "bitspire";
};
# System packages
#
# Kept deliberately thin — this is a kiosk, and every entry here is closure
# that ships to each ATM and eats eMMC headroom the nightly rebuild needs.
# Deliberately absent (see #70 sizing):
# git 70MB. nixos-rebuild fetches the flake with its OWN git-minimal,
# which stays in the closure via unit-nixos-upgrade.service, so
# auto-upgrade is unaffected.
# vim 43MB. Replaced by nano — an on-box editor is worth a few MB for
# field edits to /var/lib/bitspire/.env, vim's bulk is not.
# nodejs_22 94MB. Nothing runs it: the app is Electron (which embeds its
# own node) and fund-atm already pins pkgs-unstable.nodejs itself.
# wget curl covers it.
environment.systemPackages = with pkgs; [
# System utilities
htop
nano
curl
# Hardware debugging
usbutils
pciutils
lsof
# Serial port tools (validator/dispenser live on ttyJ5/ttyJ7 — these are
# how a field fault gets diagnosed, and they cost ~2MB between them)
minicom
screen
# For the Electron app
pkgs-unstable.electron
# Camera support. v4l-utils' default build drags in the whole Qt6 stack
# for its qv4l2 GUI (~0.5GB) — we only ever use the v4l2-ctl CLI, so drop
# the GUI.
(v4l-utils.override { withGUI = false; })
fswebcam
# ATM operations
sqlite
(writeShellScriptBin "atm-transactions" (builtins.readFile ./atm-transactions.sh))
(writeShellScriptBin "atm-reconcile" (builtins.readFile ./atm-reconcile.sh))
];
# Enable SSH for remote administration
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
PermitRootLogin = "prohibit-password";
};
};
# Root SSH key access
users.users.root.openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIES8I43AgXppATvtBqnUycakMMs68T2J52cTwNt6qPak padreug@gizmo"
];
# Auto-updates (optional - disabled by default for stability)
# system.autoUpgrade.enable = false;
# Trust the Forgejo host key up front. system.autoUpgrade fetches the flake
# over ssh AS ROOT, and a machine whose root has never connected by hand has
# no known_hosts entry, so every nightly run dies at
# "Host key verification failed" before it reaches authentication. batm3 did
# exactly that, silently, from its 2026-08-06 install until 09-22 (#98): it
# sat on its install generation for six weeks while reporting a failed unit
# nobody was watching. sintra only ever worked because a human had ssh'd as
# root once and accepted the key. Declaring it means a freshly flashed ATM
# can update from first boot with no manual step.
programs.ssh.knownHosts."git.atitlan.io".publicKey =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlo3f05o4+bk0+8x2VG91o9GubshOb46HmBPvND9pJx";
# pragma: allowlist secret
# Ensure WireGuard private key directory exists with correct permissions
system.activationScripts.wireguard-key = ''
mkdir -p /var/lib/wireguard
chmod 700 /var/lib/wireguard
if [ -f /var/lib/wireguard/wg0.key ]; then
chmod 600 /var/lib/wireguard/wg0.key
fi
'';
# The tunnel is operator-provisioned: wg0.key is written per machine after
# flashing, and until it is, `wg set … private-key` exits 1 with
# "fopen: No such file or directory". One failed unit makes
# switch-to-configuration exit 4, which marks the entire nightly
# system.autoUpgrade run as failed — so an ATM that simply never had its
# tunnel provisioned reports a broken updater for the life of the machine
# (sintra, #98). Skip the unit when there is no key instead of failing
# activation over an interface that was never set up; a provisioned machine
# is unaffected. Guarded on wg0 still being declared so the live image,
# which mkForce's the interfaces away, doesn't get a unit with no ExecStart.
systemd.services = lib.mkIf (config.networking.wireguard.interfaces ? wg0) (
let
iface = config.networking.wireguard.interfaces.wg0;
guard = { unitConfig.ConditionPathExists = "/var/lib/wireguard/wg0.key"; };
# The module emits one unit per peer alongside the interface unit, and a
# skipped interface is NOT a failed dependency, so the peer units still
# run and die on "Unable to modify interface: No such device" — same
# exit 4, different unit. Guard them too. Names come from the module's
# own `peers.*.name` option (whose default is the escaped public key)
# rather than re-deriving the escaping here; the `-refresh` suffix
# follows nixpkgs' peerUnitServiceName, where a peer's null refresh
# interval falls back to the interface's.
refreshes = peer:
(if peer.dynamicEndpointRefreshSeconds != null then
peer.dynamicEndpointRefreshSeconds
else
iface.dynamicEndpointRefreshSeconds) != 0;
peerUnit = peer:
"wireguard-wg0-peer-${peer.name}" + lib.optionalString (refreshes peer) "-refresh";
in
{ wireguard-wg0 = guard; }
// lib.listToAttrs (map (peer: lib.nameValuePair (peerUnit peer) guard) iface.peers)
);
# In-place rename migration: lamassu user → bitspire user.
# Runs after `users` activation so the bitspire user exists with its UID.
# Idempotent: re-running on an already-migrated system is a chown no-op.
# Leaves /home/lamassu in place as evidence — operator can `rm -rf` after
# confirming bitspire works.
system.activationScripts.bitspire-user-migration = {
deps = [ "users" ];
text = ''
# SSH key migration: copy authorized_keys to /home/bitspire if missing,
# so the dev box can still SSH in as bitspire after the rename.
if [ -f /home/lamassu/.ssh/authorized_keys ] \
&& [ ! -f /home/bitspire/.ssh/authorized_keys ]; then
mkdir -p /home/bitspire/.ssh
cp /home/lamassu/.ssh/authorized_keys /home/bitspire/.ssh/authorized_keys
chown -R bitspire:bitspire /home/bitspire/.ssh
chmod 700 /home/bitspire/.ssh
chmod 600 /home/bitspire/.ssh/authorized_keys
fi
# Data dir ownership: state.db / .env / branding/ may still be owned by
# the now-removed lamassu UID. Reset every boot — cheap no-op once done.
if [ -d /var/lib/bitspire ]; then
chown -R bitspire:bitspire /var/lib/bitspire
fi
'';
};
# In-place rename migration: VITE_LAMASSU_* → VITE_BITSPIRE_* in the
# provisioned .env. The machine reads MACHINE_MODEL / FIAT_CODE / CASSETTES
# from this file on every boot; renaming the keys in code without renaming
# them here would boot a live machine on preset defaults (wrong bays, wrong
# fiat) at the next nightly pull. Idempotent: a migrated file has nothing
# left to match. Runs before bitspire.service starts.
system.activationScripts.bitspire-env-migration = {
deps = [ "users" ];
text = ''
# Activation scripts run with a minimal PATH (coreutils, not gnugrep /
# gnused) — the first run of this snippet died with "sed: command not
# found" after printing success, so reference both by store path.
if [ -f /var/lib/bitspire/.env ] \
&& ${pkgs.gnugrep}/bin/grep -q '^VITE_LAMASSU_' /var/lib/bitspire/.env; then
${pkgs.gnused}/bin/sed -i 's/^VITE_LAMASSU_/VITE_BITSPIRE_/' /var/lib/bitspire/.env
echo "bitspire: migrated VITE_LAMASSU_* keys in /var/lib/bitspire/.env"
fi
'';
};
# Journal configuration
services.journald = {
extraConfig = ''
SystemMaxUse=100M
MaxRetentionSec=1week
'';
};
# Nix settings
nix = {
settings = {
experimental-features = [ "nix-command" "flakes" ];
auto-optimise-store = true;
};
# Garbage collection — daily at 03:30, 30 min before the 04:00 auto-
# upgrade so each upgrade attempt gets the freshest headroom. 15GB
# eMMC + ~7GB closure means cross-release upgrades are always tight;
# weekly was leaving up to a week of generations stacked when the
# upgrade ran (caught 2026-05-26 on the 24.05 → 24.11 attempt).
# persistent so a Sintra that was powered off at 03:30 still runs the
# GC on next boot rather than skipping until next week.
gc = {
automatic = true;
dates = "03:30";
options = "--delete-older-than 7d";
persistent = true;
};
};
}