From 3ab03d05acc2b885cf5a8de413f0496cdcab32a6 Mon Sep 17 00:00:00 2001
From: Patrick Mulligan <patjmulligan@protonmail.com>
Date: Thu, 19 Mar 2026 17:27:58 -0400
Subject: [PATCH] feat(machine): add renderer watchdog for kiosk resilience

After 6 days of uptime the Electron renderer silently crashed while the
main process kept running (blank screen, no recovery). Three-layer
detection: render-process-gone (instant), unresponsive (Chromium), and
IPC heartbeat (30s ping, 2 missed = reload). Reloads renderer via
loadFile/loadURL preserving HAL hardware state in main process.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 apps/machine/electron/main.ts        | 60 ++++++++++++++++++++++++++++
 apps/machine/electron/preload.ts     |  8 ++++
 apps/machine/src/App.vue             |  7 ++++
 apps/machine/src/types/electron.d.ts |  2 +
 4 files changed, 77 insertions(+)

diff --git a/apps/machine/electron/main.ts b/apps/machine/electron/main.ts
index 46696ab..14db403 100644
--- a/apps/machine/electron/main.ts
+++ b/apps/machine/electron/main.ts
@@ -92,11 +92,69 @@ function createWindow() {
   })
 }
 
+// =============================================================================
+// Renderer Watchdog
+// Detects renderer crashes, unresponsiveness, and JS-level death.
+// Reloads the renderer (preserving HAL state in main process).
+// Three layers: render-process-gone (instant), unresponsive (Chromium), heartbeat (JS-level).
+// =============================================================================
+
+function reloadRenderer() {
+  if (!mainWindow) return
+  console.log('[Watchdog] Reloading renderer...')
+  secretsConsumed = false // allow re-init after reload
+  if (isDev) {
+    const devServerUrl = process.env.VITE_DEV_SERVER_URL || 'http://localhost:1420'
+    mainWindow.loadURL(devServerUrl)
+  } else {
+    mainWindow.loadFile(path.join(__dirname, '../dist/index.html'))
+  }
+}
+
+let missedPongs = 0
+let heartbeatInterval: ReturnType<typeof setInterval> | null = null
+
+function startWatchdog() {
+  if (!mainWindow) return
+
+  // Layer 1: Chromium renderer process crashed or was killed
+  mainWindow.webContents.on('render-process-gone', (_event, details) => {
+    console.error('[Watchdog] Renderer gone:', details.reason)
+    missedPongs = 0
+    setTimeout(() => reloadRenderer(), 1000)
+  })
+
+  // Layer 2: Renderer stopped processing events (Chromium-detected)
+  mainWindow.webContents.on('unresponsive', () => {
+    console.error('[Watchdog] Renderer unresponsive, reloading...')
+    missedPongs = 0
+    reloadRenderer()
+  })
+
+  // Layer 3: IPC heartbeat — catches dead renderer JS while Chromium lives
+  heartbeatInterval = setInterval(() => {
+    if (!mainWindow) return
+    if (missedPongs >= 2) {
+      console.error('[Watchdog] Heartbeat: 2 pings unanswered, reloading...')
+      missedPongs = 0
+      reloadRenderer()
+      return
+    }
+    missedPongs++
+    mainWindow.webContents.send('watchdog:ping')
+  }, 30_000)
+}
+
 // IPC Handlers
 ipcMain.handle('get-version', () => {
   return app.getVersion()
 })
 
+// Watchdog pong: renderer confirms it's alive
+ipcMain.handle('watchdog:pong', () => {
+  missedPongs = 0
+})
+
 /**
  * Get runtime configuration from environment variables
  * This allows configuration to be set at runtime (not baked in at build time)
@@ -343,6 +401,7 @@ app.whenReady().then(() => {
   }
 
   createWindow()
+  startWatchdog()
 
   app.on('activate', () => {
     // macOS: re-create window when dock icon clicked
@@ -353,6 +412,7 @@ app.whenReady().then(() => {
 })
 
 app.on('window-all-closed', () => {
+  if (heartbeatInterval) clearInterval(heartbeatInterval)
   closeDatabase()
   // Quit on all platforms (ATM doesn't need macOS dock behavior)
   app.quit()
diff --git a/apps/machine/electron/preload.ts b/apps/machine/electron/preload.ts
index f121e85..ef19fe3 100644
--- a/apps/machine/electron/preload.ts
+++ b/apps/machine/electron/preload.ts
@@ -86,6 +86,12 @@ contextBridge.exposeInMainWorld('electronAPI', {
     ipcRenderer.on('hal:error', (_event, error) => callback(error))
   },
 
+  // Watchdog heartbeat (main process → renderer → main process)
+  onWatchdogPing: (callback: () => void) => {
+    ipcRenderer.on('watchdog:ping', () => callback())
+  },
+  watchdogPong: (): Promise<void> => ipcRenderer.invoke('watchdog:pong'),
+
   // Platform info
   platform: process.platform,
 })
@@ -126,6 +132,8 @@ declare global {
       onHalBillInserted: (callback: (denomination: number) => void) => void
       onHalBillRejected: (callback: (reason: string) => void) => void
       onHalError: (callback: (error: string) => void) => void
+      onWatchdogPing: (callback: () => void) => void
+      watchdogPong: () => Promise<void>
       platform: NodeJS.Platform
     }
   }
diff --git a/apps/machine/src/App.vue b/apps/machine/src/App.vue
index 7c132b0..4d06d80 100644
--- a/apps/machine/src/App.vue
+++ b/apps/machine/src/App.vue
@@ -26,6 +26,13 @@ const formattedBtcPrice = computed(() => {
 })
 
 onMounted(async () => {
+  // Watchdog: respond to heartbeat pings from main process
+  if (window.electronAPI?.onWatchdogPing) {
+    window.electronAPI.onWatchdogPing(() => {
+      window.electronAPI!.watchdogPong()
+    })
+  }
+
   try {
     if (isElectron) {
       await atmStore.initializeForProduction()
diff --git a/apps/machine/src/types/electron.d.ts b/apps/machine/src/types/electron.d.ts
index d2deef3..a6dd52b 100644
--- a/apps/machine/src/types/electron.d.ts
+++ b/apps/machine/src/types/electron.d.ts
@@ -60,6 +60,8 @@ declare global {
       onHalBillInserted: (callback: (denomination: number) => void) => void
       onHalBillRejected: (callback: (reason: string) => void) => void
       onHalError: (callback: (error: string) => void) => void
+      onWatchdogPing: (callback: () => void) => void
+      watchdogPong: () => Promise<void>
       platform: NodeJS.Platform
     }
   }