feat(machine): add renderer watchdog for kiosk resilience

After 6 days of uptime the Electron renderer silently crashed while the
main process kept running (blank screen, no recovery). Three-layer
detection: render-process-gone (instant), unresponsive (Chromium), and
IPC heartbeat (30s ping, 2 missed = reload). Reloads renderer via
loadFile/loadURL preserving HAL hardware state in main process.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Patrick Mulligan 2026-03-19 17:27:58 -04:00
commit 3ab03d05ac
4 changed files with 77 additions and 0 deletions

View file

@ -92,11 +92,69 @@ function createWindow() {
}) })
} }
// =============================================================================
// Renderer Watchdog
// Detects renderer crashes, unresponsiveness, and JS-level death.
// Reloads the renderer (preserving HAL state in main process).
// Three layers: render-process-gone (instant), unresponsive (Chromium), heartbeat (JS-level).
// =============================================================================
function reloadRenderer() {
if (!mainWindow) return
console.log('[Watchdog] Reloading renderer...')
secretsConsumed = false // allow re-init after reload
if (isDev) {
const devServerUrl = process.env.VITE_DEV_SERVER_URL || 'http://localhost:1420'
mainWindow.loadURL(devServerUrl)
} else {
mainWindow.loadFile(path.join(__dirname, '../dist/index.html'))
}
}
let missedPongs = 0
let heartbeatInterval: ReturnType<typeof setInterval> | null = null
function startWatchdog() {
if (!mainWindow) return
// Layer 1: Chromium renderer process crashed or was killed
mainWindow.webContents.on('render-process-gone', (_event, details) => {
console.error('[Watchdog] Renderer gone:', details.reason)
missedPongs = 0
setTimeout(() => reloadRenderer(), 1000)
})
// Layer 2: Renderer stopped processing events (Chromium-detected)
mainWindow.webContents.on('unresponsive', () => {
console.error('[Watchdog] Renderer unresponsive, reloading...')
missedPongs = 0
reloadRenderer()
})
// Layer 3: IPC heartbeat — catches dead renderer JS while Chromium lives
heartbeatInterval = setInterval(() => {
if (!mainWindow) return
if (missedPongs >= 2) {
console.error('[Watchdog] Heartbeat: 2 pings unanswered, reloading...')
missedPongs = 0
reloadRenderer()
return
}
missedPongs++
mainWindow.webContents.send('watchdog:ping')
}, 30_000)
}
// IPC Handlers // IPC Handlers
ipcMain.handle('get-version', () => { ipcMain.handle('get-version', () => {
return app.getVersion() return app.getVersion()
}) })
// Watchdog pong: renderer confirms it's alive
ipcMain.handle('watchdog:pong', () => {
missedPongs = 0
})
/** /**
* Get runtime configuration from environment variables * Get runtime configuration from environment variables
* This allows configuration to be set at runtime (not baked in at build time) * This allows configuration to be set at runtime (not baked in at build time)
@ -343,6 +401,7 @@ app.whenReady().then(() => {
} }
createWindow() createWindow()
startWatchdog()
app.on('activate', () => { app.on('activate', () => {
// macOS: re-create window when dock icon clicked // macOS: re-create window when dock icon clicked
@ -353,6 +412,7 @@ app.whenReady().then(() => {
}) })
app.on('window-all-closed', () => { app.on('window-all-closed', () => {
if (heartbeatInterval) clearInterval(heartbeatInterval)
closeDatabase() closeDatabase()
// Quit on all platforms (ATM doesn't need macOS dock behavior) // Quit on all platforms (ATM doesn't need macOS dock behavior)
app.quit() app.quit()

View file

@ -86,6 +86,12 @@ contextBridge.exposeInMainWorld('electronAPI', {
ipcRenderer.on('hal:error', (_event, error) => callback(error)) ipcRenderer.on('hal:error', (_event, error) => callback(error))
}, },
// Watchdog heartbeat (main process → renderer → main process)
onWatchdogPing: (callback: () => void) => {
ipcRenderer.on('watchdog:ping', () => callback())
},
watchdogPong: (): Promise<void> => ipcRenderer.invoke('watchdog:pong'),
// Platform info // Platform info
platform: process.platform, platform: process.platform,
}) })
@ -126,6 +132,8 @@ declare global {
onHalBillInserted: (callback: (denomination: number) => void) => void onHalBillInserted: (callback: (denomination: number) => void) => void
onHalBillRejected: (callback: (reason: string) => void) => void onHalBillRejected: (callback: (reason: string) => void) => void
onHalError: (callback: (error: string) => void) => void onHalError: (callback: (error: string) => void) => void
onWatchdogPing: (callback: () => void) => void
watchdogPong: () => Promise<void>
platform: NodeJS.Platform platform: NodeJS.Platform
} }
} }

View file

@ -26,6 +26,13 @@ const formattedBtcPrice = computed(() => {
}) })
onMounted(async () => { onMounted(async () => {
// Watchdog: respond to heartbeat pings from main process
if (window.electronAPI?.onWatchdogPing) {
window.electronAPI.onWatchdogPing(() => {
window.electronAPI!.watchdogPong()
})
}
try { try {
if (isElectron) { if (isElectron) {
await atmStore.initializeForProduction() await atmStore.initializeForProduction()

View file

@ -60,6 +60,8 @@ declare global {
onHalBillInserted: (callback: (denomination: number) => void) => void onHalBillInserted: (callback: (denomination: number) => void) => void
onHalBillRejected: (callback: (reason: string) => void) => void onHalBillRejected: (callback: (reason: string) => void) => void
onHalError: (callback: (error: string) => void) => void onHalError: (callback: (error: string) => void) => void
onWatchdogPing: (callback: () => void) => void
watchdogPong: () => Promise<void>
platform: NodeJS.Platform platform: NodeJS.Platform
} }
} }