aiolabs/payroll
9
0
Fork 0
Code Issues 6 Pull requests Projects Releases Packages Wiki Activity Actions

feat: payout ledger with bounded retry

Browse source 
Closes the gap the scheduler commit left open: a failed payday was retried
indefinitely with nothing but a log line to show for it.

Every attempt — paid, skipped and failed — is now written to
payroll.payouts and exposed at GET /api/v1/payouts. Failures are in the
ledger, not only in the log, because "why did nobody get paid on the 1st"
is the question the ledger exists to answer.

Ledger rows are self-describing: each copies the terms in force at the time
(amount, currency, both wallets) instead of pointing at the contract, and
there is no foreign key to contracts. A payout has to still read correctly
after its contract is edited, and deleting a contract must not take its
history with it. This is the one place duplicating a contract field is
right — the contract holds what is true now, a payout holds what was true
then.

Retries are bounded at 5 attempts per period, after which the contract is
paused rather than the period abandoned. A payday that cannot be funded is
a fact somebody has to act on; dropping it silently is the one outcome
payroll must never produce. Pausing does not advance the position, so
resuming after topping up retries the same payday. The attempt count is
derived from the ledger rather than a column on the contract, so it
survives a restart and stays auditable.

Also restores the explanatory comments on the broad `except Exception`
handlers, which ruff's RUF100 stripped along with their now-unused noqa
directives.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018jy52j9GRZ6XKa1Zt21LLj
This commit is contained in:
Padreug 2026-08-31 13:49:46 +02:00
commit 8b054d27ea
9 changed files with 389 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

116
services.py
View file

@ -22,10 +22,18 @@ from datetime import date, datetime, timedelta, timezone
from lnbits.core.crud import get_wallet
from lnbits.core.services import create_invoice, pay_invoice
from lnbits.helpers import urlsafe_short_hash
from loguru import logger
from . import crud
from .models import TERMINAL_STATUSES, Contract, ContractStatus, Frequency
from .models import (
TERMINAL_STATUSES,
Contract,
ContractStatus,
Frequency,
Payout,
PayoutStatus,
)
# Frequencies that are an exact number of days: no calendar involved, so no
# clamping is possible or needed.
@ -46,6 +54,12 @@ _MONTH_STEP = {
# Protects against a mistyped start date turning into a hundred transfers.
MAX_CATCH_UP_PERIODS = 12
# How many times one period may fail before the contract is paused for the
# operator to look at. Retrying an underfunded wallet forever is not
# resilience, it is a log the operator learns to ignore — and a missed
# payday deserves to be visible in the UI, not buried in journalctl.
MAX_PERIOD_ATTEMPTS = 5
# ---------------------------------------------------------------------------
# Schedule math (pure)
@ -141,7 +155,7 @@ class PeriodOutcome:
index: int
payday: date
status: str # "paid" | "skipped" | "failed"
status: PayoutStatus
detail: str = ""
amount_msat: int | None = None
payment_hash: str | None = None
@ -154,7 +168,7 @@ class PeriodOutcome:
what makes the next tick retry the same payday rather than dropping
it.
"""
return self.status in ("paid", "skipped")
return self.status in (PayoutStatus.paid, PayoutStatus.skipped)
# One lock per contract id. The scheduler is a single task, but an operator
@ -207,11 +221,13 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
internal=True,
extra=extra,
)
# Broad: pricing, wallet limits and funding-source errors all surface
# here, and every one of them is a failed period rather than a crash.
except Exception as exc:
return PeriodOutcome(
index=index,
payday=payday,
status="failed",
status=PayoutStatus.failed,
detail=f"could not raise invoice: {exc}",
)
@ -223,7 +239,7 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
return PeriodOutcome(
index=index,
payday=payday,
status="failed",
status=PayoutStatus.failed,
detail="source wallet not found",
amount_msat=amount_msat,
)
@ -231,7 +247,7 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
return PeriodOutcome(
index=index,
payday=payday,
status="failed",
status=PayoutStatus.failed,
detail=(
f"insufficient balance in source wallet: "
f"{source.balance_msat // 1000} sat available, "
@ -248,11 +264,11 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
tag="payroll",
extra=extra,
)
except Exception as exc:
except Exception as exc: # a failed payment is an expected outcome
return PeriodOutcome(
index=index,
payday=payday,
status="failed",
status=PayoutStatus.failed,
detail=f"payment failed: {exc}",
amount_msat=amount_msat,
payment_hash=invoice.payment_hash,
@ -261,7 +277,7 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
return PeriodOutcome(
index=index,
payday=payday,
status="paid",
status=PayoutStatus.paid,
amount_msat=amount_msat,
payment_hash=invoice.payment_hash,
)
@ -269,9 +285,12 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc
async def run_due_periods(
contract: Contract, today: date | None = None
) -> list[PeriodOutcome]:
) -> list[Payout]:
"""Settle every period this contract owes as of `today`.
Every attempt — paid, skipped or failed — is written to the ledger, and
the ledger rows are what this returns.
Stops at the first failure so a backlog cannot pay periods out of order:
if period 3 could not be funded, period 4 waits for it. Both will be
retried on the next tick, because a failed period does not advance the
@ -287,22 +306,79 @@ async def run_due_periods(
return []
contract = fresh
outcomes: list[PeriodOutcome] = []
payouts: list[Payout] = []
for index in due_period_indices(contract, today):
payday = occurrence_on(
parse_start_date(contract), contract.frequency, index
)
outcome = await _settle(contract, index, payday)
outcomes.append(outcome)
payout = await _record(contract, outcome)
payouts.append(payout)
if not outcome.consumed:
_maybe_pause_after_repeated_failure(contract, payout)
break
contract.periods_done = index + 1
if outcomes:
if payouts:
_maybe_complete(contract)
await crud.update_contract(contract)
return outcomes
return payouts
async def _record(contract: Contract, outcome: PeriodOutcome) -> Payout:
"""Write one attempt to the ledger.
The attempt number is derived by counting this period's prior failures
in the ledger itself rather than from a counter on the contract, so it
survives a restart and the rows that produced it are right there to
audit. A paid or skipped period is never retried, so it is always
attempt 1 as far as that outcome is concerned.
"""
attempt = 1
if outcome.status == PayoutStatus.failed:
attempt = await crud.count_period_failures(contract.id, outcome.index) + 1
return await crud.create_payout(
Payout(
id=urlsafe_short_hash()[:12],
contract_id=contract.id,
period_index=outcome.index,
payday=outcome.payday.isoformat(),
status=PayoutStatus(outcome.status),
attempt=attempt,
amount_msat=outcome.amount_msat,
# Terms in force at the time — a ledger row has to still read
# correctly after the contract is edited or deleted.
amount=contract.amount,
currency=contract.currency,
employee_id=contract.employee_id,
employee_wallet=contract.employee_wallet,
source_wallet=contract.source_wallet,
payment_hash=outcome.payment_hash,
detail=outcome.detail,
)
)
def _maybe_pause_after_repeated_failure(contract: Contract, payout: Payout) -> None:
"""Give up retrying a period and hand it to the operator.
The contract is paused rather than the period abandoned: a payday that
cannot be funded is a fact somebody needs to act on, and silently
dropping it is the one outcome payroll must never produce. Resuming
after fixing the funding retries the same period, because pausing did
not advance the position.
"""
if payout.attempt < MAX_PERIOD_ATTEMPTS:
return
contract.status = ContractStatus.paused
logger.error(
f"payroll: contract {contract.id} paused after {payout.attempt} failed "
f"attempts at period {payout.period_index} ({payout.payday}): "
f"{payout.detail}"
)
async def _settle(contract: Contract, index: int, payday: date) -> PeriodOutcome:
@ -322,12 +398,12 @@ async def _settle(contract: Contract, index: int, payday: date) -> PeriodOutcome
return PeriodOutcome(
index=index,
payday=payday,
status="skipped",
status=PayoutStatus.skipped,
detail="payday predates the contract and backfill is off",
)
outcome = await pay_period(contract, index, payday)
if outcome.status == "paid":
if outcome.status == PayoutStatus.paid:
logger.success(
f"payroll: contract {contract.id} period {index} paid "
f"{(outcome.amount_msat or 0) // 1000} sat to "
@ -336,12 +412,16 @@ async def _settle(contract: Contract, index: int, payday: date) -> PeriodOutcome
else:
logger.warning(
f"payroll: contract {contract.id} period {index} "
f"{outcome.status}: {outcome.detail}"
f"{outcome.status.value}: {outcome.detail}"
)
return outcome
def _maybe_complete(contract: Contract) -> None:
# Only an active contract completes: a period that just exhausted the
# retry budget has paused this one, and that has to stick.
if contract.status != ContractStatus.active:
return
if (
contract.total_periods is not None
and contract.periods_done >= contract.total_periods
@ -359,7 +439,7 @@ async def tick(today: date | None = None) -> None:
for contract in contracts:
try:
await run_due_periods(contract, today)
except Exception as exc:
except Exception as exc: # one bad row must not stop the rest of payroll
logger.error(f"payroll: contract {contract.id} tick failed: {exc}")