diff --git a/docs/operations.md b/docs/operations.md index 5a9b8fd..813e753 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -42,6 +42,42 @@ left on the employee's wallet. It was never settled and expires on its own; that is the deliberate price of not pricing the period a second time just to run a balance check. +## Pricing a back-dated period + +A period whose payday is **today or ahead** is priced by LNbits itself — +`create_invoice(amount, currency)` does the conversion, and nothing here +touches the network. Every pricing mode agrees in that case. + +A period whose payday is **in the past** is where the modes diverge: + +| mode | converts at | use when | +|---|---|---| +| `payday` (default) | BTC's value on the payday | the expense belongs to that date | +| `current` | today's rate | the debt reads "we owe EUR 800, whenever it settles" | +| `manual` | a rate you state (`100000` EUR/BTC) | the figure was agreed, not looked up | + +Set per contract; overridable per payout via `pricing_mode` / `manual_rate` +on `pay-now`, which is how you enter one back-dated payment without editing +the contract. + +Historical rates come from `rates.py` — Kraken's daily OHLC series first +(one request covers every date in a backfill), CoinGecko by date as the +fallback for currencies Kraken does not quote. **Not** from LNbits' own +exchange providers: those are spot tickers with no date parameter, and the +rate history behind the admin chart is RAM-only, single-currency and wiped +on restart. + +Where payroll converts, it hands `create_invoice` a **sat** amount, because +`create_invoice` always prices fiat itself and cannot be told a rate. The +rate used and its source are recorded on the payout row either way — in +`current` mode read back off the invoice LNbits priced, not recomputed. + +**A rate that cannot be established fails the period.** There is deliberately +no fallback to today's rate: one that moved 30% since the payday would pay +30% off and record nothing about it. The failure goes through the same +retry-then-pause path as an underfunded wallet, and names the date and +currency it could not price. + ## What a failure does A failed period does **not** advance `periods_done`, so the next tick diff --git a/migrations.py b/migrations.py index 282e1fc..b82e7d1 100644 --- a/migrations.py +++ b/migrations.py @@ -88,3 +88,27 @@ async def m002_payouts(db): "CREATE INDEX payroll.idx_payouts_employee_wallet " "ON payouts (employee_wallet);" ) + + +async def m003_pricing_mode(db): + """Let a back-dated period be priced at the day it was due. + + Until now every payout converted at whatever the rate was when it + settled, which silently misprices a period paid late. The contract now + carries how to convert, and each payout records the rate it actually + used plus where that rate came from — so a figure in the ledger can be + explained months later instead of merely trusted. + + Defaults preserve existing behaviour for anything already in flight: + every live payday is today or ahead, and for those all three modes agree + on using LNbits' own live pricing. + """ + await db.execute( + "ALTER TABLE payroll.contracts " + "ADD COLUMN pricing_mode TEXT NOT NULL DEFAULT 'payday';" + ) + await db.execute("ALTER TABLE payroll.contracts ADD COLUMN manual_rate REAL;") + await db.execute("ALTER TABLE payroll.payouts ADD COLUMN rate REAL;") + await db.execute( + "ALTER TABLE payroll.payouts ADD COLUMN rate_source TEXT NOT NULL DEFAULT '';" + ) diff --git a/models.py b/models.py index 15ccaac..524224f 100644 --- a/models.py +++ b/models.py @@ -55,6 +55,25 @@ class Frequency(str, Enum): yearly = "yearly" +class PricingMode(str, Enum): + """How a period's fiat amount becomes a sat amount. + + Only ever matters for a payday in the past — for a payday that is today + or ahead, all three agree and LNbits' own live pricing is used. + """ + + # Convert at what BTC was worth on the payday itself. The default: a + # back-dated period is priced as of when it was due, not when somebody + # got round to paying it. + payday = "payday" + # Convert at today's rate. Correct when the obligation is understood as + # "we owe them EUR 800, whenever it settles". + current = "current" + # Convert at a rate the operator states outright (e.g. 100000 EUR/BTC), + # for a figure that was agreed rather than looked up. + manual = "manual" + + class ContractStatus(str, Enum): active = "active" # eligible for payout on every scheduler tick paused = "paused" # keeps its schedule position, pays nothing @@ -101,6 +120,19 @@ class CreateContract(BaseModel): # months of back-pay in one tick. backfill: bool = False + # How to price a back-dated period. Irrelevant for a sat-denominated + # contract, and irrelevant for any payday that is not in the past. + pricing_mode: PricingMode = PricingMode.payday + # Units of `currency` per whole BTC. Required by, and only read under, + # PricingMode.manual. + manual_rate: float | None = None + + @validator("manual_rate") + def _positive_rate(cls, v: float | None) -> float | None: + if v is not None and v <= 0: + raise ValueError("manual_rate must be positive") + return v + @validator("start_date") def _valid_date(cls, v: str) -> str: datetime.strptime(v, "%Y-%m-%d") # raises for anything malformed @@ -159,6 +191,8 @@ class UpdateContract(BaseModel): total_periods: int | None = None label: str | None = None memo: str | None = None + pricing_mode: PricingMode | None = None + manual_rate: float | None = None @validator("amount") def _positive_amount(cls, v: float | None) -> float | None: @@ -239,6 +273,14 @@ class Payout(BaseModel): payment_hash: str | None = None detail: str = "" + # Units of `currency` per whole BTC actually used for this period, and + # which of the pricing modes produced it. For a live conversion the rate + # is read back off the invoice LNbits priced rather than recomputed, so + # the row records what happened and not an approximation of it. Null on + # a sat contract, and on a failure that never got as far as pricing. + rate: float | None = None + rate_source: str = "" # "current" | "payday" | "manual" | "" (sat / none) + created_at: datetime = Field(default_factory=_now) @property diff --git a/rates.py b/rates.py new file mode 100644 index 0000000..a0a9910 --- /dev/null +++ b/rates.py @@ -0,0 +1,140 @@ +"""Historical BTC rates, for pricing a payout at the day it was due. + +LNbits cannot answer this. Its eight exchange providers are spot tickers +with no date parameter, and the rate history behind the admin chart lives +in `TransientSettings` — RAM only, wiped on restart, capped at +`lnbits_exchange_history_size` points, and collected for a single currency +(`lnbits_default_accounting_currency`). Useful for a monitoring graph; +useless for "what was BTC worth on the 15th". + +So payroll asks a date-aware API directly. + +**Kraken first.** One `OHLC?interval=1440` request returns ~720 daily +candles, so a twelve-period backfill costs a single call and every date is +served from the parsed series. It is also already one of the providers +LNbits itself trusts. + +**CoinGecko second**, for currencies Kraken lists no pair for. It is +addressed by date rather than by series, so it costs one request per date — +fine as a fallback, bad as a default against free-tier rate limits. + +Both return None rather than raising. A rate that cannot be established is +a payout that must not happen: the caller records a failed period and lets +a human decide, which is the same thing an underfunded wallet does. The one +outcome this module must never produce is a plausible-looking wrong number. +""" + +from datetime import date, datetime, timedelta, timezone + +import httpx +from loguru import logger + +# Kraken quotes bitcoin as XBT. Its response key is *not* predictable +# ("XXBTZEUR", "XBTUSDT", …), so the series is read positionally instead of +# by reconstructing the name. +_KRAKEN_URL = "https://api.kraken.com/0/public/OHLC" +_COINGECKO_URL = "https://api.coingecko.com/api/v3/coins/bitcoin/history" + +_TIMEOUT = 15.0 + +# A daily series is only appended to once a day, so re-fetching more often +# than this buys nothing. Keeps a backfill plus every retry to one request. +_CACHE_TTL = timedelta(hours=6) + +# currency -> (fetched_at, {day: close}) +_series_cache: dict[str, tuple[datetime, dict[date, float]]] = {} + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +async def _kraken_series(currency: str) -> dict[date, float]: + """Daily closes for XBT/, keyed by UTC day. Empty if the pair + is unknown or the request fails.""" + cached = _series_cache.get(currency) + if cached and _now() - cached[0] < _CACHE_TTL: + return cached[1] + + params = {"pair": f"XBT{currency}", "interval": "1440"} + async with httpx.AsyncClient() as client: + response = await client.get(_KRAKEN_URL, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json() + + if payload.get("error"): + # Unknown pair is the ordinary case here, not an incident — it just + # means this currency needs the fallback. + logger.debug(f"payroll: kraken has no XBT{currency} series: {payload['error']}") + return {} + + result = payload.get("result", {}) + keys = [k for k in result if k != "last"] + if not keys: + return {} + + series = { + datetime.fromtimestamp(candle[0], tz=timezone.utc).date(): float(candle[4]) + for candle in result[keys[0]] + } + _series_cache[currency] = (_now(), series) + return series + + +async def _coingecko_rate(day: date, currency: str) -> float | None: + """Close for one specific day. One request per date — fallback only.""" + params = {"date": day.strftime("%d-%m-%Y"), "localization": "false"} + async with httpx.AsyncClient() as client: + response = await client.get(_COINGECKO_URL, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json() + + price = payload.get("market_data", {}).get("current_price", {}) + value = price.get(currency.lower()) + return float(value) if value else None + + +async def historical_btc_rate(day: date, currency: str) -> float | None: + """Units of `currency` per whole BTC on `day`, or None if unknowable. + + None is a legitimate answer — the date may predate the provider's + history, the currency may not be quoted anywhere, or the API may be + down. Callers must treat it as "do not pay", never as "use some other + number". + """ + currency = currency.upper() + if currency == "SAT": + raise ValueError("sat-denominated contracts need no rate") + + try: + series = await _kraken_series(currency) + if day in series: + return series[day] + if series: + logger.debug( + f"payroll: {day} outside kraken's XBT{currency} range " + f"({min(series)}..{max(series)}), trying coingecko" + ) + except Exception as exc: # network, malformed payload, anything + logger.warning(f"payroll: kraken rate lookup failed for {currency}: {exc}") + + try: + rate = await _coingecko_rate(day, currency) + if rate: + return rate + except Exception as exc: + logger.warning(f"payroll: coingecko rate lookup failed for {currency}: {exc}") + + logger.warning(f"payroll: no historical rate for {currency} on {day}") + return None + + +def sats_for(amount: float, rate: float) -> int: + """`amount` of fiat in sats, at `rate` units-per-BTC. + + Rounded to the nearest sat rather than truncated: over a year of + paydays, always rounding down would quietly shortchange the employee. + """ + if rate <= 0: + raise ValueError("rate must be positive") + return round(amount / rate * 100_000_000) diff --git a/services.py b/services.py index f167e8c..f1123a0 100644 --- a/services.py +++ b/services.py @@ -33,7 +33,9 @@ from .models import ( Frequency, Payout, PayoutStatus, + PricingMode, ) +from .rates import historical_btc_rate, sats_for # Frequencies that are an exact number of days: no calendar involved, so no # clamping is possible or needed. @@ -187,6 +189,8 @@ class PeriodOutcome: detail: str = "" amount_msat: int | None = None payment_hash: str | None = None + rate: float | None = None + rate_source: str = "" @property def consumed(self) -> bool: @@ -214,12 +218,86 @@ def _lock_for(contract_id: str) -> asyncio.Lock: return lock +def _today() -> date: + return datetime.now(timezone.utc).date() + + def payout_memo(contract: Contract, payday: date) -> str: label = contract.memo or contract.label or "Payroll" return f"{label} — {payday.isoformat()}" -async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutcome: +class PricingError(ValueError): + """A period's sat amount could not be established. + + Deliberately fatal for that period. The alternative — falling back to + today's rate when the payday's rate is unavailable — pays a materially + different amount than intended and hides it, which is exactly the class + of error nobody finds until an audit. + """ + + +@dataclass +class Price: + """What to hand `create_invoice`, plus the provenance to record. + + When `currency` is the contract's own, LNbits does the conversion and + `rate` is filled in afterwards from the invoice it priced — read back + rather than recomputed, so the ledger records what actually happened. + """ + + amount: float + currency: str + rate: float | None + source: str + + +async def resolve_price( + contract: Contract, + payday: date, + today: date, + mode: PricingMode | None = None, + manual_rate: float | None = None, +) -> Price: + """Decide what a period is worth, and say where the number came from. + + `mode`/`manual_rate` override the contract's own settings for a single + call, which is how an operator prices one off-cycle payout differently + without editing the contract. + """ + if contract.currency.lower() == "sat": + return Price(contract.amount, "sat", None, "") + + mode = mode or contract.pricing_mode + manual_rate = manual_rate or contract.manual_rate + + # A payday that is today or ahead needs no history: every mode agrees, + # and LNbits' own live pricing is the freshest source available. + if payday >= today or mode == PricingMode.current: + return Price(contract.amount, contract.currency, None, "current") + + if mode == PricingMode.manual: + if not manual_rate: + raise PricingError("manual pricing selected but no rate given") + return Price( + sats_for(contract.amount, manual_rate), "sat", manual_rate, "manual" + ) + + rate = await historical_btc_rate(payday, contract.currency) + if rate is None: + raise PricingError( + f"no historical {contract.currency} rate available for {payday}" + ) + return Price(sats_for(contract.amount, rate), "sat", rate, "payday") + + +async def pay_period( + contract: Contract, + index: int, + payday: date, + mode: PricingMode | None = None, + manual_rate: float | None = None, +) -> PeriodOutcome: """Move one period's money from the source wallet to the employee wallet. An internal invoice on the destination wallet, paid from the source @@ -240,11 +318,23 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc "payday": payday.isoformat(), } + try: + price = await resolve_price(contract, payday, _today(), mode, manual_rate) + except (PricingError, ValueError) as exc: + return PeriodOutcome( + index=index, + payday=payday, + status=PayoutStatus.failed, + detail=str(exc), + ) + if price.source: + extra["rate_source"] = price.source + try: invoice = await create_invoice( wallet_id=contract.employee_wallet, - amount=contract.amount, - currency=contract.currency, + amount=price.amount, + currency=price.currency, memo=memo, internal=True, extra=extra, @@ -262,6 +352,13 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc # invoice.amount is msat and is now the canonical figure for this period. amount_msat = invoice.amount + # For a live conversion LNbits stashes the rate it used on the payment; + # read it back rather than recomputing, so the ledger records the number + # that was actually applied. + rate = price.rate + if rate is None and price.source == "current": + rate = (invoice.extra or {}).get("btc_rate") + source = await get_wallet(contract.source_wallet) if not source: return PeriodOutcome( @@ -270,6 +367,8 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc status=PayoutStatus.failed, detail="source wallet not found", amount_msat=amount_msat, + rate=rate, + rate_source=price.source, ) if source.balance_msat < amount_msat: return PeriodOutcome( @@ -282,6 +381,8 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc f"{amount_msat // 1000} sat required" ), amount_msat=amount_msat, + rate=rate, + rate_source=price.source, ) try: @@ -300,6 +401,8 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc detail=f"payment failed: {exc}", amount_msat=amount_msat, payment_hash=invoice.payment_hash, + rate=rate, + rate_source=price.source, ) return PeriodOutcome( @@ -308,6 +411,8 @@ async def pay_period(contract: Contract, index: int, payday: date) -> PeriodOutc status=PayoutStatus.paid, amount_msat=amount_msat, payment_hash=invoice.payment_hash, + rate=rate, + rate_source=price.source, ) @@ -324,7 +429,7 @@ async def run_due_periods( retried on the next tick, because a failed period does not advance the contract's position. """ - today = today or datetime.now(timezone.utc).date() + today = today or _today() async with _lock_for(contract.id): # Re-read under the lock: a manual run may have moved the position @@ -386,6 +491,8 @@ async def _record(contract: Contract, outcome: PeriodOutcome) -> Payout: source_wallet=contract.source_wallet, payment_hash=outcome.payment_hash, detail=outcome.detail, + rate=outcome.rate, + rate_source=outcome.rate_source, ) ) @@ -522,7 +629,7 @@ def resume( contract.status = ContractStatus.active if not catch_up: - today = today or datetime.now(timezone.utc).date() + today = today or _today() skipped_to = fast_forward_index(contract, today) if skipped_to != contract.periods_done: logger.info( @@ -551,7 +658,11 @@ def cancel(contract: Contract) -> Contract: # --------------------------------------------------------------------------- -async def pay_now(contract: Contract) -> Payout: +async def pay_now( + contract: Contract, + mode: PricingMode | None = None, + manual_rate: float | None = None, +) -> Payout: """Settle the contract's next period immediately, whatever the calendar says. @@ -565,6 +676,10 @@ async def pay_now(contract: Contract) -> Payout: because `_settle`'s back-dated skip exists to stop a *new* contract firing surprise back-pay. An operator explicitly asking to pay a period is not a surprise. + + `mode`/`manual_rate` price this one payout differently without editing + the contract — the case an operator hits when entering a payment that + happened weeks ago at a rate they already know. """ async with _lock_for(contract.id): fresh = await crud.get_contract(contract.id) @@ -582,9 +697,9 @@ async def pay_now(contract: Contract) -> Payout: index = contract.periods_done payday = occurrence_on(parse_start_date(contract), contract.frequency, index) - today = datetime.now(timezone.utc).date() + today = _today() - outcome = await pay_period(contract, index, payday) + outcome = await pay_period(contract, index, payday, mode, manual_rate) if payday > today and outcome.status == PayoutStatus.paid: outcome.detail = f"paid early on {today.isoformat()} (due {payday})" diff --git a/static/js/index.js b/static/js/index.js index abf6f95..4f88d7a 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -15,6 +15,12 @@ const FREQUENCIES = [ {value: 'yearly', label: 'Yearly'} ] +const PRICING_MODES = [ + {value: 'payday', label: "Rate on the payday"}, + {value: 'current', label: "Rate at settlement"}, + {value: 'manual', label: 'Rate I enter'} +] + const STATUS_COLOUR = { active: 'positive', paused: 'warning', @@ -45,7 +51,11 @@ function emptyContract() { total_periods: 12, label: '', memo: '', - backfill: false + backfill: false, + // Only ever consulted for a payday in the past; for a payday today or + // ahead every mode collapses to LNbits' own live pricing. + pricing_mode: 'payday', + manual_rate: null } } @@ -69,6 +79,14 @@ window.app = Vue.createApp({ frequencyOptions: FREQUENCIES, contractDialog: {show: false, data: emptyContract(), saving: false}, + payNowDialog: { + show: false, + contract: null, + pricing_mode: 'payday', + manual_rate: null, + loading: false + }, + pricingModes: PRICING_MODES, // Live calendar of whatever is currently typed into the dialog. The // cheapest moment to notice a wrong start date or frequency is before // anything is saved. @@ -121,6 +139,7 @@ window.app = Vue.createApp({ align: 'left', field: 'contract_id' }, + {name: 'rate', label: 'Rate used', align: 'right', field: 'rate'}, {name: 'attempt', label: 'Try', align: 'center', field: 'attempt'}, {name: 'detail', label: 'Detail', align: 'left', field: 'detail'} ] @@ -331,7 +350,9 @@ window.app = Vue.createApp({ total_periods: d.open_ended ? null : Number(d.total_periods), label: d.label, memo: d.memo, - backfill: d.backfill + backfill: d.backfill, + pricing_mode: d.pricing_mode, + manual_rate: d.pricing_mode === 'manual' ? Number(d.manual_rate) : null }) this.contractDialog.show = false Quasar.Notify.create({type: 'positive', message: 'Payroll line created'}) @@ -354,7 +375,9 @@ window.app = Vue.createApp({ open_ended: contract.total_periods === null, total_periods: contract.total_periods, label: contract.label, - memo: contract.memo + memo: contract.memo, + pricing_mode: contract.pricing_mode, + manual_rate: contract.manual_rate } } }, @@ -372,7 +395,9 @@ window.app = Vue.createApp({ frequency: d.frequency, total_periods: d.open_ended ? null : Number(d.total_periods), label: d.label, - memo: d.memo + memo: d.memo, + pricing_mode: d.pricing_mode, + manual_rate: d.pricing_mode === 'manual' ? Number(d.manual_rate) : null } ) this.editDialog.show = false @@ -416,33 +441,49 @@ window.app = Vue.createApp({ }, payNow(contract) { - LNbits.utils - .confirmDialog( - `Pay the next period of "${contract.label || contract.id}" now? ` + - 'It is consumed even if its payday has not arrived yet.' + // A dialog rather than a confirm: paying a back-dated period is + // exactly when the operator may want a different rate than the + // contract's own, and that choice has to be made before it settles. + this.payNowDialog = { + show: true, + contract, + pricing_mode: contract.pricing_mode || 'payday', + manual_rate: contract.manual_rate, + loading: false + } + }, + + async confirmPayNow() { + const d = this.payNowDialog + const params = new URLSearchParams({pricing_mode: d.pricing_mode}) + if (d.pricing_mode === 'manual' && d.manual_rate) { + params.append('manual_rate', String(d.manual_rate)) + } + d.loading = true + try { + const {data} = await LNbits.api.request( + 'POST', + `${API}/contracts/${d.contract.id}/pay-now?${params.toString()}` ) - .onOk(async () => { - try { - const {data} = await LNbits.api.request( - 'POST', - `${API}/contracts/${contract.id}/pay-now` - ) - // A refused payout still returns a ledger row — surfacing its - // reason is the whole point of triggering one by hand. - Quasar.Notify.create({ - type: data.status === 'paid' ? 'positive' : 'negative', - message: - data.status === 'paid' - ? `Paid ${Math.round(data.amount_msat / 1000)} sat` - : `Payout ${data.status}: ${data.detail}`, - timeout: data.status === 'paid' ? 3000 : 8000 - }) - this.getContracts() - this.getPayouts() - } catch (err) { - this._err(err, 'Could not pay now') - } + // A refused payout still returns a ledger row — surfacing its + // reason is the whole point of triggering one by hand. + Quasar.Notify.create({ + type: data.status === 'paid' ? 'positive' : 'negative', + message: + data.status === 'paid' + ? `Paid ${Math.round(data.amount_msat / 1000)} sat` + + (data.rate ? ` at ${data.rate.toLocaleString()} / BTC` : '') + : `Payout ${data.status}: ${data.detail}`, + timeout: data.status === 'paid' ? 4000 : 9000 }) + d.show = false + this.getContracts() + this.getPayouts() + } catch (err) { + this._err(err, 'Could not pay now') + } finally { + d.loading = false + } }, deleteContract(contract) { diff --git a/templates/payroll/index.html b/templates/payroll/index.html index d28a7a2..0e5ca71 100644 --- a/templates/payroll/index.html +++ b/templates/payroll/index.html @@ -151,6 +151,18 @@ + +